From fa5f31ddf9de030dad82e65be1d77275f4daa345 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sat, 21 Feb 2015 09:00:34 +0100 Subject: [PATCH] Move stream implementation to a specific source file --- Makefile.am | 3 + cmdline/mkstream.c | 3 +- cmdline/state.c | 1 + cmdline/stream.c | 899 +++++++++++++++++++++++++++++++++++++++++++++ cmdline/stream.h | 443 ++++++++++++++++++++++ cmdline/util.c | 876 ------------------------------------------- cmdline/util.h | 466 ++--------------------- 7 files changed, 1373 insertions(+), 1318 deletions(-) create mode 100644 cmdline/stream.c create mode 100644 cmdline/stream.h diff --git a/Makefile.am b/Makefile.am index cf47ccab..8b2d7ab1 100644 --- a/Makefile.am +++ b/Makefile.am @@ -16,6 +16,7 @@ snapraid_SOURCES = \ tommyds/tommy.c \ cmdline/snapraid.c \ cmdline/util.c \ + cmdline/stream.c \ cmdline/support.c \ cmdline/elem.c \ cmdline/state.c \ @@ -67,6 +68,7 @@ noinst_HEADERS = \ cmdline/portable.h \ cmdline/snapraid.h \ cmdline/util.h \ + cmdline/stream.h \ cmdline/support.h \ cmdline/elem.h \ cmdline/state.h \ @@ -92,6 +94,7 @@ mkstream_SOURCES = \ cmdline/mingw.c \ cmdline/support.c \ cmdline/util.c \ + cmdline/stream.c \ raid/memory.c EXTRA_DIST = \ diff --git a/cmdline/mkstream.c b/cmdline/mkstream.c index ee72e79e..ae3cfb99 100644 --- a/cmdline/mkstream.c +++ b/cmdline/mkstream.c @@ -17,8 +17,7 @@ #include "portable.h" -#include "support.h" -#include "util.h" +#include "stream.h" #define STREAM_MAX 8 #define BUFFER_MAX 64 diff --git a/cmdline/state.c b/cmdline/state.c index 79d1cdcc..88bdc61f 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -23,6 +23,7 @@ #include "state.h" #include "support.h" #include "parity.h" +#include "stream.h" #include "raid/raid.h" #include "raid/cpu.h" diff --git a/cmdline/stream.c b/cmdline/stream.c new file mode 100644 index 00000000..9f9059d8 --- /dev/null +++ b/cmdline/stream.c @@ -0,0 +1,899 @@ +/* + * Copyright (C) 2011 Andrea Mazzoleni + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "portable.h" + +#include "support.h" +#include "util.h" +#include "stream.h" + +/****************************************************************************/ +/* hex conversion table */ + +static char strhexset[16] = "0123456789abcdef"; + +static unsigned strdecset[256] = +{ + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 +}; + +/****************************************************************************/ +/* stream */ + +unsigned STREAM_SIZE = 1024 * 64; + +STREAM* sopen_read(const char* file) +{ +#if HAVE_POSIX_FADVISE + int ret; +#endif + STREAM* s = malloc_nofail(sizeof(STREAM)); + + s->handle_size = 1; + s->handle = malloc_nofail(sizeof(struct stream_handle)); + + pathcpy(s->handle[0].path, sizeof(s->handle[0].path), file); + s->handle[0].f = open(file, O_RDONLY | O_BINARY | O_SEQUENTIAL); + if (s->handle[0].f == -1) { + free(s->handle); + free(s); + return 0; + } + +#if HAVE_POSIX_FADVISE + /* advise sequential access */ + ret = posix_fadvise(s->handle[0].f, 0, 0, POSIX_FADV_SEQUENTIAL); + if (ret != 0) { + /* LCOV_EXCL_START */ + close(s->handle[0].f); + free(s->handle); + free(s); + return 0; + /* LCOV_EXCL_STOP */ + } +#endif + + s->buffer = malloc_nofail_test(STREAM_SIZE); + s->pos = s->buffer; + s->end = s->buffer; + s->state = STREAM_STATE_READ; + s->state_index = 0; + s->offset = 0; + s->offset_uncached = 0; + s->crc = 0; + s->crc_uncached = 0; + s->crc_stream = CRC_IV; + + return s; +} + +STREAM* sopen_multi_write(unsigned count) +{ + unsigned i; + + STREAM* s = malloc_nofail(sizeof(STREAM)); + + s->handle_size = count; + s->handle = malloc_nofail(count * sizeof(struct stream_handle)); + + for (i = 0; i < count; ++i) + s->handle[i].f = -1; + + s->buffer = malloc_nofail_test(STREAM_SIZE); + s->pos = s->buffer; + s->end = s->buffer + STREAM_SIZE; + s->state = STREAM_STATE_WRITE; + s->state_index = 0; + s->offset = 0; + s->offset_uncached = 0; + s->crc = 0; + s->crc_uncached = 0; + s->crc_stream = CRC_IV; + + return s; +} + +int sopen_multi_file(STREAM* s, unsigned i, const char* file) +{ +#if HAVE_POSIX_FADVISE + int ret; +#endif + int f; + + pathcpy(s->handle[i].path, sizeof(s->handle[i].path), file); + + f = open(file, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY | O_SEQUENTIAL, 0600); + if (f == -1) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ + } + +#if HAVE_POSIX_FADVISE + /* advise sequential access */ + ret = posix_fadvise(f, 0, 0, POSIX_FADV_SEQUENTIAL); + if (ret != 0) { + /* LCOV_EXCL_START */ + close(f); + return -1; + /* LCOV_EXCL_STOP */ + } +#endif + + s->handle[i].f = f; + + return 0; +} + +int sclose(STREAM* s) +{ + int fail = 0; + unsigned i; + + if (s->state == STREAM_STATE_WRITE) { + if (sflush(s) != 0) { + /* LCOV_EXCL_START */ + fail = 1; + /* LCOV_EXCL_STOP */ + } + } + + for (i = 0; i < s->handle_size; ++i) { + if (close(s->handle[i].f) != 0) { + /* LCOV_EXCL_START */ + fail = 1; + /* LCOV_EXCL_STOP */ + } + } + + free(s->handle); + free(s->buffer); + free(s); + + if (fail) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ + } + + return 0; +} + +int shandle(STREAM* s) +{ + if (!s->handle_size) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ + } + + return s->handle[0].f; +} + +int sfill(STREAM* s) +{ + ssize_t ret; + + if (s->state != STREAM_STATE_READ) { + /* LCOV_EXCL_START */ + return EOF; + /* LCOV_EXCL_STOP */ + } + + ret = read(s->handle[0].f, s->buffer, STREAM_SIZE); + + if (ret < 0) { + /* LCOV_EXCL_START */ + s->state = STREAM_STATE_ERROR; + return EOF; + /* LCOV_EXCL_STOP */ + } + if (ret == 0) { + s->state = STREAM_STATE_EOF; + return EOF; + } + + /* update the crc */ + s->crc_uncached = s->crc; + s->crc = crc32c(s->crc, s->buffer, ret); + + /* update the offset */ + s->offset_uncached = s->offset; + s->offset += ret; + + s->pos = s->buffer; + s->end = s->buffer + ret; + + return *s->pos++; +} + +int sflush(STREAM* s) +{ + ssize_t ret; + ssize_t size; + unsigned i; + + if (s->state != STREAM_STATE_WRITE) { + /* LCOV_EXCL_START */ + return EOF; + /* LCOV_EXCL_STOP */ + } + + size = s->pos - s->buffer; + if (!size) + return 0; + + /* update the crc */ + s->crc = crc32c(s->crc, s->buffer, size); + s->crc_uncached = s->crc; + + for (i = 0; i < s->handle_size; ++i) { + ret = write(s->handle[i].f, s->buffer, size); + + if (ret != size) { + /* LCOV_EXCL_START */ + s->state = STREAM_STATE_ERROR; + s->state_index = i; + return EOF; + /* LCOV_EXCL_STOP */ + } + } + + /* update the offset */ + s->offset += size; + s->offset_uncached = s->offset; + + s->pos = s->buffer; + + return 0; +} + +int64_t stell(STREAM* s) +{ + return s->offset_uncached + (s->pos - s->buffer); +} + +uint32_t scrc(STREAM*s) +{ + return crc32c(s->crc_uncached, s->buffer, s->pos - s->buffer); +} + +uint32_t scrc_stream(STREAM*s) +{ + return s->crc_stream ^ CRC_IV; +} + +int sgettok(STREAM* f, char* str, int size) +{ + char* i = str; + char* send = str + size; + int c; + + while (1) { + c = sgetc(f); + if (c == EOF) { + break; + } + if (c == ' ' || c == '\t') { + sungetc(c, f); + break; + } + if (c == '\n') { + /* remove ending carrige return to support the Windows CR+LF format */ + if (i != str && i[-1] == '\r') + --i; + sungetc(c, f); + break; + } + + *i++ = c; + + if (i == send) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ + } + } + + *i = 0; + + return i - str; +} + +int sread(STREAM* f, void* void_data, unsigned size) +{ + unsigned char* data = void_data; + + /* if there is enough space in memory */ + if (sptrlookup(f, size)) { + /* optimized version with all the data in memory */ + unsigned char* pos = sptrget(f); + + /* copy it */ + while (size--) + *data++ = *pos++; + + sptrset(f, pos); + } else { + /* standard version using sgetc() */ + while (size--) { + int c = sgetc(f); + if (c == EOF) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ + } + + *data++ = c; + } + } + + return 0; +} + +int sgetline(STREAM* f, char* str, int size) +{ + char* i = str; + char* send = str + size; + int c; + + /* if there is enough data in memory */ + if (sptrlookup(f, size)) { + /* optimized version with all the data in memory */ + unsigned char* pos = sptrget(f); + + while (1) { + c = *pos++; + if (c == '\n') { + /* remove ending carrige return to support the Windows CR+LF format */ + if (i != str && i[-1] == '\r') + --i; + --pos; + break; + } + + *i++ = c; + + if (i == send) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ + } + } + + sptrset(f, pos); + } else { + while (1) { + c = sgetc(f); + if (c == EOF) { + /* LCOV_EXCL_START */ + break; + /* LCOV_EXCL_STOP */ + } + if (c == '\n') { + /* remove ending carrige return to support the Windows CR+LF format */ + if (i != str && i[-1] == '\r') + --i; + sungetc(c, f); + break; + } + + *i++ = c; + + if (i == send) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ + } + } + } + + *i = 0; + + return i - str; +} + +int sgetlasttok(STREAM* f, char* str, int size) +{ + int ret; + + ret = sgetline(f, str, size); + if (ret < 0) { + /* LCOV_EXCL_START */ + return ret; + /* LCOV_EXCL_STOP */ + } + + while (ret > 0 && (str[ret - 1] == ' ' || str[ret - 1] == '\t')) + --ret; + + str[ret] = 0; + + return ret; +} + +int sgetu32(STREAM* f, uint32_t* value) +{ + int c; + + c = sgetc(f); + if (c >= '0' && c <= '9') { + uint32_t v; + + v = c - '0'; + + c = sgetc(f); + while (c >= '0' && c <= '9') { + v *= 10; + v += c - '0'; + c = sgetc(f); + } + + *value = v; + + sungetc(c, f); + return 0; + } else { + /* LCOV_EXCL_START */ + /* nothing read */ + return -1; + /* LCOV_EXCL_STOP */ + } +} + +int sgetu64(STREAM* f, uint64_t* value) +{ + int c; + + c = sgetc(f); + if (c >= '0' && c <= '9') { + uint64_t v; + + v = c - '0'; + + c = sgetc(f); + while (c >= '0' && c <= '9') { + v *= 10; + v += c - '0'; + c = sgetc(f); + } + + *value = v; + + sungetc(c, f); + return 0; + } else { + /* LCOV_EXCL_START */ + /* nothing read */ + return -1; + /* LCOV_EXCL_STOP */ + } +} + +int sgethex(STREAM* f, void* void_data, int size) +{ + unsigned char* data = void_data; + + /* if there is enough data in memory */ + if (sptrlookup(f, size * 2)) { + /* optimized version with all the data in memory */ + unsigned char* pos = sptrget(f); + unsigned x = 0; + + while (size--) { + unsigned b0; + unsigned b1; + unsigned b; + + b0 = strdecset[pos[0]]; + b1 = strdecset[pos[1]]; + pos += 2; + + b = (b0 << 4) | b1; + + x |= b; + + *data++ = b; + } + + /* at the end check if a digit was wrong */ + if (x > 0xFF) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ + } + + sptrset(f, pos); + } else { + /* standard version using sgetc() */ + while (size--) { + unsigned b0; + unsigned b1; + unsigned b; + int c; + + c = sgetc(f); + if (c == EOF) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ + } + b0 = strdecset[(unsigned char)c]; + + c = sgetc(f); + if (c == EOF) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ + } + b1 = strdecset[(unsigned char)c]; + + b = (b0 << 4) | b1; + + if (b > 0xFF) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ + } + + *data++ = b; + } + } + + return 0; +} + +int sgetb32(STREAM* f, uint32_t* value) +{ + uint32_t v; + unsigned char b; + unsigned char s; + int c; + + v = 0; + s = 0; +loop: + c = sgetc(f); + if (c == EOF) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ + } + + b = (unsigned char)c; + if ((b & 0x80) == 0) { + v |= (uint32_t)b << s; + s += 7; + if (s >= 32) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ + } + goto loop; + } + + v |= (uint32_t)(b & 0x7f) << s; + + *value = v; + + return 0; +} + +int sgetb64(STREAM* f, uint64_t* value) +{ + uint64_t v; + unsigned char b; + unsigned char s; + int c; + + v = 0; + s = 0; +loop: + c = sgetc(f); + if (c == EOF) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ + } + + b = (unsigned char)c; + if ((b & 0x80) == 0) { + v |= (uint64_t)b << s; + s += 7; + if (s >= 64) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ + } + goto loop; + } + + v |= (uint64_t)(b & 0x7f) << s; + + *value = v; + + return 0; +} + +int sgetble32(STREAM* f, uint32_t* value) +{ + unsigned char buf[4]; + + if (sread(f, buf, 4) != 0) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ + } + + *value = buf[0] | (uint32_t)buf[1] << 8 | (uint32_t)buf[2] << 16 | (uint32_t)buf[3] << 24; + + return 0; +} + +int sgetbs(STREAM* f, char* str, int size) +{ + uint32_t len; + + if (sgetb32(f, &len) < 0) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ + } + + if (len + 1 > (uint32_t)size) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ + } + + str[len] = 0; + + return sread(f, str, (int)len); +} + +int sputs(const char* str, STREAM* f) +{ + while (*str) { + if (sputc(*str++, f) != 0) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ + } + } + + return 0; +} + +int swrite(const void* void_data, unsigned size, STREAM* f) +{ + const unsigned char* data = void_data; + + /* if there is enough space in memory */ + if (sptrlookup(f, size)) { + /* optimized version with all the data in memory */ + unsigned char* pos = sptrget(f); + + /* copy it */ + while (size--) { + f->crc_stream = crc32c_plain(f->crc_stream, *data); + *pos++ = *data++; + } + + sptrset(f, pos); + } else { + /* standard version using sputc() */ + while (size--) { + if (sputc(*data++, f) != 0) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ + } + } + } + + return 0; +} + +int sputu32(uint32_t value, STREAM* s) +{ + char buf[16]; + int i; + + if (!value) + return sputc('0', s); + + i = sizeof(buf); + + while (value) { + buf[--i] = (value % 10) + '0'; + value /= 10; + } + + return swrite(buf + i, sizeof(buf) - i, s); +} + +int sputu64(uint64_t value, STREAM* s) +{ + char buf[32]; + uint32_t value32; + int i; + + if (!value) + return sputc('0', s); + + i = sizeof(buf); + + while (value > 0xFFFFFFFF) { + buf[--i] = (value % 10) + '0'; + value /= 10; + } + + value32 = (uint32_t)value; + + while (value32) { + buf[--i] = (value32 % 10) + '0'; + value32 /= 10; + } + + return swrite(buf + i, sizeof(buf) - i, s); +} + +int sputhex(const void* void_data, int size, STREAM* f) +{ + const unsigned char* data = void_data; + + /* if there is enough space in memory */ + if (sptrlookup(f, size * 2)) { + /* optimized version with all the data in memory */ + unsigned char* pos = sptrget(f); + + while (size) { + unsigned b = *data; + + *pos++ = strhexset[b >> 4]; + *pos++ = strhexset[b & 0xF]; + + ++data; + --size; + } + + sptrset(f, pos); + } else { + /* standard version using sputc() */ + while (size) { + unsigned b = *data; + + if (sputc(strhexset[b >> 4], f) != 0) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ + } + if (sputc(strhexset[b & 0xF], f) != 0) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ + } + + ++data; + --size; + } + } + + return 0; +} + +int sputb32(uint32_t value, STREAM* s) +{ + unsigned char b; + unsigned char buf[16]; + unsigned i; + + i = 0; +loop: + b = value & 0x7f; + value >>= 7; + + if (value) { + buf[i++] = b; + goto loop; + } + + buf[i++] = b | 0x80; + + return swrite(buf, i, s); +} + +int sputb64(uint64_t value, STREAM* s) +{ + unsigned char b; + unsigned char buf[16]; + unsigned i; + + i = 0; +loop: + b = value & 0x7f; + value >>= 7; + + if (value) { + buf[i++] = b; + goto loop; + } + + buf[i++] = b | 0x80; + + return swrite(buf, i, s); +} + +int sputble32(uint32_t value, STREAM* s) +{ + unsigned char buf[4]; + + buf[0] = value & 0xFF; + buf[1] = (value >> 8) & 0xFF; + buf[2] = (value >> 16) & 0xFF; + buf[3] = (value >> 24) & 0xFF; + + return swrite(buf, 4, s); +} + +int sputbs(const char* str, STREAM* f) +{ + size_t len = strlen(str); + + if (sputb32(len, f) != 0) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ + } + + return swrite(str, len, f); +} + +#if HAVE_FSYNC +int ssync(STREAM* s) +{ + unsigned i; + + for (i = 0; i < s->handle_size; ++i) { + if (fsync(s->handle[i].f) != 0) { + /* LCOV_EXCL_START */ + s->state = STREAM_STATE_ERROR; + s->state_index = i; + return -1; + /* LCOV_EXCL_STOP */ + } + } + + return 0; +} +#endif + diff --git a/cmdline/stream.h b/cmdline/stream.h new file mode 100644 index 00000000..a26fa20f --- /dev/null +++ b/cmdline/stream.h @@ -0,0 +1,443 @@ +/* + * Copyright (C) 2011 Andrea Mazzoleni + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef __STREAM_H +#define __STREAM_H + +#include "util.h" + +/****************************************************************************/ +/* stream */ + +/** + * Size of the buffer of the stream. + * + * It's not a constant for testing purpose. + */ +unsigned STREAM_SIZE; + +#define STREAM_STATE_READ 0 /**< The stream is in a normal state of read. */ +#define STREAM_STATE_WRITE 1 /**< The stream is in a normal state of write. */ +#define STREAM_STATE_ERROR -1 /**< An error was encoutered. */ +#define STREAM_STATE_EOF 2 /**< The end of file was encoutered. */ + +struct stream_handle { + int f; /**< Handle of the file. */ + char path[PATH_MAX]; /**< Path of the file. */ +}; + +struct stream { + unsigned char* buffer; /**< Buffer of the stream. */ + unsigned char* pos; /**< Current position in the buffer. */ + unsigned char* end; /**< End position of the buffer. */ + int state; /**< State of the stream. One of STREAM_STATE. */ + int state_index; /**< Index of the handle causing a state change. */ + unsigned handle_size; /**< Number of handles. */ + struct stream_handle* handle; /**< Set of handles. */ + off_t offset; /**< Offset into the file. */ + off_t offset_uncached; /**< Offset into the file excluding the cached data. */ + + /** + * CRC of the data read or written in the file. + * + * If reading, it's the CRC of all data read from the file, + * including the one in the buffer. + * If writing it's all the data wrote to the file, + * excluding the one still in buffer yet to be written. + */ + uint32_t crc; + + /** + * CRC of the file excluding the cached data in the buffer. + * + * If reading, it's the CRC of the data read from the file, + * excluding the one in the buffer. + * If writing it's all the data wrote to the file, + * excluding the one still in buffer yet to be written. + */ + uint32_t crc_uncached; + + /** + * CRC of the data written to the stream. + * + * This is an extra check of the data that is written to + * file to ensure that it's consistent even in case + * of memory errors. + * + * This extra check takes about 2 seconds for each GB of + * content file with the Intel CRC instruction, + * and about 4 seconds without it. + * But usually this doesn't slow down the write process, + * as the disk is the bottle-neck. + * + * Note that this CRC doesn't have the IV processing. + * + * Not used in reading. + * In writing, it's all the data wrote calling sput() functions. + */ + uint32_t crc_stream; +}; + +/** + * Opaque STREAM type. Like ::FILE. + */ +typedef struct stream STREAM; + +/** + * Opens a stream for reading. Like fopen("r"). + */ +STREAM* sopen_read(const char* file); + +/** + * Opens a stream for writing. Like fopen("w"). + */ +STREAM* sopen_write(const char* file); + +/** + * Opens a set of streams for writing. Like fopen("w"). + */ +STREAM* sopen_multi_write(unsigned count); + +/** + * Specifies the file to open. + */ +int sopen_multi_file(STREAM* s, unsigned i, const char* file); + +/** + * Closes a stream. Like fclose(). + */ +int sclose(STREAM* s); + +/** + * Returns the handle of the file. + * In case of multi file, the first one is returned. + */ +int shandle(STREAM* s); + +/** + * Fills the read stream buffer and read a char. + * \note Don't call this directly, but use sgetc(). + * \return The char read, or EOF on error. + */ +int sfill(STREAM* s); + +/** + * Flushes the write stream buffer. + * \return 0 on success, or EOF on error. + */ +int sflush(STREAM* s); + +/** + * Gets the file pointer. + */ +int64_t stell(STREAM* s); + +/** + * Gets the CRC of the processed data. + */ +uint32_t scrc(STREAM* s); + +/** + * Gets the CRC of the processed data in put. + */ +uint32_t scrc_stream(STREAM* s); + +/** + * Checks if the buffer has enough data loaded. + */ +static inline int sptrlookup(STREAM* s, int size) +{ + return s->pos + size <= s->end; +} + +/** + * Gets the current stream ptr. + */ +static inline unsigned char* sptrget(STREAM* s) +{ + return s->pos; +} + +/** + * Sets the current stream ptr. + */ +static inline void sptrset(STREAM* s, unsigned char* ptr) +{ + s->pos = ptr; +} + +/** + * Checks the error status. Like ferror(). + */ +static inline int serror(STREAM* s) +{ + return s->state == STREAM_STATE_ERROR; +} + +/** + * Checks the eof status. Like feof(). + */ +static inline int seof(STREAM* s) +{ + return s->state == STREAM_STATE_EOF; +} + +/** + * Gets the index of the handle that caused the error. + */ +static inline int serrorindex(STREAM* s) +{ + return s->state_index; +} + +/** + * Gets the path of the handle that caused the error. + */ +static inline const char* serrorfile(STREAM* s) +{ + return s->handle[s->state_index].path; +} + +/** + * Sync the stream. Like fsync(). + */ +int ssync(STREAM* s); + +/****************************************************************************/ +/* get */ + +/** + * Reads a char. Like fgetc(). + */ +static inline int sgetc(STREAM* s) +{ + if (s->pos == s->end) + return sfill(s); + return *s->pos++; +} + +/** + * Unreads a char. + * Like ungetc() but you have to unget the same char read. + */ +static inline void sungetc(int c, STREAM* s) +{ + if (c != EOF) + --s->pos; +} + +/** + * Reads a fixed amount of chars. + * Returns 0 on success, or -1 on error. + */ +int sread(STREAM* f, void* void_data, unsigned size); + +/** + * Gets a char from a stream, ignoring one '\r'. + */ +static inline int sgeteol(STREAM* f) +{ + int c; + + c = sgetc(f); + if (c == '\r') + c = sgetc(f); + + return c; +} + +/** + * Reads all the spaces and tabs. + * Returns the number of spaces and tabs read. + */ +static inline int sgetspace(STREAM* f) +{ + int count = 0; + int c; + + c = sgetc(f); + while (c == ' ' || c == '\t') { + ++count; + c = sgetc(f); + } + + sungetc(c, f); + return count; +} + +/** + * Reads until the first space or tab. + * Stops at the first ' ', '\t', '\n' or EOF. + * Returns <0 if the buffer is too small, or the number of chars read. + */ +int sgettok(STREAM* f, char* str, int size); + +/** + * Reads until the end of line. + * Stops at the first '\n' or EOF. Note that '\n' is left in the stream. + * Returns <0 if the buffer is too small, or the number of chars read. + */ +int sgetline(STREAM* f, char* str, int size); + +/** + * Like sgetline() but removes ' ' and '\t' at the end. + */ +int sgetlasttok(STREAM* f, char* str, int size); + +/** + * Reads a 32 bit number. + * Stops at the first not digit char or EOF. + * Returns <0 if there isn't enough to read. + */ +int sgetu32(STREAM* f, uint32_t* value); + +/** + * Reads a 64 bit number. + * Stops at the first not digit char. + * Returns <0 if there isn't enough to read. + */ +int sgetu64(STREAM* f, uint64_t* value); + +/** + * Reads an hexadecimal string of fixed length. + * Returns <0 if there isn't enough to read. + */ +int sgethex(STREAM* f, void* data, int size); + +/****************************************************************************/ +/* binary get */ + +/** + * Reads a binary 32 bit number in packet format. + * Returns <0 if there isn't enough to read. + */ +int sgetb32(STREAM* f, uint32_t* value); + +/** + * Reads a binary 64 bit number in packet format. + * Returns <0 if there isn't enough to read. + */ +int sgetb64(STREAM* f, uint64_t* value); + +/** + * Reads a binary 32 bit number in little endian format. + * Returns <0 if there isn't enough to read. + */ +int sgetble32(STREAM* f, uint32_t* value); + +/** + * Reads a binary string. + * Returns -1 on error or if the buffer is too small, or the number of chars read. + */ +int sgetbs(STREAM* f, char* str, int size); + +/****************************************************************************/ +/* put */ + +/** + * Writes a char. Like fputc(). + * Returns 0 on success or -1 on error. + */ +static inline int sputc(int c, STREAM* s) +{ + if (s->pos == s->end) { + if (sflush(s) != 0) + return -1; + } + s->crc_stream = crc32c_plain(s->crc_stream, c); + *s->pos++ = c; + return 0; +} + +/** + * Writes a end of line. + * Returns 0 on success or -1 on error. + */ +static inline int sputeol(STREAM* s) +{ +#ifdef _WIN32 + if (sputc('\r', s) != 0) + return -1; +#endif + return sputc('\n', s); +} + +/** + * Writes a string. + * Returns 0 on success or -1 on error. + */ +int sputs(const char* str, STREAM* f); + +/** + * Write a sized string. + * Returns 0 on success or -1 on error. + */ +int swrite(const void* data, unsigned size, STREAM* f); + +/** + * Write a string literal (constant string). + * Returns 0 on success or -1 on error. + */ +#define sputsl(str, f) swrite(str, sizeof(str) - 1, f) + +/** + * Writes a 32 bit number. + * Returns 0 on success or -1 on error. + */ +int sputu32(uint32_t value, STREAM* s); + +/** + * Writes a 64 bit number. + * Returns 0 on success or -1 on error. + */ +int sputu64(uint64_t value, STREAM* s); + +/** + * Writes a hexadecimal string of fixed length. + * Returns 0 on success or -1 on error. + */ +int sputhex(const void* void_data, int size, STREAM* s); + +/****************************************************************************/ +/* binary put */ + +/** + * Writes a binary 32 bit number in packed format. + * Returns 0 on success or -1 on error. + */ +int sputb32(uint32_t value, STREAM* s); + +/** + * Writes a binary 64 bit number in packed format. + * Returns 0 on success or -1 on error. + */ +int sputb64(uint64_t value, STREAM* s); + +/** + * Writes a binary 32 bit number in little endian format. + * Returns 0 on success or -1 on error. + */ +int sputble32(uint32_t value, STREAM* s); + +/** + * Writes a binary string. + * Returns 0 on success or -1 on error. + */ +int sputbs(const char* str, STREAM* s); + +#endif + diff --git a/cmdline/util.c b/cmdline/util.c index 10edb49b..74646107 100644 --- a/cmdline/util.c +++ b/cmdline/util.c @@ -22,882 +22,6 @@ #include "raid/cpu.h" #include "raid/memory.h" -/****************************************************************************/ -/* hex conversion table */ - -static char strhexset[16] = "0123456789abcdef"; - -static unsigned strdecset[256] = -{ - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, - -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 -}; - -/****************************************************************************/ -/* stream */ - -unsigned STREAM_SIZE = 1024 * 64; - -STREAM* sopen_read(const char* file) -{ -#if HAVE_POSIX_FADVISE - int ret; -#endif - STREAM* s = malloc_nofail(sizeof(STREAM)); - - s->handle_size = 1; - s->handle = malloc_nofail(sizeof(struct stream_handle)); - - pathcpy(s->handle[0].path, sizeof(s->handle[0].path), file); - s->handle[0].f = open(file, O_RDONLY | O_BINARY | O_SEQUENTIAL); - if (s->handle[0].f == -1) { - free(s->handle); - free(s); - return 0; - } - -#if HAVE_POSIX_FADVISE - /* advise sequential access */ - ret = posix_fadvise(s->handle[0].f, 0, 0, POSIX_FADV_SEQUENTIAL); - if (ret != 0) { - /* LCOV_EXCL_START */ - close(s->handle[0].f); - free(s->handle); - free(s); - return 0; - /* LCOV_EXCL_STOP */ - } -#endif - - s->buffer = malloc_nofail_test(STREAM_SIZE); - s->pos = s->buffer; - s->end = s->buffer; - s->state = STREAM_STATE_READ; - s->state_index = 0; - s->offset = 0; - s->offset_uncached = 0; - s->crc = 0; - s->crc_uncached = 0; - s->crc_stream = CRC_IV; - - return s; -} - -STREAM* sopen_multi_write(unsigned count) -{ - unsigned i; - - STREAM* s = malloc_nofail(sizeof(STREAM)); - - s->handle_size = count; - s->handle = malloc_nofail(count * sizeof(struct stream_handle)); - - for (i = 0; i < count; ++i) - s->handle[i].f = -1; - - s->buffer = malloc_nofail_test(STREAM_SIZE); - s->pos = s->buffer; - s->end = s->buffer + STREAM_SIZE; - s->state = STREAM_STATE_WRITE; - s->state_index = 0; - s->offset = 0; - s->offset_uncached = 0; - s->crc = 0; - s->crc_uncached = 0; - s->crc_stream = CRC_IV; - - return s; -} - -int sopen_multi_file(STREAM* s, unsigned i, const char* file) -{ -#if HAVE_POSIX_FADVISE - int ret; -#endif - int f; - - pathcpy(s->handle[i].path, sizeof(s->handle[i].path), file); - - f = open(file, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY | O_SEQUENTIAL, 0600); - if (f == -1) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ - } - -#if HAVE_POSIX_FADVISE - /* advise sequential access */ - ret = posix_fadvise(f, 0, 0, POSIX_FADV_SEQUENTIAL); - if (ret != 0) { - /* LCOV_EXCL_START */ - close(f); - return -1; - /* LCOV_EXCL_STOP */ - } -#endif - - s->handle[i].f = f; - - return 0; -} - -int sclose(STREAM* s) -{ - int fail = 0; - unsigned i; - - if (s->state == STREAM_STATE_WRITE) { - if (sflush(s) != 0) { - /* LCOV_EXCL_START */ - fail = 1; - /* LCOV_EXCL_STOP */ - } - } - - for (i = 0; i < s->handle_size; ++i) { - if (close(s->handle[i].f) != 0) { - /* LCOV_EXCL_START */ - fail = 1; - /* LCOV_EXCL_STOP */ - } - } - - free(s->handle); - free(s->buffer); - free(s); - - if (fail) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ - } - - return 0; -} - -int shandle(STREAM* s) -{ - if (!s->handle_size) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ - } - - return s->handle[0].f; -} - -int sfill(STREAM* s) -{ - ssize_t ret; - - if (s->state != STREAM_STATE_READ) { - /* LCOV_EXCL_START */ - return EOF; - /* LCOV_EXCL_STOP */ - } - - ret = read(s->handle[0].f, s->buffer, STREAM_SIZE); - - if (ret < 0) { - /* LCOV_EXCL_START */ - s->state = STREAM_STATE_ERROR; - return EOF; - /* LCOV_EXCL_STOP */ - } - if (ret == 0) { - s->state = STREAM_STATE_EOF; - return EOF; - } - - /* update the crc */ - s->crc_uncached = s->crc; - s->crc = crc32c(s->crc, s->buffer, ret); - - /* update the offset */ - s->offset_uncached = s->offset; - s->offset += ret; - - s->pos = s->buffer; - s->end = s->buffer + ret; - - return *s->pos++; -} - -int sflush(STREAM* s) -{ - ssize_t ret; - ssize_t size; - unsigned i; - - if (s->state != STREAM_STATE_WRITE) { - /* LCOV_EXCL_START */ - return EOF; - /* LCOV_EXCL_STOP */ - } - - size = s->pos - s->buffer; - if (!size) - return 0; - - /* update the crc */ - s->crc = crc32c(s->crc, s->buffer, size); - s->crc_uncached = s->crc; - - for (i = 0; i < s->handle_size; ++i) { - ret = write(s->handle[i].f, s->buffer, size); - - if (ret != size) { - /* LCOV_EXCL_START */ - s->state = STREAM_STATE_ERROR; - s->state_index = i; - return EOF; - /* LCOV_EXCL_STOP */ - } - } - - /* update the offset */ - s->offset += size; - s->offset_uncached = s->offset; - - s->pos = s->buffer; - - return 0; -} - -int64_t stell(STREAM* s) -{ - return s->offset_uncached + (s->pos - s->buffer); -} - -uint32_t scrc(STREAM*s) -{ - return crc32c(s->crc_uncached, s->buffer, s->pos - s->buffer); -} - -uint32_t scrc_stream(STREAM*s) -{ - return s->crc_stream ^ CRC_IV; -} - -int sgettok(STREAM* f, char* str, int size) -{ - char* i = str; - char* send = str + size; - int c; - - while (1) { - c = sgetc(f); - if (c == EOF) { - break; - } - if (c == ' ' || c == '\t') { - sungetc(c, f); - break; - } - if (c == '\n') { - /* remove ending carrige return to support the Windows CR+LF format */ - if (i != str && i[-1] == '\r') - --i; - sungetc(c, f); - break; - } - - *i++ = c; - - if (i == send) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ - } - } - - *i = 0; - - return i - str; -} - -int sread(STREAM* f, void* void_data, unsigned size) -{ - unsigned char* data = void_data; - - /* if there is enough space in memory */ - if (sptrlookup(f, size)) { - /* optimized version with all the data in memory */ - unsigned char* pos = sptrget(f); - - /* copy it */ - while (size--) - *data++ = *pos++; - - sptrset(f, pos); - } else { - /* standard version using sgetc() */ - while (size--) { - int c = sgetc(f); - if (c == EOF) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ - } - - *data++ = c; - } - } - - return 0; -} - -int sgetline(STREAM* f, char* str, int size) -{ - char* i = str; - char* send = str + size; - int c; - - /* if there is enough data in memory */ - if (sptrlookup(f, size)) { - /* optimized version with all the data in memory */ - unsigned char* pos = sptrget(f); - - while (1) { - c = *pos++; - if (c == '\n') { - /* remove ending carrige return to support the Windows CR+LF format */ - if (i != str && i[-1] == '\r') - --i; - --pos; - break; - } - - *i++ = c; - - if (i == send) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ - } - } - - sptrset(f, pos); - } else { - while (1) { - c = sgetc(f); - if (c == EOF) { - /* LCOV_EXCL_START */ - break; - /* LCOV_EXCL_STOP */ - } - if (c == '\n') { - /* remove ending carrige return to support the Windows CR+LF format */ - if (i != str && i[-1] == '\r') - --i; - sungetc(c, f); - break; - } - - *i++ = c; - - if (i == send) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ - } - } - } - - *i = 0; - - return i - str; -} - -int sgetlasttok(STREAM* f, char* str, int size) -{ - int ret; - - ret = sgetline(f, str, size); - if (ret < 0) { - /* LCOV_EXCL_START */ - return ret; - /* LCOV_EXCL_STOP */ - } - - while (ret > 0 && (str[ret - 1] == ' ' || str[ret - 1] == '\t')) - --ret; - - str[ret] = 0; - - return ret; -} - -int sgetu32(STREAM* f, uint32_t* value) -{ - int c; - - c = sgetc(f); - if (c >= '0' && c <= '9') { - uint32_t v; - - v = c - '0'; - - c = sgetc(f); - while (c >= '0' && c <= '9') { - v *= 10; - v += c - '0'; - c = sgetc(f); - } - - *value = v; - - sungetc(c, f); - return 0; - } else { - /* LCOV_EXCL_START */ - /* nothing read */ - return -1; - /* LCOV_EXCL_STOP */ - } -} - -int sgetu64(STREAM* f, uint64_t* value) -{ - int c; - - c = sgetc(f); - if (c >= '0' && c <= '9') { - uint64_t v; - - v = c - '0'; - - c = sgetc(f); - while (c >= '0' && c <= '9') { - v *= 10; - v += c - '0'; - c = sgetc(f); - } - - *value = v; - - sungetc(c, f); - return 0; - } else { - /* LCOV_EXCL_START */ - /* nothing read */ - return -1; - /* LCOV_EXCL_STOP */ - } -} - -int sgethex(STREAM* f, void* void_data, int size) -{ - unsigned char* data = void_data; - - /* if there is enough data in memory */ - if (sptrlookup(f, size * 2)) { - /* optimized version with all the data in memory */ - unsigned char* pos = sptrget(f); - unsigned x = 0; - - while (size--) { - unsigned b0; - unsigned b1; - unsigned b; - - b0 = strdecset[pos[0]]; - b1 = strdecset[pos[1]]; - pos += 2; - - b = (b0 << 4) | b1; - - x |= b; - - *data++ = b; - } - - /* at the end check if a digit was wrong */ - if (x > 0xFF) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ - } - - sptrset(f, pos); - } else { - /* standard version using sgetc() */ - while (size--) { - unsigned b0; - unsigned b1; - unsigned b; - int c; - - c = sgetc(f); - if (c == EOF) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ - } - b0 = strdecset[(unsigned char)c]; - - c = sgetc(f); - if (c == EOF) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ - } - b1 = strdecset[(unsigned char)c]; - - b = (b0 << 4) | b1; - - if (b > 0xFF) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ - } - - *data++ = b; - } - } - - return 0; -} - -int sgetb32(STREAM* f, uint32_t* value) -{ - uint32_t v; - unsigned char b; - unsigned char s; - int c; - - v = 0; - s = 0; -loop: - c = sgetc(f); - if (c == EOF) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ - } - - b = (unsigned char)c; - if ((b & 0x80) == 0) { - v |= (uint32_t)b << s; - s += 7; - if (s >= 32) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ - } - goto loop; - } - - v |= (uint32_t)(b & 0x7f) << s; - - *value = v; - - return 0; -} - -int sgetb64(STREAM* f, uint64_t* value) -{ - uint64_t v; - unsigned char b; - unsigned char s; - int c; - - v = 0; - s = 0; -loop: - c = sgetc(f); - if (c == EOF) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ - } - - b = (unsigned char)c; - if ((b & 0x80) == 0) { - v |= (uint64_t)b << s; - s += 7; - if (s >= 64) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ - } - goto loop; - } - - v |= (uint64_t)(b & 0x7f) << s; - - *value = v; - - return 0; -} - -int sgetble32(STREAM* f, uint32_t* value) -{ - unsigned char buf[4]; - - if (sread(f, buf, 4) != 0) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ - } - - *value = buf[0] | (uint32_t)buf[1] << 8 | (uint32_t)buf[2] << 16 | (uint32_t)buf[3] << 24; - - return 0; -} - -int sgetbs(STREAM* f, char* str, int size) -{ - uint32_t len; - - if (sgetb32(f, &len) < 0) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ - } - - if (len + 1 > (uint32_t)size) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ - } - - str[len] = 0; - - return sread(f, str, (int)len); -} - -int sputs(const char* str, STREAM* f) -{ - while (*str) { - if (sputc(*str++, f) != 0) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ - } - } - - return 0; -} - -int swrite(const void* void_data, unsigned size, STREAM* f) -{ - const unsigned char* data = void_data; - - /* if there is enough space in memory */ - if (sptrlookup(f, size)) { - /* optimized version with all the data in memory */ - unsigned char* pos = sptrget(f); - - /* copy it */ - while (size--) { - f->crc_stream = crc32c_plain(f->crc_stream, *data); - *pos++ = *data++; - } - - sptrset(f, pos); - } else { - /* standard version using sputc() */ - while (size--) { - if (sputc(*data++, f) != 0) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ - } - } - } - - return 0; -} - -int sputu32(uint32_t value, STREAM* s) -{ - char buf[16]; - int i; - - if (!value) - return sputc('0', s); - - i = sizeof(buf); - - while (value) { - buf[--i] = (value % 10) + '0'; - value /= 10; - } - - return swrite(buf + i, sizeof(buf) - i, s); -} - -int sputu64(uint64_t value, STREAM* s) -{ - char buf[32]; - uint32_t value32; - int i; - - if (!value) - return sputc('0', s); - - i = sizeof(buf); - - while (value > 0xFFFFFFFF) { - buf[--i] = (value % 10) + '0'; - value /= 10; - } - - value32 = (uint32_t)value; - - while (value32) { - buf[--i] = (value32 % 10) + '0'; - value32 /= 10; - } - - return swrite(buf + i, sizeof(buf) - i, s); -} - -int sputhex(const void* void_data, int size, STREAM* f) -{ - const unsigned char* data = void_data; - - /* if there is enough space in memory */ - if (sptrlookup(f, size * 2)) { - /* optimized version with all the data in memory */ - unsigned char* pos = sptrget(f); - - while (size) { - unsigned b = *data; - - *pos++ = strhexset[b >> 4]; - *pos++ = strhexset[b & 0xF]; - - ++data; - --size; - } - - sptrset(f, pos); - } else { - /* standard version using sputc() */ - while (size) { - unsigned b = *data; - - if (sputc(strhexset[b >> 4], f) != 0) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ - } - if (sputc(strhexset[b & 0xF], f) != 0) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ - } - - ++data; - --size; - } - } - - return 0; -} - -int sputb32(uint32_t value, STREAM* s) -{ - unsigned char b; - unsigned char buf[16]; - unsigned i; - - i = 0; -loop: - b = value & 0x7f; - value >>= 7; - - if (value) { - buf[i++] = b; - goto loop; - } - - buf[i++] = b | 0x80; - - return swrite(buf, i, s); -} - -int sputb64(uint64_t value, STREAM* s) -{ - unsigned char b; - unsigned char buf[16]; - unsigned i; - - i = 0; -loop: - b = value & 0x7f; - value >>= 7; - - if (value) { - buf[i++] = b; - goto loop; - } - - buf[i++] = b | 0x80; - - return swrite(buf, i, s); -} - -int sputble32(uint32_t value, STREAM* s) -{ - unsigned char buf[4]; - - buf[0] = value & 0xFF; - buf[1] = (value >> 8) & 0xFF; - buf[2] = (value >> 16) & 0xFF; - buf[3] = (value >> 24) & 0xFF; - - return swrite(buf, 4, s); -} - -int sputbs(const char* str, STREAM* f) -{ - size_t len = strlen(str); - - if (sputb32(len, f) != 0) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ - } - - return swrite(str, len, f); -} - -#if HAVE_FSYNC -int ssync(STREAM* s) -{ - unsigned i; - - for (i = 0; i < s->handle_size; ++i) { - if (fsync(s->handle[i].f) != 0) { - /* LCOV_EXCL_START */ - s->state = STREAM_STATE_ERROR; - s->state_index = i; - return -1; - /* LCOV_EXCL_STOP */ - } - } - - return 0; -} -#endif - /****************************************************************************/ /* memory */ diff --git a/cmdline/util.h b/cmdline/util.h index fffa327f..7d394522 100644 --- a/cmdline/util.h +++ b/cmdline/util.h @@ -18,6 +18,32 @@ #ifndef __UTIL_H #define __UTIL_H +/****************************************************************************/ +/* memory */ + +/** + * Safe aligned malloc. + * If no memory is available, it aborts. + */ +void* malloc_nofail_align(size_t size, void** freeptr); + +/** + * Safe aligned vector allocation. + * If no memory is available, it aborts. + */ +void** malloc_nofail_vector_align(int nd, int n, size_t size, void** freeptr); + +/** + * Safe allocation with memory test. + */ +void* malloc_nofail_test(size_t size); + +/** + * Tests the memory vector for RAM problems. + * If a problem is found, it crashes. + */ +void mtest_vector(int n, size_t size, void** vv); + /****************************************************************************/ /* crc */ @@ -70,446 +96,6 @@ uint32_t crc32c_x86(uint32_t crc, const unsigned char* ptr, unsigned size); */ void crc32c_init(void); -/****************************************************************************/ -/* stream */ - -unsigned STREAM_SIZE; /**< Size of the buffer of the stream. */ - -#define STREAM_STATE_READ 0 /**< The stream is in a normal state of read. */ -#define STREAM_STATE_WRITE 1 /**< The stream is in a normal state of write. */ -#define STREAM_STATE_ERROR -1 /**< An error was encoutered. */ -#define STREAM_STATE_EOF 2 /**< The end of file was encoutered. */ - -struct stream_handle { - int f; /**< Handle of the file. */ - char path[PATH_MAX]; /**< Path of the file. */ -}; - -struct stream { - unsigned char* buffer; /**< Buffer of the stream. */ - unsigned char* pos; /**< Current position in the buffer. */ - unsigned char* end; /**< End position of the buffer. */ - int state; /**< State of the stream. One of STREAM_STATE. */ - int state_index; /**< Index of the handle causing a state change. */ - unsigned handle_size; /**< Number of handles. */ - struct stream_handle* handle; /**< Set of handles. */ - off_t offset; /**< Offset into the file. */ - off_t offset_uncached; /**< Offset into the file excluding the cached data. */ - - /** - * CRC of the data read or written in the file. - * - * If reading, it's the CRC of all data read from the file, - * including the one in the buffer. - * If writing it's all the data wrote to the file, - * excluding the one still in buffer yet to be written. - */ - uint32_t crc; - - /** - * CRC of the file excluding the cached data in the buffer. - * - * If reading, it's the CRC of the data read from the file, - * excluding the one in the buffer. - * If writing it's all the data wrote to the file, - * excluding the one still in buffer yet to be written. - */ - uint32_t crc_uncached; - - /** - * CRC of the data written to the stream. - * - * This is an extra check of the data that is written to - * file to ensure that it's consistent even in case - * of memory errors. - * - * This extra check takes about 2 seconds for each GB of - * content file with the Intel CRC instruction, - * and about 4 seconds without it. - * But usually this doesn't slow down the write process, - * as the disk is the bottle-neck. - * - * Note that this CRC doesn't have the IV processing. - * - * Not used in reading. - * In writing, it's all the data wrote calling sput() functions. - */ - uint32_t crc_stream; -}; - -/** - * Opaque STREAM type. Like ::FILE. - */ -typedef struct stream STREAM; - -/** - * Opens a stream for reading. Like fopen("r"). - */ -STREAM* sopen_read(const char* file); - -/** - * Opens a stream for writing. Like fopen("w"). - */ -STREAM* sopen_write(const char* file); - -/** - * Opens a set of streams for writing. Like fopen("w"). - */ -STREAM* sopen_multi_write(unsigned count); - -/** - * Specifies the file to open. - */ -int sopen_multi_file(STREAM* s, unsigned i, const char* file); - -/** - * Closes a stream. Like fclose(). - */ -int sclose(STREAM* s); - -/** - * Returns the handle of the file. - * In case of multi file, the first one is returned. - */ -int shandle(STREAM* s); - -/** - * Fills the read stream buffer and read a char. - * \note Don't call this directly, but use sgetc(). - * \return The char read, or EOF on error. - */ -int sfill(STREAM* s); - -/** - * Flushes the write stream buffer. - * \return 0 on success, or EOF on error. - */ -int sflush(STREAM* s); - -/** - * Gets the file pointer. - */ -int64_t stell(STREAM* s); - -/** - * Gets the CRC of the processed data. - */ -uint32_t scrc(STREAM* s); - -/** - * Gets the CRC of the processed data in put. - */ -uint32_t scrc_stream(STREAM* s); - -/** - * Checks if the buffer has enough data loaded. - */ -static inline int sptrlookup(STREAM* s, int size) -{ - return s->pos + size <= s->end; -} - -/** - * Gets the current stream ptr. - */ -static inline unsigned char* sptrget(STREAM* s) -{ - return s->pos; -} - -/** - * Sets the current stream ptr. - */ -static inline void sptrset(STREAM* s, unsigned char* ptr) -{ - s->pos = ptr; -} - -/** - * Checks the error status. Like ferror(). - */ -static inline int serror(STREAM* s) -{ - return s->state == STREAM_STATE_ERROR; -} - -/** - * Checks the eof status. Like feof(). - */ -static inline int seof(STREAM* s) -{ - return s->state == STREAM_STATE_EOF; -} - -/** - * Gets the index of the handle that caused the error. - */ -static inline int serrorindex(STREAM* s) -{ - return s->state_index; -} - -/** - * Gets the path of the handle that caused the error. - */ -static inline const char* serrorfile(STREAM* s) -{ - return s->handle[s->state_index].path; -} - -/** - * Sync the stream. Like fsync(). - */ -int ssync(STREAM* s); - -/****************************************************************************/ -/* get */ - -/** - * Reads a char. Like fgetc(). - */ -static inline int sgetc(STREAM* s) -{ - if (s->pos == s->end) - return sfill(s); - return *s->pos++; -} - -/** - * Unreads a char. - * Like ungetc() but you have to unget the same char read. - */ -static inline void sungetc(int c, STREAM* s) -{ - if (c != EOF) - --s->pos; -} - -/** - * Reads a fixed amount of chars. - * Returns 0 on success, or -1 on error. - */ -int sread(STREAM* f, void* void_data, unsigned size); - -/** - * Gets a char from a stream, ignoring one '\r'. - */ -static inline int sgeteol(STREAM* f) -{ - int c; - - c = sgetc(f); - if (c == '\r') - c = sgetc(f); - - return c; -} - -/** - * Reads all the spaces and tabs. - * Returns the number of spaces and tabs read. - */ -static inline int sgetspace(STREAM* f) -{ - int count = 0; - int c; - - c = sgetc(f); - while (c == ' ' || c == '\t') { - ++count; - c = sgetc(f); - } - - sungetc(c, f); - return count; -} - -/** - * Reads until the first space or tab. - * Stops at the first ' ', '\t', '\n' or EOF. - * Returns <0 if the buffer is too small, or the number of chars read. - */ -int sgettok(STREAM* f, char* str, int size); - -/** - * Reads until the end of line. - * Stops at the first '\n' or EOF. Note that '\n' is left in the stream. - * Returns <0 if the buffer is too small, or the number of chars read. - */ -int sgetline(STREAM* f, char* str, int size); - -/** - * Like sgetline() but removes ' ' and '\t' at the end. - */ -int sgetlasttok(STREAM* f, char* str, int size); - -/** - * Reads a 32 bit number. - * Stops at the first not digit char or EOF. - * Returns <0 if there isn't enough to read. - */ -int sgetu32(STREAM* f, uint32_t* value); - -/** - * Reads a 64 bit number. - * Stops at the first not digit char. - * Returns <0 if there isn't enough to read. - */ -int sgetu64(STREAM* f, uint64_t* value); - -/** - * Reads an hexadecimal string of fixed length. - * Returns <0 if there isn't enough to read. - */ -int sgethex(STREAM* f, void* data, int size); - -/****************************************************************************/ -/* binary get */ - -/** - * Reads a binary 32 bit number in packet format. - * Returns <0 if there isn't enough to read. - */ -int sgetb32(STREAM* f, uint32_t* value); - -/** - * Reads a binary 64 bit number in packet format. - * Returns <0 if there isn't enough to read. - */ -int sgetb64(STREAM* f, uint64_t* value); - -/** - * Reads a binary 32 bit number in little endian format. - * Returns <0 if there isn't enough to read. - */ -int sgetble32(STREAM* f, uint32_t* value); - -/** - * Reads a binary string. - * Returns -1 on error or if the buffer is too small, or the number of chars read. - */ -int sgetbs(STREAM* f, char* str, int size); - -/****************************************************************************/ -/* put */ - -/** - * Writes a char. Like fputc(). - * Returns 0 on success or -1 on error. - */ -static inline int sputc(int c, STREAM* s) -{ - if (s->pos == s->end) { - if (sflush(s) != 0) - return -1; - } - s->crc_stream = crc32c_plain(s->crc_stream, c); - *s->pos++ = c; - return 0; -} - -/** - * Writes a end of line. - * Returns 0 on success or -1 on error. - */ -static inline int sputeol(STREAM* s) -{ -#ifdef _WIN32 - if (sputc('\r', s) != 0) - return -1; -#endif - return sputc('\n', s); -} - -/** - * Writes a string. - * Returns 0 on success or -1 on error. - */ -int sputs(const char* str, STREAM* f); - -/** - * Write a sized string. - * Returns 0 on success or -1 on error. - */ -int swrite(const void* data, unsigned size, STREAM* f); - -/** - * Write a string literal (constant string). - * Returns 0 on success or -1 on error. - */ -#define sputsl(str, f) swrite(str, sizeof(str) - 1, f) - -/** - * Writes a 32 bit number. - * Returns 0 on success or -1 on error. - */ -int sputu32(uint32_t value, STREAM* s); - -/** - * Writes a 64 bit number. - * Returns 0 on success or -1 on error. - */ -int sputu64(uint64_t value, STREAM* s); - -/** - * Writes a hexadecimal string of fixed length. - * Returns 0 on success or -1 on error. - */ -int sputhex(const void* void_data, int size, STREAM* s); - -/****************************************************************************/ -/* binary put */ - -/** - * Writes a binary 32 bit number in packed format. - * Returns 0 on success or -1 on error. - */ -int sputb32(uint32_t value, STREAM* s); - -/** - * Writes a binary 64 bit number in packed format. - * Returns 0 on success or -1 on error. - */ -int sputb64(uint64_t value, STREAM* s); - -/** - * Writes a binary 32 bit number in little endian format. - * Returns 0 on success or -1 on error. - */ -int sputble32(uint32_t value, STREAM* s); - -/** - * Writes a binary string. - * Returns 0 on success or -1 on error. - */ -int sputbs(const char* str, STREAM* s); - -/****************************************************************************/ -/* memory */ - -/** - * Safe aligned malloc. - * If no memory is available, it aborts. - */ -void* malloc_nofail_align(size_t size, void** freeptr); - -/** - * Safe aligned vector allocation. - * If no memory is available, it aborts. - */ -void** malloc_nofail_vector_align(int nd, int n, size_t size, void** freeptr); - -/** - * Safe allocation with memory test. - */ -void* malloc_nofail_test(size_t size); - -/** - * Tests the memory vector for RAM problems. - * If a problem is found, it crashes. - */ -void mtest_vector(int n, size_t size, void** vv); - /****************************************************************************/ /* hash */