From b31dbf7fff2b9d1fb86fad333f38dcb8a849bc85 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Fri, 9 Aug 2013 13:56:03 +0200 Subject: [PATCH] Adds support for saving the content file in .gz format --- acinclude.m4 | 142 ++++++++++++++++++++++ configure.ac | 1 + portable.h | 4 + state.c | 2 +- test-raid5.conf | 4 +- test-raid6-hole.conf | 6 +- test-raid6.conf | 6 +- util.c | 273 ++++++++++++++++++++++++++++++++++++++----- util.h | 16 ++- 9 files changed, 410 insertions(+), 44 deletions(-) diff --git a/acinclude.m4 b/acinclude.m4 index 1b92cc9e..4acf1f20 100644 --- a/acinclude.m4 +++ b/acinclude.m4 @@ -17,3 +17,145 @@ fi rm -f conftest* ]) +# =========================================================================== +# http://www.gnu.org/software/autoconf-archive/ax_check_zlib.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CHECK_ZLIB([action-if-found], [action-if-not-found]) +# +# DESCRIPTION +# +# This macro searches for an installed zlib library. If nothing was +# specified when calling configure, it searches first in /usr/local and +# then in /usr, /opt/local and /sw. If the --with-zlib=DIR is specified, +# it will try to find it in DIR/include/zlib.h and DIR/lib/libz.a. If +# --without-zlib is specified, the library is not searched at all. +# +# If either the header file (zlib.h) or the library (libz) is not found, +# shell commands 'action-if-not-found' is run. If 'action-if-not-found' is +# not specified, the configuration exits on error, asking for a valid zlib +# installation directory or --without-zlib. +# +# If both header file and library are found, shell commands +# 'action-if-found' is run. If 'action-if-found' is not specified, the +# default action appends '-I${ZLIB_HOME}/include' to CPFLAGS, appends +# '-L$ZLIB_HOME}/lib' to LDFLAGS, prepends '-lz' to LIBS, and calls +# AC_DEFINE(HAVE_LIBZ). You should use autoheader to include a definition +# for this symbol in a config.h file. Sample usage in a C/C++ source is as +# follows: +# +# #ifdef HAVE_LIBZ +# #include +# #endif /* HAVE_LIBZ */ +# +# LICENSE +# +# Copyright (c) 2008 Loic Dachary +# Copyright (c) 2010 Bastien Chevreux +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +# +# As a special exception, the respective Autoconf Macro's copyright owner +# gives unlimited permission to copy, distribute and modify the configure +# scripts that are the output of Autoconf when processing the Macro. You +# need not follow the terms of the GNU General Public License when using +# or distributing such scripts, even though portions of the text of the +# Macro appear in them. The GNU General Public License (GPL) does govern +# all other use of the material that constitutes the Autoconf Macro. +# +# This special exception to the GPL applies to versions of the Autoconf +# Macro released by the Autoconf Archive. When you make and distribute a +# modified version of the Autoconf Macro, you may extend this special +# exception to the GPL to apply to your modified version as well. + +#serial 14 + +AU_ALIAS([CHECK_ZLIB], [AX_CHECK_ZLIB]) +AC_DEFUN([AX_CHECK_ZLIB], +# +# Handle user hints +# +[AC_MSG_CHECKING(if zlib is wanted) +zlib_places="/usr/local /usr /opt/local /sw" +AC_ARG_WITH([zlib], +[ --with-zlib=DIR root directory path of zlib installation @<:@defaults to + /usr/local or /usr if not found in /usr/local@:>@ + --without-zlib to disable zlib usage completely], +[if test "$withval" != no ; then + AC_MSG_RESULT(yes) + if test -d "$withval" + then + zlib_places="$withval $zlib_places" + else + AC_MSG_WARN([Sorry, $withval does not exist, checking usual places]) + fi +else + zlib_places= + AC_MSG_RESULT(no) +fi], +[AC_MSG_RESULT(yes)]) + +# +# Locate zlib, if wanted +# +if test -n "${zlib_places}" +then + # check the user supplied or any other more or less 'standard' place: + # Most UNIX systems : /usr/local and /usr + # MacPorts / Fink on OSX : /opt/local respectively /sw + for ZLIB_HOME in ${zlib_places} ; do + if test -f "${ZLIB_HOME}/include/zlib.h"; then break; fi + ZLIB_HOME="" + done + + ZLIB_OLD_LDFLAGS=$LDFLAGS + ZLIB_OLD_CPPFLAGS=$CPPFLAGS + if test -n "${ZLIB_HOME}"; then + LDFLAGS="$LDFLAGS -L${ZLIB_HOME}/lib" + CPPFLAGS="$CPPFLAGS -I${ZLIB_HOME}/include" + fi + AC_LANG_SAVE + AC_LANG_C + AC_CHECK_LIB([z], [inflateEnd], [zlib_cv_libz=yes], [zlib_cv_libz=no]) + AC_CHECK_HEADER([zlib.h], [zlib_cv_zlib_h=yes], [zlib_cv_zlib_h=no]) + AC_LANG_RESTORE + if test "$zlib_cv_libz" = "yes" && test "$zlib_cv_zlib_h" = "yes" + then + # + # If both library and header were found, action-if-found + # + m4_ifblank([$1],[ + CPPFLAGS="$CPPFLAGS -I${ZLIB_HOME}/include" + LDFLAGS="$LDFLAGS -L${ZLIB_HOME}/lib" + LIBS="-lz $LIBS" + AC_DEFINE([HAVE_LIBZ], [1], + [Define to 1 if you have `z' library (-lz)]) + ],[ + # Restore variables + LDFLAGS="$ZLIB_OLD_LDFLAGS" + CPPFLAGS="$ZLIB_OLD_CPPFLAGS" + $1 + ]) + else + # + # If either header or library was not found, action-if-not-found + # + m4_default([$2],[ + AC_MSG_ERROR([either specify a valid zlib installation with --with-zlib=DIR or disable zlib usage with --without-zlib]) + ]) + fi +fi +]) diff --git a/configure.ac b/configure.ac index 923737bc..837e0e19 100644 --- a/configure.ac +++ b/configure.ac @@ -51,6 +51,7 @@ AC_CHECK_FUNCS([memset strchr strerror strrchr mkdir gettimeofday strtoul]) AC_CHECK_FUNCS([getopt getopt_long snprintf vsnprintf]) AC_CHECK_FUNCS([ftruncate fallocate fsync access posix_fallocate posix_fadvise pread pwrite getc_unlocked ferror_unlocked fnmatch]) AC_CHECK_FUNCS([futimes futimens futimesat fstatat flock]) +AX_CHECK_ZLIB([], []) dnl Checks for architecture AC_C_BIGENDIAN diff --git a/portable.h b/portable.h index d8fd1b96..676a105e 100644 --- a/portable.h +++ b/portable.h @@ -117,6 +117,10 @@ #include "fnmatch.h" #endif +#if HAVE_LIBZ +#include +#endif + /** * Disable case check in Windows. */ diff --git a/state.c b/state.c index 838d7c85..bbd42c08 100644 --- a/state.c +++ b/state.c @@ -2134,7 +2134,7 @@ void state_write(struct snapraid_state* state) /* Use the sequence fflush() -> fsync() -> fclose() -> rename() to ensure */ /* than even in a system crash event we have one valid copy of the file. */ - if (sflush(f) != 0) { + if (sfinish(f) != 0) { fprintf(stderr, "Error writing the content file '%s', in flush(). %s.\n", serrorfile(f), strerror(errno)); exit(EXIT_FAILURE); } diff --git a/test-raid5.conf b/test-raid5.conf index 66846405..7b5b48f7 100644 --- a/test-raid5.conf +++ b/test-raid5.conf @@ -1,7 +1,7 @@ block_size 1 parity test/parity -content test/content -content test/p-content +content test/content.gz +content test/p-content.gz disk disk1 test/disk1/ disk disk2 test/disk2/ disk disk3 test/disk3/ diff --git a/test-raid6-hole.conf b/test-raid6-hole.conf index 03142c67..b97e6253 100644 --- a/test-raid6-hole.conf +++ b/test-raid6-hole.conf @@ -1,9 +1,9 @@ block_size 1 parity test/parity q-parity test/q-parity -content test/content -content test/p-content -content test/q-content +content test/content.gz +content test/p-content.gz +content test/q-content.gz disk disk1 test/disk1/ disk disk3 test/disk3/ exclude *.unrecoverable diff --git a/test-raid6.conf b/test-raid6.conf index ae1bc24d..a6cc3f76 100644 --- a/test-raid6.conf +++ b/test-raid6.conf @@ -1,9 +1,9 @@ block_size 1 parity test/parity q-parity test/q-parity -content test/content -content test/p-content -content test/q-content +content test/content.gz +content test/p-content.gz +content test/q-content.gz disk disk1 test/disk1/ disk disk2 test/disk2/ disk disk3 test/disk3/ diff --git a/util.c b/util.c index 583d9d15..36abbf1f 100644 --- a/util.c +++ b/util.c @@ -77,10 +77,36 @@ STREAM* sopen_read(const char* file) #endif s->buffer = malloc_nofail(STREAM_SIZE); + s->zbuffer = malloc_nofail(STREAM_SIZE); s->pos = s->buffer; s->end = s->buffer; s->state = STREAM_STATE_READ; s->state_index = 0; + s->compress = 0; + + /* get the compress state from the file name */ + if (strstr(file, ".gz") != 0) { + s->compress = -1; + + s->z.zalloc = Z_NULL; + s->z.zfree = Z_NULL; + + s->z.next_in = 0; + s->z.avail_in = 0; + s->z.total_in = 0; + + s->z.next_out = 0; + s->z.avail_out = 0; + s->z.total_out = 0; + + /* +16 initialize for gzip format */ + if (inflateInit2(&s->z, 15 + 16) != Z_OK) { + close(s->handle[0].f); + free(s->handle); + free(s); + return 0; + } + } return s; } @@ -98,10 +124,12 @@ STREAM* sopen_multi_write(unsigned count) s->handle[i].f = -1; s->buffer = malloc_nofail(STREAM_SIZE); + s->zbuffer = malloc_nofail(STREAM_SIZE); s->pos = s->buffer; s->end = s->buffer + STREAM_SIZE; s->state = STREAM_STATE_WRITE; s->state_index = 0; + s->compress = 0; return s; } @@ -131,23 +159,30 @@ int sopen_multi_file(STREAM* s, unsigned i, const char* file) s->handle[i].f = f; - return 0; -} + /* define the compress state from the first file name */ + if (i == 0 && strstr(file, ".gz") != 0) { + s->compress = 1; -STREAM* sopen_write(const char* file) -{ - STREAM* s; + s->z.zalloc = Z_NULL; + s->z.zfree = Z_NULL; - s = sopen_multi_write(1); - if (!s) - return 0; + s->z.next_in = 0; + s->z.avail_in = 0; + s->z.total_in = 0; - if (sopen_multi_file(s, 0, file) != 0) { - sclose(s); - return 0; + /* setup the initial output buffer */ + s->z.next_out = s->zbuffer; + s->z.avail_out = STREAM_SIZE; + s->z.total_out = 0; + + /* initialize for gzip format with a small window to increase speed */ + if (deflateInit2(&s->z, Z_BEST_SPEED, Z_DEFLATED, 8 + 16, 9, Z_HUFFMAN_ONLY) != Z_OK) { + close(f); + return -1; + } } - return s; + return 0; } int sclose(STREAM* s) @@ -165,8 +200,17 @@ int sclose(STREAM* s) fail = 1; } + if (s->compress > 0) { + if (deflateEnd(&s->z) != Z_OK) + fail = 1; + } else if (s->compress < 0) { + if (inflateEnd(&s->z) != Z_OK) + fail = 1; + } + free(s->handle); free(s->buffer); + free(s->zbuffer); free(s); if (fail) @@ -190,19 +234,83 @@ int sfill(STREAM* s) if (s->state != STREAM_STATE_READ) return EOF; - ret = read(s->handle[0].f, s->buffer, STREAM_SIZE); + if (s->compress) { + /* setup the output buffer */ + s->z.next_out = s->buffer; + s->z.avail_out = STREAM_SIZE; - if (ret < 0) { - s->state = STREAM_STATE_ERROR; - return EOF; - } - if (ret == 0) { - s->state = STREAM_STATE_EOF; - return EOF; - } + /* main decompression loop */ + /* we retry until some output is produced */ + loop: + /* decompress something */ + ret = inflate(&s->z, Z_SYNC_FLUSH); - s->pos = s->buffer; - s->end = s->buffer + ret; + /* if no outut was produced and no input is available */ + if (ret == Z_BUF_ERROR && s->z.avail_in == 0 && s->z.avail_out == STREAM_SIZE) { + /* read data */ + ret = read(s->handle[0].f, s->zbuffer, STREAM_SIZE); + + /* if nothing cannot be read, it's surely an error */ + /* even if we reached the end of file */ + if (ret <= 0) { + s->state = STREAM_STATE_ERROR; + return EOF; + } + + /* setup the input buffer */ + s->z.next_in = s->zbuffer; + s->z.avail_in = ret; + + /* retry */ + goto loop; + } + + /* if we have nothing to do and no output to process is present */ + if (ret == Z_STREAM_END) { + /* if no output was produced */ + if (s->z.avail_out == STREAM_SIZE) { + /* end of file */ + s->state = STREAM_STATE_EOF; + return EOF; + } + } else if (ret == Z_BUF_ERROR) { + /* if no output was produced */ + if (s->z.avail_out == STREAM_SIZE) { + /* error */ + s->state = STREAM_STATE_ERROR; + return EOF; + } + } else { + if (ret != Z_OK) { + s->state = STREAM_STATE_ERROR; + return EOF; + } + + /* if no output was produced */ + if (s->z.avail_out == STREAM_SIZE) { + /* retry */ + goto loop; + } + } + + /* now process the output */ + s->pos = s->buffer; + s->end = s->buffer + STREAM_SIZE - s->z.avail_out; + } else { + ret = read(s->handle[0].f, s->buffer, STREAM_SIZE); + + if (ret < 0) { + s->state = STREAM_STATE_ERROR; + return EOF; + } + if (ret == 0) { + s->state = STREAM_STATE_EOF; + return EOF; + } + + s->pos = s->buffer; + s->end = s->buffer + ret; + } return *s->pos++; } @@ -220,13 +328,52 @@ int sflush(STREAM* s) if (!size) return 0; - for(i=0;ihandle_size;++i) { - ret = write(s->handle[i].f, s->buffer, size); + if (s->compress) { + /* setup the input buffer */ + s->z.next_in = s->buffer; + s->z.avail_in = size; + + /* main compression loop */ + /* we retry until consume all the input */ + loop: + /* compress something */ + ret = deflate(&s->z, Z_NO_FLUSH); - if (ret != size) { - s->state = STREAM_STATE_ERROR; - s->state_index = i; - return EOF; + /* if the output buffer is full */ + if (ret == Z_BUF_ERROR && s->z.avail_out == 0) { + + /* write the data */ + for(i=0;ihandle_size;++i) { + ret = write(s->handle[i].f, s->zbuffer, STREAM_SIZE); + + if (ret != STREAM_SIZE) { + s->state = STREAM_STATE_ERROR; + s->state_index = i; + return EOF; + } + } + + /* setup an empty output buffer */ + s->z.next_out = s->zbuffer; + s->z.avail_out = STREAM_SIZE; + + /* retry */ + goto loop; + } + + /* if input is still available */ + if (s->z.avail_in != 0) { + goto loop; + } + } else { + for(i=0;ihandle_size;++i) { + ret = write(s->handle[i].f, s->buffer, size); + + if (ret != size) { + s->state = STREAM_STATE_ERROR; + s->state_index = i; + return EOF; + } } } @@ -235,6 +382,73 @@ int sflush(STREAM* s) return 0; } +int sfinish(STREAM* s) +{ + /* first do a normal flush */ + if (sflush(s) != 0) + return -1; + + if (s->compress) { + int ret; + unsigned i; + + /* main compression loop */ + /* we retry until we rech the stream end */ + loop: + /* compress something */ + ret = deflate(&s->z, Z_FINISH); + + /* if the output buffer is full */ + if (ret == Z_BUF_ERROR && s->z.avail_out == 0) { + + /* write the data */ + for(i=0;ihandle_size;++i) { + ret = write(s->handle[i].f, s->zbuffer, STREAM_SIZE); + + if (ret != STREAM_SIZE) { + s->state = STREAM_STATE_ERROR; + s->state_index = i; + return EOF; + } + } + + /* setup an empty output buffer */ + s->z.next_out = s->zbuffer; + s->z.avail_out = STREAM_SIZE; + + /* retry */ + goto loop; + } + + /* if input is still available */ + if (ret == Z_OK) { + goto loop; + } + if (ret != Z_STREAM_END) { + s->state = STREAM_STATE_ERROR; + s->state_index = -1; + return -1; + } + + /* write the remainig data if any */ + if (s->z.avail_out != STREAM_SIZE) { + int size = STREAM_SIZE - s->z.avail_out; + + for(i=0;ihandle_size;++i) { + ret = write(s->handle[i].f, s->zbuffer, size); + + if (ret != size) { + s->state = STREAM_STATE_ERROR; + s->state_index = i; + return EOF; + } + } + } + } + + return 0; +} + int sgettok(STREAM* f, char* str, int size) { char* i = str; @@ -578,7 +792,6 @@ int sputhex(const void* void_data, int size, STREAM* f) int ssync(STREAM* s) { unsigned i; - for(i=0;ihandle_size;++i) { if (fsync(s->handle[i].f) != 0) { s->state = STREAM_STATE_ERROR; diff --git a/util.h b/util.h index dd1e38a1..bce49d47 100644 --- a/util.h +++ b/util.h @@ -41,6 +41,11 @@ struct stream { int state_index; /**< Index of the handle causing a state change. */ unsigned handle_size; /**< Number of handles. */ struct stream_handle* handle; /**< Set of handles. */ +#if HAVE_LIBZ + int compress; /**< If the file is compressed. */ + unsigned char* zbuffer; /**< Buffer for the compressed data. */ + z_stream z; /**< State for zlib. */ +#endif }; /** @@ -53,11 +58,6 @@ typedef struct stream STREAM; */ STREAM* sopen_read(const char* file); -/** - * Opens a stream for writing. Like fopen("w"). - */ -STREAM* sopen_write(const char* file); - /** * Opens a set of streams for writing. Like fopen("w"). */ @@ -92,6 +92,12 @@ int sfill(STREAM* s); */ int sflush(STREAM* s); +/** + * Finish the write stream buffer. + * It can be called just before sclose() or ssync() to ensure that all the output is written. + */ +int sfinish(STREAM* s); + /** * Checks if the buffer has enough data loaded. */