rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-25 18:40:19 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Adds support for saving the content file in .gz format

Browse source
This commit is contained in:
Andrea Mazzoleni 2013-08-09 13:56:03 +02:00
parent b9849c82b9
commit b31dbf7fff
9 changed files with 410 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

142
acinclude.m4
View file

@ -17,3 +17,145 @@ fi
rm -f conftest*
])
# ===========================================================================
# http://www.gnu.org/software/autoconf-archive/ax_check_zlib.html
# ===========================================================================
#
# SYNOPSIS
#
# AX_CHECK_ZLIB([action-if-found], [action-if-not-found])
#
# DESCRIPTION
#
# This macro searches for an installed zlib library. If nothing was
# specified when calling configure, it searches first in /usr/local and
# then in /usr, /opt/local and /sw. If the --with-zlib=DIR is specified,
# it will try to find it in DIR/include/zlib.h and DIR/lib/libz.a. If
# --without-zlib is specified, the library is not searched at all.
#
# If either the header file (zlib.h) or the library (libz) is not found,
# shell commands 'action-if-not-found' is run. If 'action-if-not-found' is
# not specified, the configuration exits on error, asking for a valid zlib
# installation directory or --without-zlib.
#
# If both header file and library are found, shell commands
# 'action-if-found' is run. If 'action-if-found' is not specified, the
# default action appends '-I${ZLIB_HOME}/include' to CPFLAGS, appends
# '-L$ZLIB_HOME}/lib' to LDFLAGS, prepends '-lz' to LIBS, and calls
# AC_DEFINE(HAVE_LIBZ). You should use autoheader to include a definition
# for this symbol in a config.h file. Sample usage in a C/C++ source is as
# follows:
#
# #ifdef HAVE_LIBZ
# #include <zlib.h>
# #endif /* HAVE_LIBZ */
#
# LICENSE
#
# Copyright (c) 2008 Loic Dachary <loic@senga.org>
# Copyright (c) 2010 Bastien Chevreux <bach@chevreux.org>
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
#
# As a special exception, the respective Autoconf Macro's copyright owner
# gives unlimited permission to copy, distribute and modify the configure
# scripts that are the output of Autoconf when processing the Macro. You
# need not follow the terms of the GNU General Public License when using
# or distributing such scripts, even though portions of the text of the
# Macro appear in them. The GNU General Public License (GPL) does govern
# all other use of the material that constitutes the Autoconf Macro.
#
# This special exception to the GPL applies to versions of the Autoconf
# Macro released by the Autoconf Archive. When you make and distribute a
# modified version of the Autoconf Macro, you may extend this special
# exception to the GPL to apply to your modified version as well.
#serial 14
AU_ALIAS([CHECK_ZLIB], [AX_CHECK_ZLIB])
AC_DEFUN([AX_CHECK_ZLIB],
#
# Handle user hints
#
[AC_MSG_CHECKING(if zlib is wanted)
zlib_places="/usr/local /usr /opt/local /sw"
AC_ARG_WITH([zlib],
[ --with-zlib=DIR root directory path of zlib installation @<:@defaults to
/usr/local or /usr if not found in /usr/local@:>@
--without-zlib to disable zlib usage completely],
[if test "$withval" != no ; then
AC_MSG_RESULT(yes)
if test -d "$withval"
then
zlib_places="$withval $zlib_places"
else
AC_MSG_WARN([Sorry, $withval does not exist, checking usual places])
fi
else
zlib_places=
AC_MSG_RESULT(no)
fi],
[AC_MSG_RESULT(yes)])
#
# Locate zlib, if wanted
#
if test -n "${zlib_places}"
then
# check the user supplied or any other more or less 'standard' place:
# Most UNIX systems : /usr/local and /usr
# MacPorts / Fink on OSX : /opt/local respectively /sw
for ZLIB_HOME in ${zlib_places} ; do
if test -f "${ZLIB_HOME}/include/zlib.h"; then break; fi
ZLIB_HOME=""
done
ZLIB_OLD_LDFLAGS=$LDFLAGS
ZLIB_OLD_CPPFLAGS=$CPPFLAGS
if test -n "${ZLIB_HOME}"; then
LDFLAGS="$LDFLAGS -L${ZLIB_HOME}/lib"
CPPFLAGS="$CPPFLAGS -I${ZLIB_HOME}/include"
fi
AC_LANG_SAVE
AC_LANG_C
AC_CHECK_LIB([z], [inflateEnd], [zlib_cv_libz=yes], [zlib_cv_libz=no])
AC_CHECK_HEADER([zlib.h], [zlib_cv_zlib_h=yes], [zlib_cv_zlib_h=no])
AC_LANG_RESTORE
if test "$zlib_cv_libz" = "yes" && test "$zlib_cv_zlib_h" = "yes"
then
#
# If both library and header were found, action-if-found
#
m4_ifblank([$1],[
CPPFLAGS="$CPPFLAGS -I${ZLIB_HOME}/include"
LDFLAGS="$LDFLAGS -L${ZLIB_HOME}/lib"
LIBS="-lz $LIBS"
AC_DEFINE([HAVE_LIBZ], [1],
[Define to 1 if you have `z' library (-lz)])
],[
# Restore variables
LDFLAGS="$ZLIB_OLD_LDFLAGS"
CPPFLAGS="$ZLIB_OLD_CPPFLAGS"
$1
])
else
#
# If either header or library was not found, action-if-not-found
#
m4_default([$2],[
AC_MSG_ERROR([either specify a valid zlib installation with --with-zlib=DIR or disable zlib usage with --without-zlib])
])
fi
fi
])

1
configure.ac
View file

@ -51,6 +51,7 @@ AC_CHECK_FUNCS([memset strchr strerror strrchr mkdir gettimeofday strtoul])
AC_CHECK_FUNCS([getopt getopt_long snprintf vsnprintf])
AC_CHECK_FUNCS([ftruncate fallocate fsync access posix_fallocate posix_fadvise pread pwrite getc_unlocked ferror_unlocked fnmatch])
AC_CHECK_FUNCS([futimes futimens futimesat fstatat flock])
AX_CHECK_ZLIB([], [])
dnl Checks for architecture
AC_C_BIGENDIAN

4
portable.h
View file

@ -117,6 +117,10 @@
#include "fnmatch.h"
#endif
#if HAVE_LIBZ
#include <zlib.h>
#endif
/**
* Disable case check in Windows.
*/

2
state.c
View file

@ -2134,7 +2134,7 @@ void state_write(struct snapraid_state* state)
/* Use the sequence fflush() -> fsync() -> fclose() -> rename() to ensure */
/* than even in a system crash event we have one valid copy of the file. */
if (sflush(f) != 0) {
if (sfinish(f) != 0) {
fprintf(stderr, "Error writing the content file '%s', in flush(). %s.\n", serrorfile(f), strerror(errno));
exit(EXIT_FAILURE);
}

4
test-raid5.conf
View file

@ -1,7 +1,7 @@
block_size 1
parity test/parity
content test/content
content test/p-content
content test/content.gz
content test/p-content.gz
disk disk1 test/disk1/
disk disk2 test/disk2/
disk disk3 test/disk3/

6
test-raid6-hole.conf
View file

@ -1,9 +1,9 @@
block_size 1
parity test/parity
q-parity test/q-parity
content test/content
content test/p-content
content test/q-content
content test/content.gz
content test/p-content.gz
content test/q-content.gz
disk disk1 test/disk1/
disk disk3 test/disk3/
exclude *.unrecoverable

6
test-raid6.conf
View file

@ -1,9 +1,9 @@
block_size 1
parity test/parity
q-parity test/q-parity
content test/content
content test/p-content
content test/q-content
content test/content.gz
content test/p-content.gz
content test/q-content.gz
disk disk1 test/disk1/
disk disk2 test/disk2/
disk disk3 test/disk3/

273
util.c
View file

@ -77,10 +77,36 @@ STREAM* sopen_read(const char* file)
#endif
s->buffer = malloc_nofail(STREAM_SIZE);
s->zbuffer = malloc_nofail(STREAM_SIZE);
s->pos = s->buffer;
s->end = s->buffer;
s->state = STREAM_STATE_READ;
s->state_index = 0;
s->compress = 0;
/* get the compress state from the file name */
if (strstr(file, ".gz") != 0) {
s->compress = -1;
s->z.zalloc = Z_NULL;
s->z.zfree = Z_NULL;
s->z.next_in = 0;
s->z.avail_in = 0;
s->z.total_in = 0;
s->z.next_out = 0;
s->z.avail_out = 0;
s->z.total_out = 0;
/* +16 initialize for gzip format */
if (inflateInit2(&s->z, 15 + 16) != Z_OK) {
close(s->handle[0].f);
free(s->handle);
free(s);
return 0;
}
}
return s;
}
@ -98,10 +124,12 @@ STREAM* sopen_multi_write(unsigned count)
s->handle[i].f = -1;
s->buffer = malloc_nofail(STREAM_SIZE);
s->zbuffer = malloc_nofail(STREAM_SIZE);
s->pos = s->buffer;
s->end = s->buffer + STREAM_SIZE;
s->state = STREAM_STATE_WRITE;
s->state_index = 0;
s->compress = 0;
return s;
}
@ -131,23 +159,30 @@ int sopen_multi_file(STREAM* s, unsigned i, const char* file)
s->handle[i].f = f;
return 0;
}
/* define the compress state from the first file name */
if (i == 0 && strstr(file, ".gz") != 0) {
s->compress = 1;
STREAM* sopen_write(const char* file)
{
STREAM* s;
s->z.zalloc = Z_NULL;
s->z.zfree = Z_NULL;
s = sopen_multi_write(1);
if (!s)
return 0;
s->z.next_in = 0;
s->z.avail_in = 0;
s->z.total_in = 0;
if (sopen_multi_file(s, 0, file) != 0) {
sclose(s);
return 0;
/* setup the initial output buffer */
s->z.next_out = s->zbuffer;
s->z.avail_out = STREAM_SIZE;
s->z.total_out = 0;
/* initialize for gzip format with a small window to increase speed */
if (deflateInit2(&s->z, Z_BEST_SPEED, Z_DEFLATED, 8 + 16, 9, Z_HUFFMAN_ONLY) != Z_OK) {
close(f);
return -1;
}
}
return s;
return 0;
}
int sclose(STREAM* s)
@ -165,8 +200,17 @@ int sclose(STREAM* s)
fail = 1;
}
if (s->compress > 0) {
if (deflateEnd(&s->z) != Z_OK)
fail = 1;
} else if (s->compress < 0) {
if (inflateEnd(&s->z) != Z_OK)
fail = 1;
}
free(s->handle);
free(s->buffer);
free(s->zbuffer);
free(s);
if (fail)
@ -190,19 +234,83 @@ int sfill(STREAM* s)
if (s->state != STREAM_STATE_READ)
return EOF;
ret = read(s->handle[0].f, s->buffer, STREAM_SIZE);
if (s->compress) {
/* setup the output buffer */
s->z.next_out = s->buffer;
s->z.avail_out = STREAM_SIZE;
if (ret < 0) {
s->state = STREAM_STATE_ERROR;
return EOF;
}
if (ret == 0) {
s->state = STREAM_STATE_EOF;
return EOF;
}
/* main decompression loop */
/* we retry until some output is produced */
loop:
/* decompress something */
ret = inflate(&s->z, Z_SYNC_FLUSH);
s->pos = s->buffer;
s->end = s->buffer + ret;
/* if no outut was produced and no input is available */
if (ret == Z_BUF_ERROR && s->z.avail_in == 0 && s->z.avail_out == STREAM_SIZE) {
/* read data */
ret = read(s->handle[0].f, s->zbuffer, STREAM_SIZE);
/* if nothing cannot be read, it's surely an error */
/* even if we reached the end of file */
if (ret <= 0) {
s->state = STREAM_STATE_ERROR;
return EOF;
}
/* setup the input buffer */
s->z.next_in = s->zbuffer;
s->z.avail_in = ret;
/* retry */
goto loop;
}
/* if we have nothing to do and no output to process is present */
if (ret == Z_STREAM_END) {
/* if no output was produced */
if (s->z.avail_out == STREAM_SIZE) {
/* end of file */
s->state = STREAM_STATE_EOF;
return EOF;
}
} else if (ret == Z_BUF_ERROR) {
/* if no output was produced */
if (s->z.avail_out == STREAM_SIZE) {
/* error */
s->state = STREAM_STATE_ERROR;
return EOF;
}
} else {
if (ret != Z_OK) {
s->state = STREAM_STATE_ERROR;
return EOF;
}
/* if no output was produced */
if (s->z.avail_out == STREAM_SIZE) {
/* retry */
goto loop;
}
}
/* now process the output */
s->pos = s->buffer;
s->end = s->buffer + STREAM_SIZE - s->z.avail_out;
} else {
ret = read(s->handle[0].f, s->buffer, STREAM_SIZE);
if (ret < 0) {
s->state = STREAM_STATE_ERROR;
return EOF;
}
if (ret == 0) {
s->state = STREAM_STATE_EOF;
return EOF;
}
s->pos = s->buffer;
s->end = s->buffer + ret;
}
return *s->pos++;
}
@ -220,13 +328,52 @@ int sflush(STREAM* s)
if (!size)
return 0;
for(i=0;i<s->handle_size;++i) {
ret = write(s->handle[i].f, s->buffer, size);
if (s->compress) {
/* setup the input buffer */
s->z.next_in = s->buffer;
s->z.avail_in = size;
/* main compression loop */
/* we retry until consume all the input */
loop:
/* compress something */
ret = deflate(&s->z, Z_NO_FLUSH);
if (ret != size) {
s->state = STREAM_STATE_ERROR;
s->state_index = i;
return EOF;
/* if the output buffer is full */
if (ret == Z_BUF_ERROR && s->z.avail_out == 0) {
/* write the data */
for(i=0;i<s->handle_size;++i) {
ret = write(s->handle[i].f, s->zbuffer, STREAM_SIZE);
if (ret != STREAM_SIZE) {
s->state = STREAM_STATE_ERROR;
s->state_index = i;
return EOF;
}
}
/* setup an empty output buffer */
s->z.next_out = s->zbuffer;
s->z.avail_out = STREAM_SIZE;
/* retry */
goto loop;
}
/* if input is still available */
if (s->z.avail_in != 0) {
goto loop;
}
} else {
for(i=0;i<s->handle_size;++i) {
ret = write(s->handle[i].f, s->buffer, size);
if (ret != size) {
s->state = STREAM_STATE_ERROR;
s->state_index = i;
return EOF;
}
}
}
@ -235,6 +382,73 @@ int sflush(STREAM* s)
return 0;
}
int sfinish(STREAM* s)
{
/* first do a normal flush */
if (sflush(s) != 0)
return -1;
if (s->compress) {
int ret;
unsigned i;
/* main compression loop */
/* we retry until we rech the stream end */
loop:
/* compress something */
ret = deflate(&s->z, Z_FINISH);
/* if the output buffer is full */
if (ret == Z_BUF_ERROR && s->z.avail_out == 0) {
/* write the data */
for(i=0;i<s->handle_size;++i) {
ret = write(s->handle[i].f, s->zbuffer, STREAM_SIZE);
if (ret != STREAM_SIZE) {
s->state = STREAM_STATE_ERROR;
s->state_index = i;
return EOF;
}
}
/* setup an empty output buffer */
s->z.next_out = s->zbuffer;
s->z.avail_out = STREAM_SIZE;
/* retry */
goto loop;
}
/* if input is still available */
if (ret == Z_OK) {
goto loop;
}
if (ret != Z_STREAM_END) {
s->state = STREAM_STATE_ERROR;
s->state_index = -1;
return -1;
}
/* write the remainig data if any */
if (s->z.avail_out != STREAM_SIZE) {
int size = STREAM_SIZE - s->z.avail_out;
for(i=0;i<s->handle_size;++i) {
ret = write(s->handle[i].f, s->zbuffer, size);
if (ret != size) {
s->state = STREAM_STATE_ERROR;
s->state_index = i;
return EOF;
}
}
}
}
return 0;
}
int sgettok(STREAM* f, char* str, int size)
{
char* i = str;
@ -578,7 +792,6 @@ int sputhex(const void* void_data, int size, STREAM* f)
int ssync(STREAM* s)
{
unsigned i;
for(i=0;i<s->handle_size;++i) {
if (fsync(s->handle[i].f) != 0) {
s->state = STREAM_STATE_ERROR;

16
util.h
View file

@ -41,6 +41,11 @@ struct stream {
int state_index; /**< Index of the handle causing a state change. */
unsigned handle_size; /**< Number of handles. */
struct stream_handle* handle; /**< Set of handles. */
#if HAVE_LIBZ
int compress; /**< If the file is compressed. */
unsigned char* zbuffer; /**< Buffer for the compressed data. */
z_stream z; /**< State for zlib. */
#endif
};
/**
@ -53,11 +58,6 @@ typedef struct stream STREAM;
*/
STREAM* sopen_read(const char* file);
/**
* Opens a stream for writing. Like fopen("w").
*/
STREAM* sopen_write(const char* file);
/**
* Opens a set of streams for writing. Like fopen("w").
*/
@ -92,6 +92,12 @@ int sfill(STREAM* s);
*/
int sflush(STREAM* s);
/**
* Finish the write stream buffer.
* It can be called just before sclose() or ssync() to ensure that all the output is written.
*/
int sfinish(STREAM* s);
/**
* Checks if the buffer has enough data loaded.
*/