Import lz4 r104

lz4/Makefile

@@ -1,6 +1,38 @@
+# ################################################################
+# LZ4 Makefile
+# Copyright (C) Yann Collet 2011-2013
+# GPL v2 License
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# You can contact the author at :
+# - LZ4 source repository : http://code.google.com/p/lz4/
+# ################################################################
+# lz4 : Command Line Utility, supporting gzip-like arguments
+# lz4c : CLU, supporting also legacy lz4demo arguments
+# lz4c32: Same as lz4c, but forced to compile in 32-bits mode
+# fuzzer : Test tool, to check lz4 integrity on target platform
+# fuzzer32: Same as fuzzer, but forced to compile in 32-bits mode
+# fullbench : Precisely measure speed for each LZ4 function variant
+# fullbench32: Same as fullbench, but forced to compile in 32-bits mode
+# ################################################################
+
 CC=gcc
 CFLAGS=-I. -std=c99 -Wall -W -Wundef -Wno-implicit-function-declaration
 
+# Define *.exe as extension for Windows systems
 OS := $(shell uname)
 ifeq ($(OS),Linux)
 EXT =
@@ -10,20 +42,21 @@ endif
 
 default: lz4c
 
-all: lz4c lz4cs lz4c32 fuzzer fuzzer32 fullbench fullbench32
+all: lz4 lz4c lz4c32 fuzzer fuzzer32 fullbench fullbench32
 
-lz4c: lz4.c lz4hc.c bench.c xxhash.c lz4c.c
- $(CC) -O3 $(CFLAGS) $^ -o $@$(EXT)
+lz4: lz4.c lz4hc.c bench.c xxhash.c lz4cli.c
+ $(CC) -O3 $(CFLAGS) -DDISABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT)
 
-lz4cs: lz4.c lz4hc.c bench.c xxhash.c lz4c.c
- $(CC) -Os $(CFLAGS) $^ -o $@$(EXT)
+lz4c: lz4.c lz4hc.c bench.c xxhash.c lz4cli.c
+ $(CC) -O3 $(CFLAGS) $^ -o $@$(EXT)
 
-lz4c32: lz4.c lz4hc.c bench.c xxhash.c lz4c.c
+lz4c32: lz4.c lz4hc.c bench.c xxhash.c lz4cli.c
  $(CC) -m32 -O3 $(CFLAGS) $^ -o $@$(EXT)
 
 fuzzer : lz4.c lz4hc.c fuzzer.c
+ @echo fuzzer is a test tool to check lz4 integrity on target platform
  $(CC) -O3 $(CFLAGS) $^ -o $@$(EXT)
- 
+
 fuzzer32 : lz4.c lz4hc.c fuzzer.c
  $(CC) -m32 -O3 $(CFLAGS) $^ -o $@$(EXT)
 
@@ -34,4 +67,4 @@ fullbench32 : lz4.c lz4hc.c xxhash.c fullbench.c
  $(CC) -m32 -O3 $(CFLAGS) $^ -o $@$(EXT)
 
 clean:
- rm -f core *.o lz4c$(EXT) lz4cs$(EXT) lz4c32$(EXT) fuzzer$(EXT) fullbench$(EXT) fullbench32$(EXT)
+ rm -f core *.o lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) fuzzer$(EXT) fuzzer32$(EXT) fullbench$(EXT) fullbench32$(EXT)

lz4/bench.c

@@ -143,11 +143,7 @@ static int chunkSize = DEFAULT_CHUNKSIZE;
 static int nbIterations = NBLOOPS;
 static int BMK_pause = 0;
 
-void BMK_SetBlocksize(int bsize)
-{
- chunkSize = bsize;
- DISPLAY("-Using Block Size of %i KB-\n", chunkSize>>10);
-}
+void BMK_SetBlocksize(int bsize) { chunkSize = bsize; }
 
 void BMK_SetNbIterations(int nbLoops)
 {

lz4/cmake/CMakeLists.txt

@@ -32,7 +32,7 @@ endif()
 
 set(SRC_DIR ../)
 set(LZ4_SRCS_LIB ${SRC_DIR}lz4.c ${SRC_DIR}lz4hc.c ${SRC_DIR}lz4.h )
-set(LZ4_SRCS ${SRC_DIR}xxhash.c ${SRC_DIR}bench.c ${SRC_DIR}lz4c.c)
+set(LZ4_SRCS ${SRC_DIR}xxhash.c ${SRC_DIR}bench.c ${SRC_DIR}lz4cli.c)
 
 if(NOT BUILD_SHARED_LIBS)
  set(LZ4_SRCS ${LZ4_SRCS} ${LZ4_SRCS_LIB})

lz4/cmake/pack/CMakeLists.txt

@@ -42,7 +42,7 @@ INCLUDE_DIRECTORIES (${SRC_DIR})
 
 
 set(LZ4_SRCS_LIB ${SRC_DIR}lz4.c ${SRC_DIR}lz4hc.c ${SRC_DIR}lz4.h ${SRC_DIR}lz4_format_description.txt)
-set(LZ4_SRCS ${LZ4_SRCS_LIB} ${SRC_DIR}xxhash.c ${SRC_DIR}bench.c ${SRC_DIR}lz4c.c )
+set(LZ4_SRCS ${LZ4_SRCS_LIB} ${SRC_DIR}xxhash.c ${SRC_DIR}bench.c ${SRC_DIR}lz4cli.c )
 set(FUZZER_SRCS ${SRC_DIR}lz4.c ${SRC_DIR}lz4hc.c ${SRC_DIR}lz4.h ${SRC_DIR}fuzzer.c)
 
 # EXECUTABLES FOR 32 Bit and 64 versions

lz4/fullbench.c

@@ -103,11 +103,11 @@
 //****************************
 // Constants
 //****************************
-#define COMPRESSOR_NAME "Full LZ4 speed analyzer"
+#define COMPRESSOR_NAME "LZ4 speed analyzer"
 #define COMPRESSOR_VERSION ""
 #define COMPILED __DATE__
 #define AUTHOR "Yann Collet"
-#define WELCOME_MESSAGE "*** %s %s, by %s (%s) ***\n", COMPRESSOR_NAME, COMPRESSOR_VERSION, AUTHOR, COMPILED
+#define WELCOME_MESSAGE "*** %s %s %i-bits, by %s (%s) ***\n", COMPRESSOR_NAME, COMPRESSOR_VERSION, (int)(sizeof(void*)*8), AUTHOR, COMPILED
 
 #define NBLOOPS 6
 #define TIMELOOP 2500

lz4/lz4.c

@@ -177,16 +177,14 @@
  typedef unsigned long long U64;
 #endif
 
-typedef const BYTE* Ptr;
-
 #if defined(__GNUC__) && !defined(LZ4_FORCE_UNALIGNED_ACCESS)
 # define _PACKED __attribute__ ((packed))
 #else
 # define _PACKED
 #endif
 
 #if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__)
-# ifdef __IBMC__
+# if defined(__IBMC__) || defined(__SUNPRO_C) || defined(__SUNPRO_CC)
 # pragma pack(1)
 # else
 # pragma pack(push, 1)
@@ -199,7 +197,11 @@ typedef struct { U64 v; } _PACKED U64_S;
 typedef struct {size_t v;} _PACKED size_t_S;
 
 #if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__)
-# pragma pack(pop)
+# if defined(__SUNPRO_C) || defined(__SUNPRO_CC)
+# pragma pack(0)
+# else
+# pragma pack(pop)
+# endif
 #endif
 
 #define A16(x) (((U16_S *)(x))->v)
@@ -244,9 +246,9 @@ const int LZ4_minLength = (MFLIMIT+1);
 
 typedef struct {
  U32 hashTable[HASHNBCELLS4];
- Ptr bufferStart;
- Ptr base;
- Ptr nextBlock;
+ const BYTE* bufferStart;
+ const BYTE* base;
+ const BYTE* nextBlock;
 } LZ4_Data_Structure;
 
 typedef enum { notLimited = 0, limited = 1 } limitedOutput_directive;
@@ -268,9 +270,9 @@ typedef enum { full = 0, partial = 1 } earlyEnd_directive;
 
 #if LZ4_ARCH64 // 64-bit
 # define HTYPE U32
-# define INITBASE(base) Ptr const base = ip
+# define INITBASE(base) const BYTE* const base = ip
 #else // 32-bit
-# define HTYPE Ptr
+# define HTYPE const BYTE*
 # define INITBASE(base) const int base = 0
 #endif
 
@@ -369,32 +371,32 @@ FORCE_INLINE int LZ4_hashSequence(U32 sequence, tableType_t tableType)
  return (((sequence) * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG));
 }
 
-FORCE_INLINE int LZ4_hashPosition(Ptr p, tableType_t tableType) { return LZ4_hashSequence(A32(p), tableType); }
+FORCE_INLINE int LZ4_hashPosition(const BYTE* p, tableType_t tableType) { return LZ4_hashSequence(A32(p), tableType); }
 
-FORCE_INLINE void LZ4_putPositionOnHash(Ptr p, U32 h, void* tableBase, tableType_t tableType, Ptr srcBase)
+FORCE_INLINE void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase)
 {
  switch (tableType)
  {
- case byPtr: { Ptr* hashTable = (Ptr*) tableBase; hashTable[h] = p; break; }
- case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = p-srcBase; break; }
+ case byPtr: { const BYTE** hashTable = (const BYTE**) tableBase; hashTable[h] = p; break; }
+ case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); break; }
  case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); break; }
  }
 }
 
-FORCE_INLINE void LZ4_putPosition(Ptr p, void* tableBase, tableType_t tableType, Ptr srcBase)
+FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
 {
  U32 h = LZ4_hashPosition(p, tableType);
  LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase);
 }
 
-FORCE_INLINE Ptr LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, Ptr srcBase)
+FORCE_INLINE const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase)
 {
- if (tableType == byPtr) { Ptr* hashTable = (Ptr*) tableBase; return hashTable[h]; }
+ if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; }
  if (tableType == byU32) { U32* hashTable = (U32*) tableBase; return hashTable[h] + srcBase; }
  { U16* hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } // default, to ensure a return
 }
 
-FORCE_INLINE Ptr LZ4_getPosition(Ptr p, void* tableBase, tableType_t tableType, Ptr srcBase)
+FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
 {
  U32 h = LZ4_hashPosition(p, tableType);
  return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase);
@@ -412,13 +414,13 @@ FORCE_INLINE int LZ4_compress_generic(
  tableType_t tableType,
  prefix64k_directive prefix)
 {
- Ptr ip = (Ptr) source;
- Ptr const base = (prefix==withPrefix) ? ((LZ4_Data_Structure*)ctx)->base : (Ptr) source;
- Ptr const lowLimit = ((prefix==withPrefix) ? ((LZ4_Data_Structure*)ctx)->bufferStart : (Ptr)source);
- Ptr anchor = (Ptr) source;
- Ptr const iend = ip + inputSize;
- Ptr const mflimit = iend - MFLIMIT;
- Ptr const matchlimit = iend - LASTLITERALS;
+ const BYTE* ip = (const BYTE*) source;
+ const BYTE* const base = (prefix==withPrefix) ? ((LZ4_Data_Structure*)ctx)->base : (const BYTE*) source;
+ const BYTE* const lowLimit = ((prefix==withPrefix) ? ((LZ4_Data_Structure*)ctx)->bufferStart : (const BYTE*)source);
+ const BYTE* anchor = (const BYTE*) source;
+ const BYTE* const iend = ip + inputSize;
+ const BYTE* const mflimit = iend - MFLIMIT;
+ const BYTE* const matchlimit = iend - LASTLITERALS;
 
  BYTE* op = (BYTE*) dest;
  BYTE* const oend = op + maxOutputSize;
@@ -441,8 +443,8 @@ FORCE_INLINE int LZ4_compress_generic(
  for ( ; ; )
  {
  int findMatchAttempts = (1U << skipStrength) + 3;
- Ptr forwardIp = ip;
- Ptr ref;
+ const BYTE* forwardIp = ip;
+ const BYTE* ref;
  BYTE* token;
 
  // Find a match
@@ -599,7 +601,7 @@ int LZ4_compress_limitedOutput_continue (void* LZ4_Data, const char* source, cha
 // Stream functions
 //****************************
 
-FORCE_INLINE void LZ4_init(LZ4_Data_Structure* lz4ds, Ptr base)
+FORCE_INLINE void LZ4_init(LZ4_Data_Structure* lz4ds, const BYTE* base)
 {
  MEM_INIT(lz4ds->hashTable, 0, sizeof(lz4ds->hashTable));
  lz4ds->bufferStart = base;
@@ -611,7 +613,7 @@ FORCE_INLINE void LZ4_init(LZ4_Data_Structure* lz4ds, Ptr base)
 void* LZ4_create (const char* inputBuffer)
 {
  void* lz4ds = ALLOCATOR(1, sizeof(LZ4_Data_Structure));
- LZ4_init ((LZ4_Data_Structure*)lz4ds, (Ptr)inputBuffer);
+ LZ4_init ((LZ4_Data_Structure*)lz4ds, (const BYTE*)inputBuffer);
  return lz4ds;
 }
 
@@ -636,7 +638,7 @@ char* LZ4_slideInputBuffer (void* LZ4_Data)
  for (nH=0; nH < HASHNBCELLS4; nH++)
  {
  if (lz4ds->hashTable[nH] < (U32)newBaseDelta) lz4ds->hashTable[nH] = 0;
- else lz4ds->hashTable[nH] -= newBaseDelta;
+ else lz4ds->hashTable[nH] -= (U32)newBaseDelta;
  }
  lz4ds->base += newBaseDelta;
  }
@@ -660,7 +662,7 @@ FORCE_INLINE int LZ4_decompress_generic(
  const char* source,
  char* dest,
  int inputSize, //
- int outputSize, // OutputSize must be != 0; if endOnInput==endOnInputSize, this value is the max size of Output Buffer.
+ int outputSize, // If endOnInput==endOnInputSize, this value is the max size of Output Buffer.
 
  int endOnInput, // endOnOutputSize, endOnInputSize
  int prefix64k, // noPrefix, withPrefix
@@ -669,19 +671,17 @@ FORCE_INLINE int LZ4_decompress_generic(
  )
 {
  // Local Variables
- Ptr restrict ip = (Ptr) source;
- Ptr ref;
- Ptr const iend = ip + inputSize;
+ const BYTE* restrict ip = (const BYTE*) source;
+ const BYTE* ref;
+ const BYTE* const iend = ip + inputSize;
 
  BYTE* op = (BYTE*) dest;
  BYTE* const oend = op + outputSize;
  BYTE* cpy;
  BYTE* oexit = op + targetOutputSize;
 
- size_t dec32table[] = {0, 3, 2, 3, 0, 0, 0, 0};
-#if LZ4_ARCH64
- size_t dec64table[] = {0, 0, 0, (size_t)-1, 0, 1, 2, 3};
-#endif
+ const size_t dec32table[] = {0, 3, 2, 3, 0, 0, 0, 0}; // static reduces speed for LZ4_decompress_safe() on GCC64
+ static const size_t dec64table[] = {0, 0, 0, (size_t)-1, 0, 1, 2, 3};
 
 
  // Special cases
@@ -749,11 +749,7 @@ FORCE_INLINE int LZ4_decompress_generic(
  // copy repeated sequence
  if unlikely((op-ref)<(int)STEPSIZE)
  {
-#if LZ4_ARCH64
- size_t dec64 = dec64table[op-ref];
-#else
- const size_t dec64 = 0;
-#endif
+ const size_t dec64 = dec64table[(sizeof(void*)==4) ? 0 : op-ref];
  op[0] = ref[0];
  op[1] = ref[1];
  op[2] = ref[2];
@@ -803,6 +799,11 @@ int LZ4_decompress_safe_partial(const char* source, char* dest, int inputSize, i
  return LZ4_decompress_generic(source, dest, inputSize, maxOutputSize, endOnInputSize, noPrefix, partial, targetOutputSize);
 }
 
+int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int outputSize)
+{
+ return LZ4_decompress_generic(source, dest, 0, outputSize, endOnOutputSize, withPrefix, full, 0);
+}
+
 int LZ4_decompress_fast(const char* source, char* dest, int outputSize)
 {
 #ifdef _MSC_VER // This version is faster with Visual
@@ -812,8 +813,3 @@ int LZ4_decompress_fast(const char* source, char* dest, int outputSize)
 #endif
 }
 
-int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int outputSize)
-{
- return LZ4_decompress_generic(source, dest, 0, outputSize, endOnOutputSize, withPrefix, full, 0);
-}
-

lz4/lz4.h

@@ -83,7 +83,7 @@ LZ4_compressBound() :
  Provides the maximum size that LZ4 may output in a "worst case" scenario (input data not compressible)
  primarily useful for memory allocation of output buffer.
  inline function is recommended for the general case,
- macro is also provided when result needs to be evaluated at compilation (such as table size allocation).
+ macro is also provided when result needs to be evaluated at compilation (such as stack memory allocation).
 
  isize : is the input size. Max supported value is ~1.9GB
  return : maximum output size in a "worst case" scenario
@@ -130,23 +130,11 @@ LZ4_decompress_safe_partial() :
  return : the number of bytes decoded in the destination buffer (necessarily <= maxOutputSize)
  Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller.
  Always control how many bytes were decoded.
- If the source stream is malformed, the function will stop decoding and return a negative result.
+ If the source stream is detected malformed, the function will stop decoding and return a negative result.
  This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets
 */
 
 
-int LZ4_decompress_safe_withPrefix64k (const char* source, char* dest, int inputSize, int maxOutputSize);
-int LZ4_decompress_fast_withPrefix64k (const char* source, char* dest, int outputSize);
-
-/*
-*_withPrefix64k() :
- These decoding functions work the same as their "normal name" versions,
- but will potentially use up to 64KB of data in front of 'char* dest'.
- These functions are used for decoding inter-dependant blocks.
-*/
-
-
-
 //****************************
 // Stream Functions
 //****************************
@@ -187,6 +175,17 @@ When compression is completed, a call to LZ4_free() will release the memory used
 */
 
 
+int LZ4_decompress_safe_withPrefix64k (const char* source, char* dest, int inputSize, int maxOutputSize);
+int LZ4_decompress_fast_withPrefix64k (const char* source, char* dest, int outputSize);
+
+/*
+*_withPrefix64k() :
+ These decoding functions work the same as their "normal name" versions,
+ but can use up to 64KB of data in front of 'char* dest'.
+ These functions are necessary to decode inter-dependant blocks.
+*/
+
+
 //****************************
 // Obsolete Functions
 //****************************