Adam Ierymenko
13 years ago
33 changed files with 793 additions and 1930 deletions
@ -1,477 +0,0 @@
|
||||
/*
|
||||
bench.c - Demo program to benchmark open-source compression algorithm |
||||
Copyright (C) Yann Collet 2012 |
||||
GPL v2 License |
||||
|
||||
This program is free software; you can redistribute it and/or modify |
||||
it under the terms of the GNU General Public License as published by |
||||
the Free Software Foundation; either version 2 of the License, or |
||||
(at your option) any later version. |
||||
|
||||
This program is distributed in the hope that it will be useful, |
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of |
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||||
GNU General Public License for more details. |
||||
|
||||
You should have received a copy of the GNU General Public License along |
||||
with this program; if not, write to the Free Software Foundation, Inc., |
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. |
||||
|
||||
You can contact the author at : |
||||
- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
|
||||
- LZ4 source repository : http://code.google.com/p/lz4/
|
||||
*/ |
||||
|
||||
//**************************************
|
||||
// Compiler Options
|
||||
//**************************************
|
||||
// Disable some Visual warning messages
|
||||
#define _CRT_SECURE_NO_WARNINGS |
||||
#define _CRT_SECURE_NO_DEPRECATE // VS2005
|
||||
|
||||
// Unix Large Files support (>4GB)
|
||||
#if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions
|
||||
# define _LARGEFILE_SOURCE |
||||
# define FILE_OFFSET_BITS=64 |
||||
#elif ! defined(__LP64__) // No point defining Large file for 64 bit
|
||||
# define _LARGEFILE64_SOURCE |
||||
#endif |
||||
|
||||
// S_ISREG & gettimeofday() are not supported by MSVC
|
||||
#if defined(_MSC_VER) |
||||
# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) |
||||
# define BMK_LEGACY_TIMER 1 |
||||
#endif |
||||
|
||||
// GCC does not support _rotl outside of Windows
|
||||
#if !defined(_WIN32) |
||||
# define _rotl(x,r) ((x << r) | (x >> (32 - r))) |
||||
#endif |
||||
|
||||
|
||||
//**************************************
|
||||
// Includes
|
||||
//**************************************
|
||||
#include <stdlib.h> // malloc |
||||
#include <stdio.h> // fprintf, fopen, ftello64 |
||||
#include <sys/types.h> // stat64 |
||||
#include <sys/stat.h> // stat64 |
||||
|
||||
// Use ftime() if gettimeofday() is not available on your target
|
||||
#if defined(BMK_LEGACY_TIMER) |
||||
# include <sys/timeb.h> // timeb, ftime
|
||||
#else |
||||
# include <sys/time.h> // gettimeofday
|
||||
#endif |
||||
|
||||
#include "lz4.h" |
||||
#define COMPRESSOR0 LZ4_compress |
||||
#include "lz4hc.h" |
||||
#define COMPRESSOR1 LZ4_compressHC |
||||
#define DEFAULTCOMPRESSOR LZ4_compress |
||||
|
||||
|
||||
|
||||
//**************************************
|
||||
// Basic Types
|
||||
//**************************************
|
||||
#if defined(_MSC_VER) // Visual Studio does not support 'stdint' natively
|
||||
#define BYTE unsigned __int8 |
||||
#define U16 unsigned __int16 |
||||
#define U32 unsigned __int32 |
||||
#define S32 __int32 |
||||
#define U64 unsigned __int64 |
||||
#else |
||||
#include <stdint.h> |
||||
#define BYTE uint8_t |
||||
#define U16 uint16_t |
||||
#define U32 uint32_t |
||||
#define S32 int32_t |
||||
#define U64 uint64_t |
||||
#endif |
||||
|
||||
|
||||
//**************************************
|
||||
// Constants
|
||||
//**************************************
|
||||
#define NBLOOPS 3 |
||||
#define TIMELOOP 2000 |
||||
|
||||
#define KNUTH 2654435761U |
||||
#define MAX_MEM (1984<<20) |
||||
#define DEFAULT_CHUNKSIZE (8<<20) |
||||
|
||||
|
||||
//**************************************
|
||||
// Local structures
|
||||
//**************************************
|
||||
struct chunkParameters |
||||
{ |
||||
U32 id; |
||||
char* inputBuffer; |
||||
char* outputBuffer; |
||||
int inputSize; |
||||
int outputSize; |
||||
}; |
||||
|
||||
struct compressionParameters |
||||
{ |
||||
int (*compressionFunction)(const char*, char*, int); |
||||
int (*decompressionFunction)(const char*, char*, int); |
||||
}; |
||||
|
||||
|
||||
//**************************************
|
||||
// MACRO
|
||||
//**************************************
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) |
||||
|
||||
|
||||
|
||||
//**************************************
|
||||
// Benchmark Parameters
|
||||
//**************************************
|
||||
static int chunkSize = DEFAULT_CHUNKSIZE; |
||||
static int nbIterations = NBLOOPS; |
||||
static int BMK_pause = 0; |
||||
|
||||
void BMK_SetBlocksize(int bsize) |
||||
{ |
||||
chunkSize = bsize; |
||||
DISPLAY("-Using Block Size of %i KB-", chunkSize>>10); |
||||
} |
||||
|
||||
void BMK_SetNbIterations(int nbLoops) |
||||
{ |
||||
nbIterations = nbLoops; |
||||
DISPLAY("- %i iterations-", nbIterations); |
||||
} |
||||
|
||||
void BMK_SetPause() |
||||
{ |
||||
BMK_pause = 1; |
||||
} |
||||
|
||||
//*********************************************************
|
||||
// Private functions
|
||||
//*********************************************************
|
||||
|
||||
#if defined(BMK_LEGACY_TIMER) |
||||
|
||||
static int BMK_GetMilliStart() |
||||
{ |
||||
// Based on Legacy ftime()
|
||||
// Rolls over every ~ 12.1 days (0x100000/24/60/60)
|
||||
// Use GetMilliSpan to correct for rollover
|
||||
struct timeb tb; |
||||
int nCount; |
||||
ftime( &tb ); |
||||
nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000); |
||||
return nCount; |
||||
} |
||||
|
||||
#else |
||||
|
||||
static int BMK_GetMilliStart() |
||||
{ |
||||
// Based on newer gettimeofday()
|
||||
// Use GetMilliSpan to correct for rollover
|
||||
struct timeval tv; |
||||
int nCount; |
||||
gettimeofday(&tv, NULL); |
||||
nCount = (int) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000); |
||||
return nCount; |
||||
} |
||||
|
||||
#endif |
||||
|
||||
|
||||
static int BMK_GetMilliSpan( int nTimeStart ) |
||||
{ |
||||
int nSpan = BMK_GetMilliStart() - nTimeStart; |
||||
if ( nSpan < 0 ) |
||||
nSpan += 0x100000 * 1000; |
||||
return nSpan; |
||||
} |
||||
|
||||
|
||||
static U32 BMK_checksum_MMH3A (char* buff, U32 length) |
||||
{ |
||||
const BYTE* data = (const BYTE*)buff; |
||||
const int nblocks = length >> 2; |
||||
|
||||
U32 h1 = KNUTH; |
||||
U32 c1 = 0xcc9e2d51; |
||||
U32 c2 = 0x1b873593; |
||||
|
||||
const U32* blocks = (const U32*)(data + nblocks*4); |
||||
int i; |
||||
|
||||
for(i = -nblocks; i; i++) |
||||
{ |
||||
U32 k1 = blocks[i]; |
||||
|
||||
k1 *= c1; |
||||
k1 = _rotl(k1,15); |
||||
k1 *= c2; |
||||
|
||||
h1 ^= k1; |
||||
h1 = _rotl(h1,13); |
||||
h1 = h1*5+0xe6546b64; |
||||
} |
||||
|
||||
{ |
||||
const BYTE* tail = (const BYTE*)(data + nblocks*4); |
||||
U32 k1 = 0; |
||||
|
||||
switch(length & 3) |
||||
{ |
||||
case 3: k1 ^= tail[2] << 16; |
||||
case 2: k1 ^= tail[1] << 8; |
||||
case 1: k1 ^= tail[0]; |
||||
k1 *= c1; k1 = _rotl(k1,15); k1 *= c2; h1 ^= k1; |
||||
}; |
||||
} |
||||
|
||||
h1 ^= length; |
||||
h1 ^= h1 >> 16; |
||||
h1 *= 0x85ebca6b; |
||||
h1 ^= h1 >> 13; |
||||
h1 *= 0xc2b2ae35; |
||||
h1 ^= h1 >> 16; |
||||
|
||||
return h1; |
||||
} |
||||
|
||||
|
||||
static size_t BMK_findMaxMem(U64 requiredMem) |
||||
{ |
||||
size_t step = (64U<<20); // 64 MB
|
||||
BYTE* testmem=NULL; |
||||
|
||||
requiredMem = (((requiredMem >> 25) + 1) << 26); |
||||
if (requiredMem > MAX_MEM) requiredMem = MAX_MEM; |
||||
|
||||
requiredMem += 2*step; |
||||
while (!testmem) |
||||
{ |
||||
requiredMem -= step; |
||||
testmem = malloc ((size_t)requiredMem); |
||||
} |
||||
|
||||
free (testmem); |
||||
return (size_t) (requiredMem - step); |
||||
} |
||||
|
||||
|
||||
static U64 BMK_GetFileSize(char* infilename) |
||||
{ |
||||
int r; |
||||
#if defined(_MSC_VER) |
||||
struct _stat64 statbuf; |
||||
r = _stat64(infilename, &statbuf); |
||||
#else |
||||
struct stat statbuf; |
||||
r = stat(infilename, &statbuf); |
||||
#endif |
||||
if (r || !S_ISREG(statbuf.st_mode)) return 0; // No good...
|
||||
return (U64)statbuf.st_size; |
||||
} |
||||
|
||||
|
||||
//*********************************************************
|
||||
// Public function
|
||||
//*********************************************************
|
||||
|
||||
int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel) |
||||
{ |
||||
int fileIdx=0; |
||||
FILE* fileIn; |
||||
char* infilename; |
||||
U64 largefilesize; |
||||
size_t benchedsize; |
||||
int nbChunks; |
||||
int maxCChunkSize; |
||||
size_t readSize; |
||||
char* in_buff; |
||||
char* out_buff; int out_buff_size; |
||||
struct chunkParameters* chunkP; |
||||
U32 crcc, crcd=0; |
||||
struct compressionParameters compP; |
||||
|
||||
U64 totals = 0; |
||||
U64 totalz = 0; |
||||
double totalc = 0.; |
||||
double totald = 0.; |
||||
|
||||
|
||||
// Init
|
||||
switch (cLevel) |
||||
{ |
||||
#ifdef COMPRESSOR0 |
||||
case 0 : compP.compressionFunction = COMPRESSOR0; break; |
||||
#endif |
||||
#ifdef COMPRESSOR1 |
||||
case 1 : compP.compressionFunction = COMPRESSOR1; break; |
||||
#endif |
||||
default : compP.compressionFunction = DEFAULTCOMPRESSOR; |
||||
} |
||||
compP.decompressionFunction = LZ4_uncompress; |
||||
|
||||
// Loop for each file
|
||||
while (fileIdx<nbFiles) |
||||
{ |
||||
// Check file existence
|
||||
infilename = fileNamesTable[fileIdx++]; |
||||
fileIn = fopen( infilename, "rb" ); |
||||
if (fileIn==NULL) |
||||
{ |
||||
DISPLAY( "Pb opening %s\n", infilename); |
||||
return 11; |
||||
} |
||||
|
||||
// Memory allocation & restrictions
|
||||
largefilesize = BMK_GetFileSize(infilename); |
||||
benchedsize = (size_t) BMK_findMaxMem(largefilesize) / 2; |
||||
if ((U64)benchedsize > largefilesize) benchedsize = (size_t)largefilesize; |
||||
if (benchedsize < largefilesize) |
||||
{ |
||||
DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", infilename, (int)(benchedsize>>20)); |
||||
} |
||||
|
||||
// Alloc
|
||||
chunkP = (struct chunkParameters*) malloc(((benchedsize / chunkSize)+1) * sizeof(struct chunkParameters)); |
||||
in_buff = malloc((size_t )benchedsize); |
||||
nbChunks = (int) (benchedsize / chunkSize) + 1; |
||||
maxCChunkSize = LZ4_compressBound(chunkSize); |
||||
out_buff_size = nbChunks * maxCChunkSize; |
||||
out_buff = malloc((size_t )out_buff_size); |
||||
|
||||
|
||||
if(!in_buff || !out_buff) |
||||
{ |
||||
DISPLAY("\nError: not enough memory!\n"); |
||||
free(in_buff); |
||||
free(out_buff); |
||||
fclose(fileIn); |
||||
return 12; |
||||
} |
||||
|
||||
// Init chunks data
|
||||
{ |
||||
int i; |
||||
size_t remaining = benchedsize; |
||||
char* in = in_buff; |
||||
char* out = out_buff; |
||||
for (i=0; i<nbChunks; i++) |
||||
{ |
||||
chunkP[i].id = i; |
||||
chunkP[i].inputBuffer = in; in += chunkSize; |
||||
if ((int)remaining > chunkSize) { chunkP[i].inputSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].inputSize = (int)remaining; remaining = 0; } |
||||
chunkP[i].outputBuffer = out; out += maxCChunkSize; |
||||
chunkP[i].outputSize = 0; |
||||
} |
||||
} |
||||
|
||||
// Fill input buffer
|
||||
DISPLAY("Loading %s... \r", infilename); |
||||
readSize = fread(in_buff, 1, benchedsize, fileIn); |
||||
fclose(fileIn); |
||||
|
||||
if(readSize != benchedsize) |
||||
{ |
||||
DISPLAY("\nError: problem reading file '%s' !! \n", infilename); |
||||
free(in_buff); |
||||
free(out_buff); |
||||
return 13; |
||||
} |
||||
|
||||
// Calculating input Checksum
|
||||
crcc = BMK_checksum_MMH3A(in_buff, (unsigned int)benchedsize); |
||||
|
||||
|
||||
// Bench
|
||||
{ |
||||
int loopNb, nb_loops, chunkNb; |
||||
size_t cSize=0; |
||||
int milliTime; |
||||
double fastestC = 100000000., fastestD = 100000000.; |
||||
double ratio=0.; |
||||
|
||||
DISPLAY("\r%79s\r", ""); |
||||
for (loopNb = 1; loopNb <= nbIterations; loopNb++) |
||||
{ |
||||
// Compression
|
||||
DISPLAY("%1i-%-14.14s : %9i ->\r", loopNb, infilename, (int)benchedsize); |
||||
{ size_t i; for (i=0; i<benchedsize; i++) out_buff[i]=(char)i; } // warmimg up memory
|
||||
|
||||
nb_loops = 0; |
||||
milliTime = BMK_GetMilliStart(); |
||||
while(BMK_GetMilliStart() == milliTime); |
||||
milliTime = BMK_GetMilliStart(); |
||||
while(BMK_GetMilliSpan(milliTime) < TIMELOOP) |
||||
{ |
||||
for (chunkNb=0; chunkNb<nbChunks; chunkNb++) |
||||
chunkP[chunkNb].outputSize = compP.compressionFunction(chunkP[chunkNb].inputBuffer, chunkP[chunkNb].outputBuffer, chunkP[chunkNb].inputSize); |
||||
nb_loops++; |
||||
} |
||||
milliTime = BMK_GetMilliSpan(milliTime); |
||||
|
||||
if ((double)milliTime < fastestC*nb_loops) fastestC = (double)milliTime/nb_loops; |
||||
cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].outputSize; |
||||
ratio = (double)cSize/(double)benchedsize*100.; |
||||
|
||||
DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, infilename, (int)benchedsize, (int)cSize, ratio, (double)benchedsize / fastestC / 1000.); |
||||
|
||||
// Decompression
|
||||
{ size_t i; for (i=0; i<benchedsize; i++) in_buff[i]=0; } // zeroing area, for CRC checking
|
||||
|
||||
nb_loops = 0; |
||||
milliTime = BMK_GetMilliStart(); |
||||
while(BMK_GetMilliStart() == milliTime); |
||||
milliTime = BMK_GetMilliStart(); |
||||
while(BMK_GetMilliSpan(milliTime) < TIMELOOP) |
||||
{ |
||||
for (chunkNb=0; chunkNb<nbChunks; chunkNb++) |
||||
chunkP[chunkNb].outputSize = LZ4_uncompress(chunkP[chunkNb].outputBuffer, chunkP[chunkNb].inputBuffer, chunkP[chunkNb].inputSize); |
||||
//chunkP[chunkNb].inputSize = LZ4_uncompress_unknownOutputSize(chunkP[chunkNb].outputBuffer, chunkP[chunkNb].inputBuffer, chunkP[chunkNb].outputSize, chunkSize);
|
||||
nb_loops++; |
||||
} |
||||
milliTime = BMK_GetMilliSpan(milliTime); |
||||
|
||||
if ((double)milliTime < fastestD*nb_loops) fastestD = (double)milliTime/nb_loops; |
||||
DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s\r", loopNb, infilename, (int)benchedsize, (int)cSize, ratio, (double)benchedsize / fastestC / 1000., (double)benchedsize / fastestD / 1000.); |
||||
|
||||
// CRC Checking
|
||||
crcd = BMK_checksum_MMH3A(in_buff, (unsigned int)benchedsize); |
||||
if (crcc!=crcd) { DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", infilename, (unsigned)crcc, (unsigned)crcd); break; } |
||||
} |
||||
|
||||
if (crcc==crcd) |
||||
{ |
||||
if (ratio<100.) |
||||
DISPLAY("%-16.16s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s\n", infilename, (int)benchedsize, (int)cSize, ratio, (double)benchedsize / fastestC / 1000., (double)benchedsize / fastestD / 1000.); |
||||
else |
||||
DISPLAY("%-16.16s : %9i -> %9i (%5.1f%%),%7.1f MB/s ,%7.1f MB/s \n", infilename, (int)benchedsize, (int)cSize, ratio, (double)benchedsize / fastestC / 1000., (double)benchedsize / fastestD / 1000.); |
||||
} |
||||
totals += benchedsize; |
||||
totalz += cSize; |
||||
totalc += fastestC; |
||||
totald += fastestD; |
||||
} |
||||
|
||||
free(in_buff); |
||||
free(out_buff); |
||||
free(chunkP); |
||||
} |
||||
|
||||
if (nbFiles > 1) |
||||
printf("%-16.16s :%10llu ->%10llu (%5.2f%%), %6.1f MB/s , %6.1f MB/s\n", " TOTAL", (long long unsigned int)totals, (long long unsigned int)totalz, (double)totalz/(double)totals*100., (double)totals/totalc/1000., (double)totals/totald/1000.); |
||||
|
||||
if (BMK_pause) { printf("press enter...\n"); getchar(); } |
||||
|
||||
return 0; |
||||
} |
||||
|
||||
|
||||
|
||||
@ -1,41 +0,0 @@
|
||||
/*
|
||||
bench.h - Demo program to benchmark open-source compression algorithm |
||||
Copyright (C) Yann Collet 2012 |
||||
|
||||
This program is free software; you can redistribute it and/or modify |
||||
it under the terms of the GNU General Public License as published by |
||||
the Free Software Foundation; either version 2 of the License, or |
||||
(at your option) any later version. |
||||
|
||||
This program is distributed in the hope that it will be useful, |
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of |
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||||
GNU General Public License for more details. |
||||
|
||||
You should have received a copy of the GNU General Public License along |
||||
with this program; if not, write to the Free Software Foundation, Inc., |
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. |
||||
|
||||
You can contact the author at : |
||||
- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
|
||||
- LZ4 source repository : http://code.google.com/p/lz4/
|
||||
*/ |
||||
#pragma once |
||||
|
||||
#if defined (__cplusplus) |
||||
extern "C" { |
||||
#endif |
||||
|
||||
|
||||
int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel); |
||||
|
||||
// Parameters
|
||||
void BMK_SetBlocksize(int bsize); |
||||
void BMK_SetNbIterations(int nbLoops); |
||||
void BMK_SetPause(); |
||||
|
||||
|
||||
|
||||
#if defined (__cplusplus) |
||||
} |
||||
#endif |
||||
@ -1,227 +0,0 @@
|
||||
/*
|
||||
fuzzer.c - Fuzzer test tool for LZ4 |
||||
Copyright (C) Andrew Mahone - Yann Collet 2012 |
||||
Original code by Andrew Mahone / Modified by Yann Collet |
||||
GPL v2 License |
||||
|
||||
This program is free software; you can redistribute it and/or modify |
||||
it under the terms of the GNU General Public License as published by |
||||
the Free Software Foundation; either version 2 of the License, or |
||||
(at your option) any later version. |
||||
|
||||
This program is distributed in the hope that it will be useful, |
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of |
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||||
GNU General Public License for more details. |
||||
|
||||
You should have received a copy of the GNU General Public License along |
||||
with this program; if not, write to the Free Software Foundation, Inc., |
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. |
||||
|
||||
You can contact the author at : |
||||
- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
|
||||
- LZ4 source repository : http://code.google.com/p/lz4/
|
||||
*/ |
||||
|
||||
//**************************************
|
||||
// Remove Visual warning messages
|
||||
//**************************************
|
||||
#define _CRT_SECURE_NO_WARNINGS // fgets
|
||||
|
||||
|
||||
//**************************************
|
||||
// Includes
|
||||
//**************************************
|
||||
#include <stdlib.h> |
||||
#include <stdio.h> // fgets, sscanf |
||||
#include <sys/timeb.h> // timeb |
||||
#include "lz4.h" |
||||
|
||||
|
||||
//**************************************
|
||||
// Constants
|
||||
//**************************************
|
||||
#define NB_ATTEMPTS (1<<18) |
||||
#define LEN ((1<<15)) |
||||
#define SEQ_POW 2 |
||||
#define NUM_SEQ (1 << SEQ_POW) |
||||
#define SEQ_MSK ((NUM_SEQ) - 1) |
||||
#define MOD_SEQ(x) ((((x) >> 8) & 255) == 0) |
||||
#define NEW_SEQ(x) ((((x) >> 10) %10) == 0) |
||||
#define PAGE_SIZE 4096 |
||||
#define ROUND_PAGE(x) (((x) + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1)) |
||||
#define PRIME1 2654435761U |
||||
#define PRIME2 2246822519U |
||||
#define PRIME3 3266489917U |
||||
|
||||
|
||||
|
||||
//*********************************************************
|
||||
// Functions
|
||||
//*********************************************************
|
||||
static int FUZ_GetMilliStart() |
||||
{ |
||||
struct timeb tb; |
||||
int nCount; |
||||
ftime( &tb ); |
||||
nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000); |
||||
return nCount; |
||||
} |
||||
|
||||
static int FUZ_GetMilliSpan( int nTimeStart ) |
||||
{ |
||||
int nSpan = FUZ_GetMilliStart() - nTimeStart; |
||||
if ( nSpan < 0 ) |
||||
nSpan += 0x100000 * 1000; |
||||
return nSpan; |
||||
} |
||||
|
||||
|
||||
unsigned int FUZ_rand(unsigned int* src) |
||||
{ |
||||
*src = ((*src) * PRIME1) + PRIME2; |
||||
return *src; |
||||
} |
||||
|
||||
|
||||
int test_canary(unsigned char *buf) { |
||||
int i; |
||||
for (i = 0; i < 2048; i++) |
||||
if (buf[i] != buf[i + 2048]) |
||||
return 0; |
||||
return 1; |
||||
} |
||||
|
||||
int FUZ_SecurityTest() |
||||
{ |
||||
char* output; |
||||
char* input; |
||||
int i, r; |
||||
|
||||
printf("Starting security tests..."); |
||||
input = (char*) malloc (20<<20); |
||||
output = (char*) malloc (20<<20); |
||||
input[0] = 0x0F; |
||||
input[1] = 0x00; |
||||
input[2] = 0x00; |
||||
for(i = 3; i < 16840000; i++) |
||||
input[i] = 0xff; |
||||
r = LZ4_uncompress(input, output, 20<<20); |
||||
|
||||
free(input); |
||||
free(output); |
||||
printf(" Completed (r=%i)\n",r); |
||||
return 0; |
||||
} |
||||
|
||||
|
||||
//int main(int argc, char *argv[]) {
|
||||
int main() { |
||||
unsigned long long bytes = 0; |
||||
unsigned long long cbytes = 0; |
||||
unsigned char buf[LEN]; |
||||
unsigned char testOut[LEN+1]; |
||||
# define FUZ_max LZ4_COMPRESSBOUND(LEN) |
||||
# define FUZ_avail ROUND_PAGE(FUZ_max) |
||||
const int off_full = FUZ_avail - FUZ_max; |
||||
unsigned char cbuf[FUZ_avail + PAGE_SIZE]; |
||||
unsigned int seed, cur_seq=PRIME3, seeds[NUM_SEQ], timestamp=FUZ_GetMilliStart(); |
||||
int i, j, k, ret, len; |
||||
char userInput[30] = {0}; |
||||
|
||||
printf("starting LZ4 fuzzer\n"); |
||||
printf("Select an Initialisation number (default : random) : "); |
||||
fflush(stdout); |
||||
if ( fgets(userInput, sizeof userInput, stdin) ) |
||||
{ |
||||
if ( sscanf(userInput, "%d", &seed) == 1 ) {} |
||||
else seed = FUZ_GetMilliSpan(timestamp); |
||||
} |
||||
printf("Seed = %u\n", seed); |
||||
|
||||
FUZ_SecurityTest(); |
||||
|
||||
for (i = 0; i < 2048; i++) |
||||
cbuf[FUZ_avail + i] = cbuf[FUZ_avail + 2048 + i] = FUZ_rand(&seed) >> 16; |
||||
|
||||
for (i = 0; i < NB_ATTEMPTS; i++)
|
||||
{ |
||||
printf("\r%7i /%7i\r", i, NB_ATTEMPTS); |
||||
|
||||
FUZ_rand(&seed); |
||||
for (j = 0; j < NUM_SEQ; j++) { |
||||
seeds[j] = FUZ_rand(&seed) << 8; |
||||
seeds[j] ^= (FUZ_rand(&seed) >> 8) & 65535; |
||||
} |
||||
for (j = 0; j < LEN; j++) { |
||||
k = FUZ_rand(&seed); |
||||
if (j == 0 || NEW_SEQ(k)) |
||||
cur_seq = seeds[(FUZ_rand(&seed) >> 16) & SEQ_MSK]; |
||||
if (MOD_SEQ(k)) { |
||||
k = (FUZ_rand(&seed) >> 16) & SEQ_MSK; |
||||
seeds[k] = FUZ_rand(&seed) << 8; |
||||
seeds[k] ^= (FUZ_rand(&seed) >> 8) & 65535; |
||||
} |
||||
buf[j] = FUZ_rand(&cur_seq) >> 16; |
||||
} |
||||
|
||||
// Test compression
|
||||
ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[off_full], LEN, FUZ_max); |
||||
if (ret == 0) { printf("compression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; } |
||||
len = ret; |
||||
|
||||
// Test decoding with output size being exactly what's necessary => must work
|
||||
ret = LZ4_uncompress((char*)&cbuf[off_full], (char*)testOut, LEN); |
||||
if (ret<0) { printf("decompression failed despite correct space: seed %u, len %d\n", seed, LEN); goto _output_error; } |
||||
|
||||
// Test decoding with one byte missing => must fail
|
||||
ret = LZ4_uncompress((char*)&cbuf[off_full], (char*)testOut, LEN-1); |
||||
if (ret>=0) { printf("decompression should have failed, due to Output Size being too small : seed %u, len %d\n", seed, LEN); goto _output_error; } |
||||
|
||||
// Test decoding with one byte too much => must fail
|
||||
ret = LZ4_uncompress((char*)&cbuf[off_full], (char*)testOut, LEN+1); |
||||
if (ret>=0) { printf("decompression should have failed, due to Output Size being too large : seed %u, len %d\n", seed, LEN); goto _output_error; } |
||||
|
||||
// Test decoding with enough output size => must work
|
||||
ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len, LEN+1); |
||||
if (ret<0) { printf("decompression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; } |
||||
|
||||
// Test decoding with output size being exactly what's necessary => must work
|
||||
ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len, LEN); |
||||
if (ret<0) { printf("decompression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; } |
||||
|
||||
// Test decoding with output size being one byte too short => must fail
|
||||
ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len, LEN-1); |
||||
if (ret>=0) { printf("decompression should have failed, due to Output Size being too small : seed %u, len %d\n", seed, LEN); goto _output_error; } |
||||
|
||||
// Test decoding with input size being one byte too short => must fail
|
||||
ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len-1, LEN); |
||||
if (ret>=0) { printf("decompression should have failed, due to input size being too small : seed %u, len %d\n", seed, LEN); goto _output_error; } |
||||
|
||||
// Test decoding with input size being one byte too large => must fail
|
||||
ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len+1, LEN); |
||||
if (ret>=0) { printf("decompression should have failed, due to input size being too large : seed %u, len %d\n", seed, LEN); goto _output_error; } |
||||
|
||||
// Test compression with output size being exactly what's necessary (should work)
|
||||
ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-len], LEN, len); |
||||
if (!test_canary(&cbuf[FUZ_avail])) { printf("compression overran output buffer: seed %u, len %d, olen %d\n", seed, LEN, len); goto _output_error; } |
||||
if (ret == 0) { printf("compression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; } |
||||
|
||||
// Test compression with just one missing byte into output buffer => must fail
|
||||
ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-(len-1)], LEN, len-1); |
||||
if (ret) { printf("compression overran output buffer: seed %u, len %d, olen %d => ret %d", seed, LEN, len-1, ret); goto _output_error; } |
||||
if (!test_canary(&cbuf[FUZ_avail])) { printf("compression overran output buffer: seed %u, len %d, olen %d", seed, LEN, len-1); goto _output_error; } |
||||
|
||||
bytes += LEN; |
||||
cbytes += len; |
||||
} |
||||
|
||||
printf("all tests completed successfully \n"); |
||||
printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100); |
||||
getchar(); |
||||
return 0; |
||||
|
||||
_output_error: |
||||
getchar(); |
||||
return 1; |
||||
} |
||||
@ -1,121 +0,0 @@
|
||||
LZ4 Format Description |
||||
Last revised: 2012-02-27 |
||||
Author : Y. Collet |
||||
|
||||
|
||||
|
||||
This small specification intents to provide enough information |
||||
to anyone willing to produce LZ4-compatible compressed streams |
||||
using any programming language. |
||||
|
||||
LZ4 is an LZ77-type compressor with a fixed, byte-oriented encoding. |
||||
The most important design principle behind LZ4 is simplicity. |
||||
It helps to create an easy to read and maintain source code. |
||||
It also helps later on for optimisations, compactness, and speed. |
||||
There is no entropy encoder backend nor framing layer. |
||||
The latter is assumed to be handled by other parts of the system. |
||||
|
||||
This document only describes the format, |
||||
not how the LZ4 compressor nor decompressor actually work. |
||||
The correctness of the decompressor should not depend |
||||
on implementation details of the compressor, and vice versa. |
||||
|
||||
|
||||
|
||||
-- Compressed stream format -- |
||||
|
||||
An LZ4 compressed stream is composed of sequences. |
||||
Schematically, a sequence is a suite of literals, followed by a match copy. |
||||
|
||||
Each sequence starts with a token. |
||||
The token is a one byte value, separated into two 4-bits fields. |
||||
Therefore each field ranges from 0 to 15. |
||||
|
||||
|
||||
The first field uses the 4 high-bits of the token. |
||||
It provides the length of literals to follow. |
||||
(Note : a literal is a not-compressed byte). |
||||
If the field value is 0, then there is no literal. |
||||
If it is 15, then we need to add some more bytes to indicate the full length. |
||||
Each additionnal byte then represent a value from 0 to 255, |
||||
which is added to the previous value to produce a total length. |
||||
When the byte value is 255, another byte is output. |
||||
There can be any number of bytes following the token. There is no "size limit". |
||||
(Sidenote this is why a not-compressible input stream is expanded by 0.4%). |
||||
|
||||
Example 1 : A length of 48 will be represented as : |
||||
- 15 : value for the 4-bits High field |
||||
- 33 : (=48-15) remaining length to reach 48 |
||||
|
||||
Example 2 : A length of 280 will be represented as : |
||||
- 15 : value for the 4-bits High field |
||||
- 255 : following byte is maxed, since 280-15 >= 255 |
||||
- 10 : (=280 - 15 - 255) ) remaining length to reach 280 |
||||
|
||||
Example 3 : A length of 15 will be represented as : |
||||
- 15 : value for the 4-bits High field |
||||
- 0 : (=15-15) yes, the zero must be output |
||||
|
||||
Following the token and optional length bytes, are the literals themselves. |
||||
They are exactly as numerous as previously decoded (length of literals). |
||||
It's possible that there are zero literal. |
||||
|
||||
|
||||
Following the literals is the match copy operation. |
||||
|
||||
It starts by the offset. |
||||
This is a 2 bytes value, in little endian format : |
||||
the lower byte is the first one in the stream. |
||||
|
||||
The offset represents the position of the match to be copied from. |
||||
1 means "current position - 1 byte". |
||||
The maximum offset value is 65535, 65536 cannot be coded. |
||||
Note that 0 is an invalid value, not used. |
||||
|
||||
Then we need to extract the match length. |
||||
For this, we use the second token field, the low 4-bits. |
||||
Value, obviously, ranges from 0 to 15. |
||||
However here, 0 means that the copy operation will be minimal. |
||||
The minimum length of a match, called minmatch, is 4. |
||||
As a consequence, a 0 value means 4 bytes, and a value of 15 means 19+ bytes. |
||||
Similar to literal length, on reaching the highest possible value (15), |
||||
we output additional bytes, one at a time, with values ranging from 0 to 255. |
||||
They are added to total to provide the final match length. |
||||
A 255 value means there is another byte to read and add. |
||||
There is no limit to the number of optional bytes that can be output this way. |
||||
(This points towards a maximum achievable compression ratio of ~250). |
||||
|
||||
With the offset and the matchlength, |
||||
the decoder can now proceed to copy the data from the already decoded buffer. |
||||
On decoding the matchlength, we reach the end of the compressed sequence, |
||||
and therefore start another one. |
||||
|
||||
|
||||
-- Parsing restrictions -- |
||||
|
||||
There are specific parsing rules to respect in order to remain compatible |
||||
with assumptions made by the decoder : |
||||
1) The last 5 bytes are always literals |
||||
2) The last match must start at least 12 bytes before end of stream |
||||
Consequently, a file with less than 13 bytes cannot be compressed. |
||||
These rules are in place to ensure that the decoder |
||||
will never read beyond the input buffer, nor write beyond the output buffer. |
||||
|
||||
Note that the last sequence is also incomplete, |
||||
and stops right after literals. |
||||
|
||||
|
||||
-- Additional notes -- |
||||
|
||||
There is no assumption nor limits to the way the compressor |
||||
searches and selects matches within the source stream. |
||||
It could be a fast scan, a multi-probe, a full search using BST, |
||||
standard hash chains or MMC, well whatever. |
||||
|
||||
Advanced parsing strategies can also be implemented, such as lazy match, |
||||
or full optimal parsing. |
||||
|
||||
All these trade-off offer distinctive speed/memory/compression advantages. |
||||
Whatever the method used by the compressor, its result will be decodable |
||||
by any LZ4 decoder if it follows the format specification described above. |
||||
|
||||
@ -1,402 +0,0 @@
|
||||
/*
|
||||
LZ4Demo - Demo CLI program using LZ4 compression |
||||
Copyright (C) Yann Collet 2011-2012 |
||||
GPL v2 License |
||||
|
||||
This program is free software; you can redistribute it and/or modify |
||||
it under the terms of the GNU General Public License as published by |
||||
the Free Software Foundation; either version 2 of the License, or |
||||
(at your option) any later version. |
||||
|
||||
This program is distributed in the hope that it will be useful, |
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of |
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||||
GNU General Public License for more details. |
||||
|
||||
You should have received a copy of the GNU General Public License along |
||||
with this program; if not, write to the Free Software Foundation, Inc., |
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. |
||||
|
||||
You can contact the author at : |
||||
- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
|
||||
- LZ4 source repository : http://code.google.com/p/lz4/
|
||||
*/ |
||||
/*
|
||||
Note : this is *only* a demo program, an example to show how LZ4 can be used. |
||||
It is not considered part of LZ4 compression library. |
||||
The license of LZ4 is BSD. |
||||
The license of the demo program is GPL. |
||||
*/ |
||||
|
||||
//**************************************
|
||||
// Compiler Options
|
||||
//**************************************
|
||||
// Disable some Visual warning messages
|
||||
#define _CRT_SECURE_NO_WARNINGS |
||||
#define _CRT_SECURE_NO_DEPRECATE // VS2005
|
||||
|
||||
|
||||
//****************************
|
||||
// Includes
|
||||
//****************************
|
||||
#include <stdio.h> // fprintf, fopen, fread, _fileno(?) |
||||
#include <stdlib.h> // malloc |
||||
#include <string.h> // strcmp |
||||
#include <time.h> // clock |
||||
#ifdef _WIN32 |
||||
#include <io.h> // _setmode |
||||
#include <fcntl.h> // _O_BINARY |
||||
#endif |
||||
#include "lz4.h" |
||||
#include "lz4hc.h" |
||||
#include "bench.h" |
||||
|
||||
|
||||
//**************************************
|
||||
// Compiler-specific functions
|
||||
//**************************************
|
||||
#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) |
||||
|
||||
#if defined(_MSC_VER) // Visual Studio
|
||||
#define swap32 _byteswap_ulong |
||||
#elif GCC_VERSION >= 403 |
||||
#define swap32 __builtin_bswap32 |
||||
#else |
||||
static inline unsigned int swap32(unsigned int x) { |
||||
return ((x << 24) & 0xff000000 ) | |
||||
((x << 8) & 0x00ff0000 ) | |
||||
((x >> 8) & 0x0000ff00 ) | |
||||
((x >> 24) & 0x000000ff ); |
||||
} |
||||
#endif |
||||
|
||||
|
||||
//****************************
|
||||
// Constants
|
||||
//****************************
|
||||
#define COMPRESSOR_NAME "Compression CLI using LZ4 algorithm" |
||||
#define COMPRESSOR_VERSION "" |
||||
#define COMPILED __DATE__ |
||||
#define AUTHOR "Yann Collet" |
||||
#define EXTENSION ".lz4" |
||||
#define WELCOME_MESSAGE "*** %s %s, by %s (%s) ***\n", COMPRESSOR_NAME, COMPRESSOR_VERSION, AUTHOR, COMPILED |
||||
|
||||
#define CHUNKSIZE (8<<20) // 8 MB
|
||||
#define CACHELINE 64 |
||||
#define ARCHIVE_MAGICNUMBER 0x184C2102 |
||||
#define ARCHIVE_MAGICNUMBER_SIZE 4 |
||||
|
||||
|
||||
//**************************************
|
||||
// Architecture Macros
|
||||
//**************************************
|
||||
static const int one = 1; |
||||
#define CPU_LITTLE_ENDIAN (*(char*)(&one)) |
||||
#define CPU_BIG_ENDIAN (!CPU_LITTLE_ENDIAN) |
||||
#define LITTLE_ENDIAN32(i) if (CPU_BIG_ENDIAN) { i = swap32(i); } |
||||
|
||||
|
||||
//**************************************
|
||||
// Macros
|
||||
//**************************************
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) |
||||
|
||||
|
||||
//****************************
|
||||
// Functions
|
||||
//****************************
|
||||
int usage(char* exename) |
||||
{ |
||||
DISPLAY( "Usage :\n"); |
||||
DISPLAY( " %s [arg] input output\n", exename); |
||||
DISPLAY( "Arguments :\n"); |
||||
DISPLAY( " -c0: Fast compression (default) \n"); |
||||
DISPLAY( " -c1: High compression \n"); |
||||
DISPLAY( " -d : decompression \n"); |
||||
DISPLAY( " -b#: Benchmark files, using # compression level\n"); |
||||
DISPLAY( " -t : check compressed file \n"); |
||||
DISPLAY( " -h : help (this text)\n"); |
||||
DISPLAY( "input : can be 'stdin' (pipe) or a filename\n"); |
||||
DISPLAY( "output : can be 'stdout'(pipe) or a filename or 'null'\n"); |
||||
return 0; |
||||
} |
||||
|
||||
|
||||
int badusage(char* exename) |
||||
{ |
||||
DISPLAY("Wrong parameters\n"); |
||||
usage(exename); |
||||
return 0; |
||||
} |
||||
|
||||
|
||||
|
||||
int get_fileHandle(char* input_filename, char* output_filename, FILE** pfinput, FILE** pfoutput) |
||||
{ |
||||
char stdinmark[] = "stdin"; |
||||
char stdoutmark[] = "stdout"; |
||||
|
||||
if (!strcmp (input_filename, stdinmark)) { |
||||
DISPLAY( "Using stdin for input\n"); |
||||
*pfinput = stdin; |
||||
#ifdef _WIN32 // Need to set stdin/stdout to binary mode specifically for windows
|
||||
_setmode( _fileno( stdin ), _O_BINARY ); |
||||
#endif |
||||
} else { |
||||
*pfinput = fopen( input_filename, "rb" ); |
||||
} |
||||
|
||||
if (!strcmp (output_filename, stdoutmark)) { |
||||
DISPLAY( "Using stdout for output\n"); |
||||
*pfoutput = stdout; |
||||
#ifdef _WIN32 // Need to set stdin/stdout to binary mode specifically for windows
|
||||
_setmode( _fileno( stdout ), _O_BINARY ); |
||||
#endif |
||||
} else { |
||||
*pfoutput = fopen( output_filename, "wb" ); |
||||
} |
||||
|
||||
if ( *pfinput==0 ) { DISPLAY( "Pb opening %s\n", input_filename); return 2; } |
||||
if ( *pfoutput==0) { DISPLAY( "Pb opening %s\n", output_filename); return 3; } |
||||
|
||||
return 0; |
||||
} |
||||
|
||||
|
||||
|
||||
int compress_file(char* input_filename, char* output_filename, int compressionlevel) |
||||
{ |
||||
int (*compressionFunction)(const char*, char*, int); |
||||
unsigned long long filesize = 0; |
||||
unsigned long long compressedfilesize = ARCHIVE_MAGICNUMBER_SIZE; |
||||
unsigned int u32var; |
||||
char* in_buff; |
||||
char* out_buff; |
||||
FILE* finput; |
||||
FILE* foutput; |
||||
int r; |
||||
int displayLevel = (compressionlevel>0); |
||||
clock_t start, end; |
||||
size_t sizeCheck; |
||||
|
||||
|
||||
// Init
|
||||
switch (compressionlevel) |
||||
{ |
||||
case 0 : compressionFunction = LZ4_compress; break; |
||||
case 1 : compressionFunction = LZ4_compressHC; break; |
||||
default : compressionFunction = LZ4_compress; |
||||
} |
||||
start = clock(); |
||||
r = get_fileHandle(input_filename, output_filename, &finput, &foutput); |
||||
if (r) return r; |
||||
|
||||
// Allocate Memory
|
||||
in_buff = (char*)malloc(CHUNKSIZE); |
||||
out_buff = (char*)malloc(LZ4_compressBound(CHUNKSIZE)); |
||||
if (!in_buff || !out_buff) { DISPLAY("Allocation error : not enough memory\n"); return 8; } |
||||
|
||||
// Write Archive Header
|
||||
u32var = ARCHIVE_MAGICNUMBER; |
||||
LITTLE_ENDIAN32(u32var); |
||||
*(unsigned int*)out_buff = u32var; |
||||
sizeCheck = fwrite(out_buff, 1, ARCHIVE_MAGICNUMBER_SIZE, foutput); |
||||
if (sizeCheck!=ARCHIVE_MAGICNUMBER_SIZE) { DISPLAY("write error\n"); return 10; } |
||||
|
||||
// Main Loop
|
||||
while (1) |
||||
{ |
||||
int outSize; |
||||
// Read Block
|
||||
int inSize = (int) fread(in_buff, (size_t)1, (size_t)CHUNKSIZE, finput); |
||||
if( inSize<=0 ) break; |
||||
filesize += inSize; |
||||
if (displayLevel) DISPLAY("Read : %i MB \r", (int)(filesize>>20)); |
||||
|
||||
// Compress Block
|
||||
outSize = compressionFunction(in_buff, out_buff+4, inSize); |
||||
compressedfilesize += outSize+4; |
||||
if (displayLevel) DISPLAY("Read : %i MB ==> %.2f%%\r", (int)(filesize>>20), (double)compressedfilesize/filesize*100); |
||||
|
||||
// Write Block
|
||||
LITTLE_ENDIAN32(outSize); |
||||
* (unsigned int*) out_buff = outSize; |
||||
LITTLE_ENDIAN32(outSize); |
||||
sizeCheck = fwrite(out_buff, 1, outSize+4, foutput); |
||||
if (sizeCheck!=(size_t)(outSize+4)) { DISPLAY("write error\n"); return 11; } |
||||
} |
||||
|
||||
// Status
|
||||
end = clock(); |
||||
DISPLAY( "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", |
||||
(unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); |
||||
{ |
||||
double seconds = (double)(end - start)/CLOCKS_PER_SEC; |
||||
DISPLAY( "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); |
||||
} |
||||
|
||||
// Close & Free
|
||||
free(in_buff); |
||||
free(out_buff); |
||||
fclose(finput); |
||||
fclose(foutput); |
||||
|
||||
return 0; |
||||
} |
||||
|
||||
|
||||
int decode_file(char* input_filename, char* output_filename) |
||||
{ |
||||
unsigned long long filesize = 0; |
||||
char* in_buff; |
||||
char* out_buff; |
||||
size_t uselessRet; |
||||
int sinkint; |
||||
unsigned int chunkSize; |
||||
FILE* finput; |
||||
FILE* foutput; |
||||
clock_t start, end; |
||||
int r; |
||||
size_t sizeCheck; |
||||
|
||||
|
||||
// Init
|
||||
start = clock(); |
||||
r = get_fileHandle(input_filename, output_filename, &finput, &foutput); |
||||
if (r) return r; |
||||
|
||||
// Allocate Memory
|
||||
in_buff = (char*)malloc(LZ4_compressBound(CHUNKSIZE)); |
||||
out_buff = (char*)malloc(CHUNKSIZE); |
||||
if (!in_buff || !out_buff) { DISPLAY("Allocation error : not enough memory\n"); return 7; } |
||||
|
||||
// Check Archive Header
|
||||
chunkSize = 0; |
||||
uselessRet = fread(&chunkSize, 1, ARCHIVE_MAGICNUMBER_SIZE, finput); |
||||
LITTLE_ENDIAN32(chunkSize); |
||||
if (chunkSize != ARCHIVE_MAGICNUMBER) { DISPLAY("Unrecognized header : file cannot be decoded\n"); return 6; } |
||||
|
||||
// Main Loop
|
||||
while (1) |
||||
{ |
||||
// Block Size
|
||||
uselessRet = fread(&chunkSize, 1, 4, finput); |
||||
if( uselessRet==0 ) break; // Nothing to read : file read is completed
|
||||
LITTLE_ENDIAN32(chunkSize); |
||||
if (chunkSize == ARCHIVE_MAGICNUMBER)
|
||||
continue; // appended compressed stream
|
||||
|
||||
// Read Block
|
||||
uselessRet = fread(in_buff, 1, chunkSize, finput); |
||||
|
||||
// Decode Block
|
||||
sinkint = LZ4_uncompress_unknownOutputSize(in_buff, out_buff, chunkSize, CHUNKSIZE); |
||||
if (sinkint < 0) { DISPLAY("Decoding Failed ! Corrupted input !\n"); return 9; } |
||||
filesize += sinkint; |
||||
|
||||
// Write Block
|
||||
sizeCheck = fwrite(out_buff, 1, sinkint, foutput); |
||||
if (sizeCheck != (size_t)sinkint) { DISPLAY("write error\n"); return 12; } |
||||
} |
||||
|
||||
// Status
|
||||
end = clock(); |
||||
DISPLAY( "Successfully decoded %llu bytes \n", (unsigned long long)filesize); |
||||
{ |
||||
double seconds = (double)(end - start)/CLOCKS_PER_SEC; |
||||
DISPLAY( "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); |
||||
} |
||||
|
||||
// Close & Free
|
||||
free(in_buff); |
||||
free(out_buff); |
||||
fclose(finput); |
||||
fclose(foutput); |
||||
|
||||
return 0; |
||||
} |
||||
|
||||
|
||||
int main(int argc, char** argv) |
||||
{ |
||||
int i, |
||||
cLevel=0, |
||||
decode=0, |
||||
bench=0, |
||||
filenamesStart=2; |
||||
char* exename=argv[0]; |
||||
char* input_filename=0; |
||||
char* output_filename=0; |
||||
#ifdef _WIN32 |
||||
char nulmark[] = "nul"; |
||||
#else |
||||
char nulmark[] = "/dev/null"; |
||||
#endif |
||||
char nullinput[] = "null"; |
||||
|
||||
// Welcome message
|
||||
DISPLAY( WELCOME_MESSAGE); |
||||
|
||||
if (argc<2) { badusage(exename); return 1; } |
||||
|
||||
for(i=1; i<argc; i++) |
||||
{ |
||||
char* argument = argv[i]; |
||||
|
||||
if(!argument) continue; // Protection if argument empty
|
||||
|
||||
// Select command
|
||||
if (argument[0]=='-') |
||||
{ |
||||
argument ++; |
||||
|
||||
// Display help on usage
|
||||
if ( argument[0] =='h' ) { usage(exename); return 0; } |
||||
|
||||
// Compression (default)
|
||||
if ( argument[0] =='c' ) { if (argument[1] >='0') cLevel=argument[1] - '0'; continue; } |
||||
|
||||
// Decoding
|
||||
if ( argument[0] =='d' ) { decode=1; continue; } |
||||
|
||||
// Bench
|
||||
if ( argument[0] =='b' ) { bench=1; if (argument[1] >= '0') cLevel=argument[1] - '0'; continue; }
|
||||
|
||||
// Modify Block Size (benchmark only)
|
||||
if ( argument[0] =='B' ) { int B = argument[1] - '0'; int S = 1 << (10 + 2*B); BMK_SetBlocksize(S); continue; } |
||||
|
||||
// Modify Nb Iterations (benchmark only)
|
||||
if ( argument[0] =='i' ) { int iters = argument[1] - '0'; BMK_SetNbIterations(iters); continue; } |
||||
|
||||
// Pause at the end (benchmark only)
|
||||
if ( argument[0] =='p' ) { BMK_SetPause(); continue; } |
||||
|
||||
// Test
|
||||
if ( argument[0] =='t' ) { decode=1; output_filename=nulmark; continue; } |
||||
} |
||||
|
||||
// first provided filename is input
|
||||
if (!input_filename) { input_filename=argument; filenamesStart=i; continue; } |
||||
|
||||
// second provided filename is output
|
||||
if (!output_filename) |
||||
{ |
||||
output_filename=argument; |
||||
if (!strcmp (output_filename, nullinput)) output_filename = nulmark; |
||||
continue; |
||||
} |
||||
} |
||||
|
||||
// No input filename ==> Error
|
||||
if(!input_filename) { badusage(exename); return 1; } |
||||
|
||||
if (bench) return BMK_benchFile(argv+filenamesStart, argc-filenamesStart, cLevel); |
||||
|
||||
// No output filename ==> Error
|
||||
if (!output_filename) { badusage(exename); return 1; } |
||||
|
||||
if (decode) return decode_file(input_filename, output_filename); |
||||
|
||||
return compress_file(input_filename, output_filename, cLevel); // Compression is 'default' action
|
||||
|
||||
} |
||||
Loading…
Reference in new issue