Add ppmd based on LZMA SDK 22.01 (thanks to @pps83)

Makefile

@@ -187,6 +187,12 @@ else
  BZIP2_FILES += bzip2/blocksort.o bzip2/huffman.o bzip2/crctable.o bzip2/randtable.o bzip2/compress.o bzip2/decompress.o bzip2/bzlib.o
 endif
 
+ifeq "$(DONT_BUILD_PPMD)" "1"
+ DEFINES += -DDONT_BUILD_PPMD
+else
+ PPMD_FILES += libppmd/lib/libppmd.o
+endif
+
 ifeq "$(DONT_BUILD_SNAPPY)" "1"
  DEFINES += -DBENCH_REMOVE_SNAPPY
 else
@@ -353,7 +359,7 @@ $(NVCOMP_CPP_OBJ): %.cpp.o: %.cpp
 
 _lzbench/lzbench.o: _lzbench/lzbench.cpp _lzbench/lzbench.h
 
-lzbench: $(BZIP2_FILES) $(DENSITY_FILES) $(FASTLZMA2_OBJ) $(ZSTD_FILES) $(GLZA_FILES) $(LZSSE_FILES) $(LZFSE_FILES) $(XPACK_FILES) $(GIPFELI_FILES) $(XZ_FILES) $(LIBLZG_FILES) $(BRIEFLZ_FILES) $(LZF_FILES) $(LZRW_FILES) $(BROTLI_FILES) $(CSC_FILES) $(LZMA_FILES) $(ZLING_FILES) $(QUICKLZ_FILES) $(SNAPPY_FILES) $(ZLIB_FILES) $(LZHAM_FILES) $(LZO_FILES) $(UCL_FILES) $(LZMAT_FILES) $(LZ4_FILES) $(LIBDEFLATE_FILES) $(MISC_FILES) $(NVCOMP_FILES) $(LZBENCH_FILES)
+lzbench: $(BZIP2_FILES) $(DENSITY_FILES) $(FASTLZMA2_OBJ) $(ZSTD_FILES) $(GLZA_FILES) $(LZSSE_FILES) $(LZFSE_FILES) $(XPACK_FILES) $(GIPFELI_FILES) $(XZ_FILES) $(LIBLZG_FILES) $(BRIEFLZ_FILES) $(LZF_FILES) $(LZRW_FILES) $(BROTLI_FILES) $(CSC_FILES) $(LZMA_FILES) $(ZLING_FILES) $(QUICKLZ_FILES) $(SNAPPY_FILES) $(ZLIB_FILES) $(LZHAM_FILES) $(LZO_FILES) $(UCL_FILES) $(LZMAT_FILES) $(LZ4_FILES) $(LIBDEFLATE_FILES) $(MISC_FILES) $(NVCOMP_FILES) $(LZBENCH_FILES) $(PPMD_FILES)
  $(CXX) $^ -o $@ $(LDFLAGS)
  @echo Linked GCC_VERSION=$(GCC_VERSION) CLANG_VERSION=$(CLANG_VERSION) COMPILER=$(COMPILER)
 
@@ -370,4 +376,4 @@ lzbench: $(BZIP2_FILES) $(DENSITY_FILES) $(FASTLZMA2_OBJ) $(ZSTD_FILES) $(GLZA_F
  $(CXX) $(CFLAGS) $< -c -o $@
 
 clean:
- rm -rf lzbench lzbench.exe *.o _lzbench/*.o bzip2/*.o fast-lzma2/*.o slz/*.o zstd/lib/*.o zstd/lib/*.a zstd/lib/common/*.o zstd/lib/compress/*.o zstd/lib/decompress/*.o zstd/lib/dictBuilder/*.o lzsse/lzsse2/*.o lzsse/lzsse4/*.o lzsse/lzsse8/*.o lzfse/*.o xpack/lib/*.o blosclz/*.o gipfeli/*.o xz/*.o xz/common/*.o xz/check/*.o xz/lzma/*.o xz/lz/*.o xz/rangecoder/*.o liblzg/*.o lzlib/*.o brieflz/*.o brotli/common/*.o brotli/enc/*.o brotli/dec/*.o libcsc/*.o wflz/*.o lzjb/*.o lzma/*.o density/buffers/*.o density/algorithms/*.o density/algorithms/cheetah/core/*.o density/algorithms/*.o density/algorithms/lion/forms/*.o density/algorithms/lion/core/*.o density/algorithms/chameleon/core/*.o density/*.o density/structure/*.o pithy/*.o glza/*.o libzling/*.o yappy/*.o shrinker/*.o fastlz/*.o ucl/*.o zlib/*.o lzham/*.o lzmat/*.o lz4/*.o crush/*.o lzf/*.o lzrw/*.o lzo/*.o snappy/*.o quicklz/*.o tornado/*.o libdeflate/lib/*.o libdeflate/lib/x86/*.o libdeflate/lib/arm/*.o nakamichi/*.o nvcomp/*.o
+ rm -rf lzbench lzbench.exe *.o _lzbench/*.o bzip2/*.o fast-lzma2/*.o slz/*.o zstd/lib/*.o zstd/lib/*.a zstd/lib/common/*.o zstd/lib/compress/*.o zstd/lib/decompress/*.o zstd/lib/dictBuilder/*.o lzsse/lzsse2/*.o lzsse/lzsse4/*.o lzsse/lzsse8/*.o lzfse/*.o xpack/lib/*.o blosclz/*.o gipfeli/*.o xz/*.o xz/common/*.o xz/check/*.o xz/lzma/*.o xz/lz/*.o xz/rangecoder/*.o liblzg/*.o lzlib/*.o brieflz/*.o brotli/common/*.o brotli/enc/*.o brotli/dec/*.o libcsc/*.o wflz/*.o lzjb/*.o lzma/*.o density/buffers/*.o density/algorithms/*.o density/algorithms/cheetah/core/*.o density/algorithms/*.o density/algorithms/lion/forms/*.o density/algorithms/lion/core/*.o density/algorithms/chameleon/core/*.o density/*.o density/structure/*.o pithy/*.o glza/*.o libzling/*.o yappy/*.o shrinker/*.o fastlz/*.o ucl/*.o zlib/*.o lzham/*.o lzmat/*.o lz4/*.o crush/*.o lzf/*.o lzrw/*.o lzo/*.o snappy/*.o quicklz/*.o tornado/*.o libdeflate/lib/*.o libdeflate/lib/x86/*.o libdeflate/lib/arm/*.o nakamichi/*.o nvcomp/*.o libppmd/lib/*.o

NEWS

@@ -1,3 +1,6 @@
+v1.8.2
+- added ppmd based on LZMA SDK 22.01 (thanks to @pps83)
+
 v1.8.1
 - fixed linking issues with glza (multiple definition of ###)
 - update BriefLZ to 1.3.0 (thanks to @jibsen)

README.md

@@ -106,6 +106,7 @@ see the [CompFuzz Results](https://github.com/nemequ/compfuzz/wiki/Results) page
  - [lzrw 15-Jul-1991](https://en.wikipedia.org/wiki/LZRW)
  - [lzsse 2019-04-18 (1847c3e827)](https://github.com/ConorStokes/LZSSE)
  - [pithy 2011-12-24](https://github.com/johnezang/pithy) - WARNING: it contains bugs (decompression error; returns 0)
+ - [ppmd 22.01](https://github.com/pps83/libppmd)
  - [quicklz 1.5.0](http://www.quicklz.com)
  - [shrinker 0.1](https://code.google.com/p/data-shrinker) - WARNING: it can throw SEGFAULT compiled with gcc 4.9+ -O3
  - [slz 1.2.0](http://www.libslz.org/) - only a compressor, uses zlib for decompression

_lzbench/compressors.cpp

@@ -1161,6 +1161,128 @@ int64_t lzbench_pithy_decompress(char *inbuf, size_t insize, char *outbuf, size_
 
 #endif
 
+#ifndef BENCH_REMOVE_PPMD
+#include "libppmd/lib/Ppmd8.h"
+
+int64_t lzbench_ppmd_compress(char* inbuf, size_t insize, char* outbuf, size_t outsize, size_t level, size_t, char*)
+{
+ struct CharWriter
+ {
+ IByteOut streamOut;
+ char* ptr;
+
+ static void* pmalloc(ISzAllocPtr ip, size_t size)
+ {
+ (void)ip;
+ return malloc(size);
+ }
+
+ static void pfree(ISzAllocPtr ip, void* addr)
+ {
+ (void)ip;
+ free(addr);
+ }
+
+ static void write(const IByteOut* p, Byte b)
+ {
+ CharWriter* cw = (CharWriter*)p;
+ *cw->ptr++ = (char)b;
+ }
+ };
+
+ const int modelOrder = level < 1 ? 1 : (level > 16 ? 16 : (int)level);
+ const int memMb = 16;
+ const int restoreMethod = 0;
+ unsigned short wPPMd = (modelOrder - 1) +
+ ((memMb - 1) << 4) +
+ (restoreMethod << 12);
+ CharWriter cw;
+ cw.streamOut.Write = &CharWriter::write;
+ cw.ptr = outbuf;
+ CPpmd8 ppmd;
+ ppmd.Stream.Out = &cw.streamOut;
+ ISzAlloc ialloc = { CharWriter::pmalloc, CharWriter::pfree };
+
+ Ppmd8_Construct(&ppmd);
+ Ppmd8_Alloc(&ppmd, memMb << 20, &ialloc);
+ Ppmd8_Init_RangeEnc(&ppmd);
+ Ppmd8_Init(&ppmd, modelOrder, restoreMethod);
+
+ ppmd.Stream.Out->Write(&cw.streamOut, wPPMd & 0xff);
+ ppmd.Stream.Out->Write(&cw.streamOut, wPPMd >> 8);
+
+ for (size_t i = 0; i < insize; ++i)
+ Ppmd8_EncodeSymbol(&ppmd, (unsigned char)inbuf[i]);
+ Ppmd8_EncodeSymbol(&ppmd, -1); /* EndMark */
+ Ppmd8_Flush_RangeEnc(&ppmd);
+ Ppmd8_Free(&ppmd, &ialloc);
+ return cw.ptr - outbuf;
+}
+
+int64_t lzbench_ppmd_decompress(char* inbuf, size_t insize, char* outbuf, size_t outsize, size_t, size_t, char*)
+{
+ struct CharReader
+ {
+ IByteIn streamIn;
+ const char* ptr;
+ const char* end;
+
+ static void* pmalloc(ISzAllocPtr ip, size_t size)
+ {
+ (void)ip;
+ return malloc(size);
+ }
+
+ static void pfree(ISzAllocPtr ip, void* addr)
+ {
+ (void)ip;
+ free(addr);
+ }
+
+ static Byte read(const IByteIn* p)
+ {
+ CharReader* cr = (CharReader*)p;
+ if (cr->ptr >= cr->end)
+ return 0;
+ return *cr->ptr++;
+ }
+ };
+
+ CharReader cr;
+ cr.streamIn.Read = &CharReader::read;
+ cr.ptr = inbuf;
+ cr.end = inbuf + insize;
+
+ unsigned short wPPMd = CharReader::read(&cr.streamIn) | ((unsigned short)(CharReader::read(&cr.streamIn)) << 8);
+
+ const int modelOrder = (wPPMd & 0xf) + 1;
+ const int memMb = ((wPPMd >> 4) & 0xff) + 1;
+ const int restoreMethod = wPPMd >> 12;
+
+ CPpmd8 ppmd;
+ ppmd.Stream.In = &cr.streamIn;
+ ISzAlloc ialloc = { CharReader::pmalloc, CharReader::pfree };
+
+ Ppmd8_Construct(&ppmd);
+ Ppmd8_Alloc(&ppmd, memMb << 20, &ialloc);
+ Ppmd8_Init_RangeDec(&ppmd);
+ Ppmd8_Init(&ppmd, modelOrder, restoreMethod);
+
+ size_t sz = 0;
+ for (;;)
+ {
+ int c = Ppmd8_DecodeSymbol(&ppmd);
+ if (cr.ptr > cr.end || c < 0)
+ break;
+ outbuf[sz++] = (char)(unsigned)c;
+ }
+ int ret = Ppmd8_RangeDec_IsFinishedOK(&ppmd) && cr.ptr >= cr.end ? 0 : -1;
+ Ppmd8_Free(&ppmd, &ialloc);
+ return ret == 0 ? (int64_t)sz : (int64_t)0;
+}
+
+#endif
+
 
 #ifndef BENCH_REMOVE_QUICKLZ
 #include "quicklz/quicklz151b7.h"

_lzbench/compressors.h

@@ -336,6 +336,15 @@ int64_t lzbench_return_0(char *inbuf, size_t insize, char *outbuf, size_t outsiz
 #endif
 
 
+#ifndef BENCH_REMOVE_PPMD
+int64_t lzbench_ppmd_compress(char* inbuf, size_t insize, char* outbuf, size_t outsize, size_t level, size_t, char*);
+int64_t lzbench_ppmd_decompress(char* inbuf, size_t insize, char* outbuf, size_t outsize, size_t, size_t, char*);
+#else
+#define lzbench_ppmd_compress NULL
+#define lzbench_ppmd_decompress NULL
+#endif // BENCH_REMOVE_PPMD
+
+
 #ifndef BENCH_REMOVE_QUICKLZ
  int64_t lzbench_quicklz_compress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t level, size_t, char*);
  int64_t lzbench_quicklz_decompress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t, size_t, char*);

_lzbench/lzbench.h

@@ -136,7 +136,7 @@ typedef struct
 
 
 
-#define LZBENCH_COMPRESSOR_COUNT 72
+#define LZBENCH_COMPRESSOR_COUNT 73
 
 static const compressor_desc_t comp_desc[LZBENCH_COMPRESSOR_COUNT] =
 {
@@ -185,6 +185,7 @@ static const compressor_desc_t comp_desc[LZBENCH_COMPRESSOR_COUNT] =
  { "lzsse8fast", "2019-04-18", 0, 0, 0, 0, lzbench_lzsse8fast_compress, lzbench_lzsse8_decompress, lzbench_lzsse8fast_init, lzbench_lzsse8fast_deinit },
  { "lzvn", "2017-03-08", 0, 0, 0, 0, lzbench_lzvn_compress, lzbench_lzvn_decompress, lzbench_lzvn_init, lzbench_lzvn_deinit },
  { "pithy", "2011-12-24", 0, 9, 0, 0, lzbench_pithy_compress, lzbench_pithy_decompress, NULL, NULL }, // decompression error (returns 0)
+ { "ppmd", "23.01", 1, 16, 0, 0, lzbench_ppmd_compress, lzbench_ppmd_decompress, NULL, NULL },
  { "quicklz", "1.5.0", 1, 3, 0, 0, lzbench_quicklz_compress, lzbench_quicklz_decompress, NULL, NULL },
  { "shrinker", "0.1", 0, 0, 0, 128<<20, lzbench_shrinker_compress, lzbench_shrinker_decompress, NULL, NULL },
  { "slz_deflate","1.2.0", 1, 3, 2, 0, lzbench_slz_compress, lzbench_slz_decompress, NULL, NULL },
@@ -195,7 +196,7 @@ static const compressor_desc_t comp_desc[LZBENCH_COMPRESSOR_COUNT] =
  { "ucl_nrv2b", "1.03", 1, 9, 0, 0, lzbench_ucl_nrv2b_compress, lzbench_ucl_nrv2b_decompress, NULL, NULL },
  { "ucl_nrv2d", "1.03", 1, 9, 0, 0, lzbench_ucl_nrv2d_compress, lzbench_ucl_nrv2d_decompress, NULL, NULL },
  { "ucl_nrv2e", "1.03", 1, 9, 0, 0, lzbench_ucl_nrv2e_compress, lzbench_ucl_nrv2e_decompress, NULL, NULL },
- { "wflz", "2015-09-16", 0, 0, 0, 0, lzbench_wflz_compress, lzbench_wflz_decompress, lzbench_wflz_init, lzbench_wflz_deinit }, // SEGFAULT on decompressiom with gcc 4.9+ -O3 on Ubuntu
+ { "wflz", "2015-09-16", 0, 0, 0, 0, lzbench_wflz_compress, lzbench_wflz_decompress, lzbench_wflz_init, lzbench_wflz_deinit }, // SEGFAULT on decompression with gcc 4.9+ -O3 on Ubuntu
  { "xpack", "2016-06-02", 1, 9, 0, 1<<19, lzbench_xpack_compress, lzbench_xpack_decompress, lzbench_xpack_init, lzbench_xpack_deinit },
  { "xz", "5.2.5", 0, 9, 0, 0, lzbench_xz_compress, lzbench_xz_decompress, NULL, NULL },
  { "yalz77", "2015-09-19", 1, 12, 0, 0, lzbench_yalz77_compress, lzbench_yalz77_decompress, NULL, NULL },
@@ -226,7 +227,7 @@ static const alias_desc_t alias_desc[LZBENCH_ALIASES_COUNT] =
  { "all", "blosclz,1,3,6,9/brieflz,1,3,6,8/brotli,0,2,5,8,11/bzip2,1,5,9/" \
  "crush,0,1,2/csc,1,3,5/density,1,2,3/fastlz,1,2/fastlzma2,1,3,5,8,10/gipfeli/libdeflate,1,3,6,9,12/lz4/lz4fast,3,17/lz4hc,1,4,9,12/" \
  "lzf,0,1/lzfse/lzg,1,4,6,8/lzham,0,1/lzjb/lzlib,0,3,6,9/lzma,0,2,4,5,9/lzo1/lzo1a/lzo1b,1,3,6,9,99,999/lzo1c,1,3,6,9,99,999/lzo1f/lzo1x/lzo1y/lzo1z/lzo2a/" \
- "lzrw,1,3,4,5/lzsse2,1,6,12,16/lzsse4,1,6,12,16/lzsse8,1,6,12,16/lzvn/pithy,0,3,6,9/quicklz,1,2,3/slz_gzip/snappy/tornado,1,2,3,4,5,6,7,10,13,16/" \
+ "lzrw,1,3,4,5/lzsse2,1,6,12,16/lzsse4,1,6,12,16/lzsse8,1,6,12,16/lzvn/pithy,0,3,6,9/ppmd,4/quicklz,1,2,3/slz_gzip/snappy/tornado,1,2,3,4,5,6,7,10,13,16/" \
  "ucl_nrv2b,1,6,9/ucl_nrv2d,1,6,9/ucl_nrv2e,1,6,9/xpack,1,6,9/xz,0,3,6,9/yalz77,1,4,8,12/yappy,1,10,100/zlib,1,6,9/zling,0,1,2,3,4/zstd,1,2,5,8,11,15,18,22/" \
  "shrinker/wflz/lzmat" }, // these can SEGFAULT
 #else

libppmd/.gitignore

@@ -0,0 +1,3 @@
+vs/.vs
+vs/msvc.build
+*.vcxproj.user

libppmd/Makefile

@@ -0,0 +1,3 @@
+CFLAGS = -g -O3 -flto -fwhole-program -Wall -Wextra
+ppmd-mini: ppmd-mini.c lib/Ppmd*.c
+ $(CC) -Ilib $(CFLAGS) -o $@ $^

libppmd/README.md

@@ -0,0 +1,9 @@
+# ppmd-mini - standalone PPMd compressor
+
+ppmd-mini is a data compression tool with command line syntax similar to gzip.
+It uses the PPMd algorithm which provides superior compression for text files.
+Specifically, it uses "variant I" of this algorithm which is also used in Zip.
+PPMd was developed by Dmitry Shkarin; Igor Pavlov adapted his work for use in
+the 7-Zip project. ppmd-mini employs Ppmd8 compression and decompression
+routines from 7-Zip. Files created by ppmd-mini are compatible with the
+original PPMd implementation; they can also be decompressed with 7-Zip.