inikep · ChristianSchindler · Mar 14, 2024 · Mar 14, 2024
diff --git a/README.md b/README.md
@@ -93,7 +93,7 @@ see the [CompFuzz Results](https://github.com/nemequ/compfuzz/wiki/Results) page
  - [glza 0.8](https://encode.su/threads/2427-GLZA)
  - [libdeflate v1.6](https://github.com/ebiggers/libdeflate)
  - [lizard v1.0 (formerly lz5)](https://github.com/inikep/lizard)
- - [lz4/lz4hc v1.9.3](https://github.com/lz4/lz4)
+ - [lz4/lz4hc v1.9.4](https://github.com/lz4/lz4)
  - [lzf 3.6](http://software.schmorp.de/pkg/liblzf.html)
  - [lzfse/lzvn 1.0](https://github.com/lzfse/lzfse)
  - [lzg 1.0.10](https://liblzg.bitsnbites.eu/)

diff --git a/_lzbench/lzbench.h b/_lzbench/lzbench.h
@@ -155,7 +155,7 @@ static const compressor_desc_t comp_desc[LZBENCH_COMPRESSOR_COUNT] =
  { "gipfeli", "2016-07-13", 0, 0, 0, 0, lzbench_gipfeli_compress, lzbench_gipfeli_decompress, NULL, NULL },
  { "glza", "0.8", 0, 0, 0, 0, lzbench_glza_compress, lzbench_glza_decompress, NULL, NULL },
  { "libdeflate", "1.9", 1, 12, 0, 0, lzbench_libdeflate_compress, lzbench_libdeflate_decompress, NULL, NULL },
- { "lz4", "1.9.3", 0, 0, 0, 0, lzbench_lz4_compress, lzbench_lz4_decompress, NULL, NULL },
+ { "lz4", "1.9.4", 0, 0, 0, 0, lzbench_lz4_compress, lzbench_lz4_decompress, NULL, NULL },
  { "lz4fast", "1.9.3", 1, 99, 0, 0, lzbench_lz4fast_compress, lzbench_lz4_decompress, NULL, NULL },
  { "lz4hc", "1.9.3", 1, 12, 0, 0, lzbench_lz4hc_compress, lzbench_lz4_decompress, NULL, NULL },
  { "lzf", "3.6", 0, 1, 0, 0, lzbench_lzf_compress, lzbench_lzf_decompress, NULL, NULL },

diff --git a/lz4/LICENSE b/lz4/LICENSE
@@ -1,5 +1,5 @@
 LZ4 Library
-Copyright (c) 2011-2016, Yann Collet
+Copyright (c) 2011-2020, Yann Collet
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification,

diff --git a/lz4/Makefile b/lz4/Makefile
@@ -1,6 +1,6 @@
 # ################################################################
 # LZ4 library - Makefile
-# Copyright (C) Yann Collet 2011-2016
+# Copyright (C) Yann Collet 2011-2020
 # All rights reserved.
 #
 # This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets
@@ -31,11 +31,12 @@
 # - LZ4 source repository : https://github.com/lz4/lz4
 # - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c
 # ################################################################
+SED = sed
 
 # Version numbers
-LIBVER_MAJOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./lz4.h`
-LIBVER_MINOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./lz4.h`
-LIBVER_PATCH_SCRIPT:=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./lz4.h`
+LIBVER_MAJOR_SCRIPT:=`$(SED) -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./lz4.h`
+LIBVER_MINOR_SCRIPT:=`$(SED) -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./lz4.h`
+LIBVER_PATCH_SCRIPT:=`$(SED) -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./lz4.h`
 LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT)
 LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT))
 LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT))
@@ -46,12 +47,13 @@ BUILD_SHARED:=yes
 BUILD_STATIC:=yes
 
 CPPFLAGS+= -DXXH_NAMESPACE=LZ4_
+CPPFLAGS+= $(MOREFLAGS)
 CFLAGS ?= -O3
 DEBUGFLAGS:= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
  -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
  -Wundef -Wpointer-arith -Wstrict-aliasing=1
-CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
-FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
+CFLAGS += $(DEBUGFLAGS)
+FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
 
 SRCFILES := $(sort $(wildcard *.c))
 
@@ -74,27 +76,33 @@ endif
 .PHONY: default
 default: lib-release
 
+# silent mode by default; verbose can be triggered by V=1 or VERBOSE=1
+$(V)$(VERBOSE).SILENT:
+
 lib-release: DEBUGFLAGS :=
 lib-release: lib
 
+.PHONY: lib
 lib: liblz4.a liblz4
 
+.PHONY: all
 all: lib
 
+.PHONY: all32
 all32: CFLAGS+=-m32
 all32: all
 
 liblz4.a: $(SRCFILES)
 ifeq ($(BUILD_STATIC),yes) # can be disabled on command line
  @echo compiling static library
- $(Q)$(CC) $(CPPFLAGS) $(CFLAGS) -c $^
- $(Q)$(AR) rcs $@ *.o
+ $(COMPILE.c) $^
+ $(AR) rcs $@ *.o
 endif
 
 ifeq ($(WINBASED),yes)
 liblz4-dll.rc: liblz4-dll.rc.in
  @echo creating library resource
- $(Q)sed -e 's|@LIBLZ4@|$(LIBLZ4)|' \
+ $(SED) -e 's|@LIBLZ4@|$(LIBLZ4)|' \
  -e 's|@LIBVER_MAJOR@|$(LIBVER_MAJOR)|g' \
  -e 's|@LIBVER_MINOR@|$(LIBVER_MINOR)|g' \
  -e 's|@LIBVER_PATCH@|$(LIBVER_PATCH)|g' \
@@ -104,31 +112,30 @@ liblz4-dll.o: liblz4-dll.rc
  $(WINDRES) -i liblz4-dll.rc -o liblz4-dll.o
 
 $(LIBLZ4): $(SRCFILES) liblz4-dll.o
-else
+ @echo compiling dynamic library $(LIBVER)
+ $(CC) $(FLAGS) -DLZ4_DLL_EXPORT=1 -shared $^ -o dll/[email protected] -Wl,--out-implib,dll/$(LIBLZ4_EXP)
+
+else # not windows
+
 $(LIBLZ4): $(SRCFILES)
-endif
-ifeq ($(BUILD_SHARED),yes) # can be disabled on command line
  @echo compiling dynamic library $(LIBVER)
- ifeq ($(WINBASED),yes)
- $(Q)$(CC) $(FLAGS) -DLZ4_DLL_EXPORT=1 -shared $^ -o dll/[email protected] -Wl,--out-implib,dll/$(LIBLZ4_EXP)
- else
- $(Q)$(CC) $(FLAGS) -shared $^ -fPIC -fvisibility=hidden $(SONAME_FLAGS) -o $@
+ $(CC) $(FLAGS) -shared $^ -fPIC -fvisibility=hidden $(SONAME_FLAGS) -o $@
  @echo creating versioned links
- $(Q)$(LN_SF) $@ liblz4.$(SHARED_EXT_MAJOR)
- $(Q)$(LN_SF) $@ liblz4.$(SHARED_EXT)
- endif
+ $(LN_SF) $@ liblz4.$(SHARED_EXT_MAJOR)
+ $(LN_SF) $@ liblz4.$(SHARED_EXT)
+
 endif
 
-ifeq (,$(filter MINGW%,$(TARGET_OS)))
+.PHONY: liblz4
 liblz4: $(LIBLZ4)
-endif
 
+.PHONY: clean
 clean:
 ifeq ($(WINBASED),yes)
- $(Q)$(RM) *.rc
+ $(RM) *.rc
 endif
- $(Q)$(RM) core *.o liblz4.pc dll/$(LIBLZ4).dll dll/$(LIBLZ4_EXP)
- $(Q)$(RM) *.a *.$(SHARED_EXT) *.$(SHARED_EXT_MAJOR) *.$(SHARED_EXT_VER)
+ $(RM) core *.o liblz4.pc dll/$(LIBLZ4).dll dll/$(LIBLZ4_EXP)
+ $(RM) *.a *.$(SHARED_EXT) *.$(SHARED_EXT_MAJOR) *.$(SHARED_EXT_VER)
  @echo Cleaning library completed
 
 #-----------------------------------------------------------------------------
@@ -164,54 +171,55 @@ pkgconfigdir ?= $(PKGCONFIGDIR)
 
 liblz4.pc: liblz4.pc.in Makefile
  @echo creating pkgconfig
- $(Q)sed -e 's|@PREFIX@|$(prefix)|' \
+ $(SED) -e 's|@PREFIX@|$(prefix)|' \
  -e 's|@LIBDIR@|$(libdir)|' \
  -e 's|@INCLUDEDIR@|$(includedir)|' \
  -e 's|@VERSION@|$(LIBVER)|' \
+ -e 's|=${prefix}/|=$${prefix}/|' \
  $< >$@
 
 install: lib liblz4.pc
- $(Q)$(INSTALL_DIR) $(DESTDIR)$(pkgconfigdir)/ $(DESTDIR)$(includedir)/ $(DESTDIR)$(libdir)/ $(DESTDIR)$(bindir)/
- $(Q)$(INSTALL_DATA) liblz4.pc $(DESTDIR)$(pkgconfigdir)/
- @echo Installing libraries
+ $(INSTALL_DIR) $(DESTDIR)$(pkgconfigdir)/ $(DESTDIR)$(includedir)/ $(DESTDIR)$(libdir)/ $(DESTDIR)$(bindir)/
+ $(INSTALL_DATA) liblz4.pc $(DESTDIR)$(pkgconfigdir)/
+ @echo Installing libraries in $(DESTDIR)$(libdir)
  ifeq ($(BUILD_STATIC),yes)
- $(Q)$(INSTALL_DATA) liblz4.a $(DESTDIR)$(libdir)/liblz4.a
- $(Q)$(INSTALL_DATA) lz4frame_static.h $(DESTDIR)$(includedir)/lz4frame_static.h
+ $(INSTALL_DATA) liblz4.a $(DESTDIR)$(libdir)/liblz4.a
+ $(INSTALL_DATA) lz4frame_static.h $(DESTDIR)$(includedir)/lz4frame_static.h
  endif
  ifeq ($(BUILD_SHARED),yes)
-# Traditionnally, one installs the DLLs in the bin directory as programs
+# Traditionally, one installs the DLLs in the bin directory as programs
 # search them first in their directory. This allows to not pollute system
 # directories (like c:/windows/system32), nor modify the PATH variable.
  ifeq ($(WINBASED),yes)
- $(Q)$(INSTALL_PROGRAM) dll/$(LIBLZ4).dll $(DESTDIR)$(bindir)
- $(Q)$(INSTALL_PROGRAM) dll/$(LIBLZ4_EXP) $(DESTDIR)$(libdir)
+ $(INSTALL_PROGRAM) dll/$(LIBLZ4).dll $(DESTDIR)$(bindir)
+ $(INSTALL_PROGRAM) dll/$(LIBLZ4_EXP) $(DESTDIR)$(libdir)
  else
- $(Q)$(INSTALL_PROGRAM) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)
- $(Q)$(LN_SF) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_MAJOR)
- $(Q)$(LN_SF) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT)
+ $(INSTALL_PROGRAM) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)
+ $(LN_SF) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_MAJOR)
+ $(LN_SF) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT)
  endif
  endif
- @echo Installing headers in $(includedir)
- $(Q)$(INSTALL_DATA) lz4.h $(DESTDIR)$(includedir)/lz4.h
- $(Q)$(INSTALL_DATA) lz4hc.h $(DESTDIR)$(includedir)/lz4hc.h
- $(Q)$(INSTALL_DATA) lz4frame.h $(DESTDIR)$(includedir)/lz4frame.h
+ @echo Installing headers in $(DESTDIR)$(includedir)
+ $(INSTALL_DATA) lz4.h $(DESTDIR)$(includedir)/lz4.h
+ $(INSTALL_DATA) lz4hc.h $(DESTDIR)$(includedir)/lz4hc.h
+ $(INSTALL_DATA) lz4frame.h $(DESTDIR)$(includedir)/lz4frame.h
  @echo lz4 libraries installed
 
 uninstall:
- $(Q)$(RM) $(DESTDIR)$(pkgconfigdir)/liblz4.pc
+ $(RM) $(DESTDIR)$(pkgconfigdir)/liblz4.pc
  ifeq (WINBASED,1)
- $(Q)$(RM) $(DESTDIR)$(bindir)/$(LIBLZ4).dll
- $(Q)$(RM) $(DESTDIR)$(libdir)/$(LIBLZ4_EXP)
+ $(RM) $(DESTDIR)$(bindir)/$(LIBLZ4).dll
+ $(RM) $(DESTDIR)$(libdir)/$(LIBLZ4_EXP)
  else
- $(Q)$(RM) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT)
- $(Q)$(RM) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_MAJOR)
- $(Q)$(RM) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_VER)
+ $(RM) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT)
+ $(RM) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_MAJOR)
+ $(RM) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_VER)
  endif
- $(Q)$(RM) $(DESTDIR)$(libdir)/liblz4.a
- $(Q)$(RM) $(DESTDIR)$(includedir)/lz4.h
- $(Q)$(RM) $(DESTDIR)$(includedir)/lz4hc.h
- $(Q)$(RM) $(DESTDIR)$(includedir)/lz4frame.h
- $(Q)$(RM) $(DESTDIR)$(includedir)/lz4frame_static.h
+ $(RM) $(DESTDIR)$(libdir)/liblz4.a
+ $(RM) $(DESTDIR)$(includedir)/lz4.h
+ $(RM) $(DESTDIR)$(includedir)/lz4hc.h
+ $(RM) $(DESTDIR)$(includedir)/lz4frame.h
+ $(RM) $(DESTDIR)$(includedir)/lz4frame_static.h
  @echo lz4 libraries successfully uninstalled
 
 endif
diff --git a/lz4/README.md b/lz4/README.md
@@ -2,16 +2,20 @@ LZ4 - Library Files
 ================================
 
 The `/lib` directory contains many files, but depending on project's objectives,
-not all of them are necessary.
+not all of them are required.
+Limited systems may want to reduce the nb of source files to include
+as a way to reduce binary size and dependencies.
 
-#### Minimal LZ4 build
+Capabilities are added at the "level" granularity, detailed below.
+
+#### Level 1 : Minimal LZ4 build
 
 The minimum required is **`lz4.c`** and **`lz4.h`**,
 which provides the fast compression and decompression algorithms.
 They generate and decode data using the [LZ4 block format].
 
 
-#### High Compression variant
+#### Level 2 : High Compression variant
 
 For more compression ratio at the cost of compression speed,
 the High Compression variant called **lz4hc** is available.
@@ -20,22 +24,37 @@ This variant also compresses data using the [LZ4 block format],
 and depends on regular `lib/lz4.*` source files.
 
 
-#### Frame support, for interoperability
+#### Level 3 : Frame support, for interoperability
 
 In order to produce compressed data compatible with `lz4` command line utility,
 it's necessary to use the [official interoperable frame format].
 This format is generated and decoded automatically by the **lz4frame** library.
 Its public API is described in `lib/lz4frame.h`.
 In order to work properly, lz4frame needs all other modules present in `/lib`,
 including, lz4 and lz4hc, and also **xxhash**.
-So it's necessary to include all `*.c` and `*.h` files present in `/lib`.
+So it's necessary to also include `xxhash.c` and `xxhash.h`.
+
+
+#### Level 4 : File compression operations
+
+As a helper around file operations,
+the library has been recently extended with `lz4file.c` and `lz4file.h`
+(still considered experimental at the time of this writing).
+These helpers allow opening, reading, writing, and closing files
+using transparent LZ4 compression / decompression.
+As a consequence, using `lz4file` adds a dependency on `<stdio.h>`.
+
+`lz4file` relies on `lz4frame` in order to produce compressed data
+conformant to the [LZ4 Frame format] specification.
+Consequently, to enable this capability,
+it's necessary to include all `*.c` and `*.h` files from `lib/` directory.
 
 
 #### Advanced / Experimental API
 
 Definitions which are not guaranteed to remain stable in future versions,
 are protected behind macros, such as `LZ4_STATIC_LINKING_ONLY`.
-As the name strongly implies, these definitions should only be invoked
+As the name suggests, these definitions should only be invoked
 in the context of static linking ***only***.
 Otherwise, dependent application may fail on API or ABI break in the future.
 The associated symbols are also not exposed by the dynamic library by default.
@@ -58,7 +77,7 @@ The following build macro can be selected to adjust source code behavior at comp
  Set to 65535 by default, which is the maximum value supported by lz4 format.
  Reducing maximum distance will reduce opportunities for LZ4 to find matches,
  hence will produce a worse compression ratio.
- However, a smaller max distance can allow compatibility with specific decoders using limited memory budget.
+ Setting a smaller max distance could allow compatibility with specific decoders with limited memory budget.
  This build macro only influences the compressed output of the compressor.
 
 - `LZ4_DISABLE_DEPRECATE_WARNINGS` : invoking a deprecated function will make the compiler generate a warning.
@@ -69,22 +88,34 @@ The following build macro can be selected to adjust source code behavior at comp
  This build macro offers another project-specific method
  by defining `LZ4_DISABLE_DEPRECATE_WARNINGS` before including the LZ4 header files.
 
-- `LZ4_USER_MEMORY_FUNCTIONS` : replace calls to <stdlib>'s `malloc`, `calloc` and `free`
- by user-defined functions, which must be called `LZ4_malloc()`, `LZ4_calloc()` and `LZ4_free()`.
- User functions must be available at link time.
-
 - `LZ4_FORCE_SW_BITCOUNT` : by default, the compression algorithm tries to determine lengths
  by using bitcount instructions, generally implemented as fast single instructions in many cpus.
  In case the target cpus doesn't support it, or compiler intrinsic doesn't work, or feature bad performance,
  it's possible to use an optimized software path instead.
- This is achieved by setting this build macros .
+ This is achieved by setting this build macros.
  In most cases, it's not expected to be necessary,
  but it can be legitimately considered for less common platforms.
 
 - `LZ4_ALIGN_TEST` : alignment test ensures that the memory area
  passed as argument to become a compression state is suitably aligned.
  This test can be disabled if it proves flaky, by setting this value to 0.
 
+- `LZ4_USER_MEMORY_FUNCTIONS` : replace calls to `<stdlib,h>`'s `malloc()`, `calloc()` and `free()`
+ by user-defined functions, which must be named `LZ4_malloc()`, `LZ4_calloc()` and `LZ4_free()`.
+ User functions must be available at link time.
+
+- `LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION` :
+ Remove support of dynamic memory allocation.
+ For more details, see description of this macro in `lib/lz4.c`.
+
+- `LZ4_FREESTANDING` : by setting this build macro to 1,
+ LZ4/HC removes dependencies on the C standard library,
+ including allocation functions and `memmove()`, `memcpy()`, and `memset()`.
+ This build macro is designed to help use LZ4/HC in restricted environments
+ (embedded, bootloader, etc).
+ For more details, see description of this macro in `lib/lz4.h`.
+
+
 
 #### Amalgamation
 
@@ -101,7 +132,7 @@ All `*.h` files present in `/lib` remain necessary to compile `lz4_all.c`.
 
 DLL can be created using MinGW+MSYS with the `make liblz4` command.
 This command creates `dll\liblz4.dll` and the import library `dll\liblz4.lib`.
-To override the `dlltool` command  when cross-compiling on Linux, just set the `DLLTOOL` variable. Example of cross compilation on Linux with mingw-w64 64 bits:
+To override the `dlltool` command when cross-compiling on Linux, just set the `DLLTOOL` variable. Example of cross compilation on Linux with mingw-w64 64 bits:
 ```
 make BUILD_STATIC=no CC=x86_64-w64-mingw32-gcc DLLTOOL=x86_64-w64-mingw32-dlltool OS=Windows_NT
 ```
@@ -126,8 +157,9 @@ Other files present in the directory are not source code. They are :
  - `liblz4.pc.in` : for `pkg-config` (used in `make install`)
  - `README.md` : this file
 
-[official interoperable frame format]: ../doc/lz4_Frame_format.md
-[LZ4 block format]: ../doc/lz4_Block_format.md
+[official interoperable frame format]: ../../../Downloads/lz4-1.9.4/lz4-1.9.4/doc/lz4_Frame_format.md
+[LZ4 Frame format]: ../../../Downloads/lz4-1.9.4/lz4-1.9.4/doc/lz4_Frame_format.md
+[LZ4 block format]: ../../../Downloads/lz4-1.9.4/lz4-1.9.4/doc/lz4_Block_format.md
 
 
 #### License

diff --git a/lz4/dll/example/Makefile b/lz4/dll/example/Makefile
@@ -1,6 +1,6 @@
 # ##########################################################################
 # LZ4 programs - Makefile
-# Copyright (C) Yann Collet 2016
+# Copyright (C) Yann Collet 2016-2020
 #
 # GPL v2 License
 #