Legacy default off

.buckconfig

@@ -1,7 +1,7 @@
 [cxx]
-  cppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=4
+  cppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=0
   cflags = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef -Wpointer-arith
-  cxxppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=4
+  cxxppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=0
   cxxflags = -std=c++11 -Wno-deprecated-declarations
   gtest_dep = //contrib/pzstd:gtest
 

.github/workflows/dev-long-tests.yml

@@ -160,6 +160,16 @@ jobs:
     - name: clang + ASan + UBSan + Fuzz Test
       run: CC=clang FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest
 
+  # the only job decoding real legacy frames under sanitizers.
+  # fuzzers do build these decoders, but only reach them if an input
+  # happens to carry a legacy magic number.
+  clang-asan-ubsan-legacy:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
+    - name: clang + ASan + UBSan + legacy decoders
+      run: CC=clang make uasan-test-legacy
+
   gcc-asan-ubsan-fuzz32:
     runs-on: ubuntu-latest
     steps:

.github/workflows/dev-short-tests.yml

@@ -237,6 +237,7 @@ jobs:
             -Dbin_tests=true \
             -Dbin_contrib=true \
             -Ddefault_library=both \
+            -Dlegacy_level=1 \
             build/meson mesonBuild
           ninja -C mesonBuild/
           meson test -C mesonBuild/ --print-errorlogs

build/cmake/programs/CMakeLists.txt

@@ -16,11 +16,6 @@ set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
 set(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
 include_directories(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/compress ${LIBRARY_DIR}/dictBuilder)
 
-if (ZSTD_LEGACY_SUPPORT)
-    set(PROGRAMS_LEGACY_DIR ${PROGRAMS_DIR}/legacy)
-    include_directories(${PROGRAMS_LEGACY_DIR} ${LIBRARY_DIR}/legacy)
-endif ()
-
 if (ZSTD_PROGRAMS_LINK_SHARED)
     set(PROGRAMS_ZSTD_LINK_TARGET libzstd_shared)
 else ()

build/meson/tests/meson.build

@@ -102,12 +102,11 @@ invalidDictionaries = executable('invalidDictionaries',
   dependencies: [ libzstd_dep ],
   install: false)
 
-if 0 < legacy_level and legacy_level <= 4
+if legacy_level > 0
   legacy_sources = [join_paths(zstd_rootdir, 'tests/legacy.c')]
+  # the test adapts to ZSTD_LEGACY_SUPPORT, set project wide from legacy_level
   legacy = executable('legacy',
     legacy_sources,
-    # Use -Dlegacy_level build option to control it
-    #c_args: '-DZSTD_LEGACY_SUPPORT=4',
     dependencies: [ libzstd_dep ],
     install: false)
 endif
@@ -199,7 +198,7 @@ test('test-zstream-3',
   timeout: 120)
 test('test-longmatch', longmatch, timeout: 36)
 test('test-invalidDictionaries', invalidDictionaries) # should be fast
-if 0 < legacy_level and legacy_level <= 4
+if legacy_level > 0
   test('test-legacy', legacy) # should be fast
 endif
 test('test-decodecorpus',

contrib/VS2005/zstd/zstd.vcproj

@@ -44,7 +44,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -121,7 +121,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
@@ -196,7 +196,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -274,7 +274,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"

contrib/VS2005/zstdlib/zstdlib.vcproj

@@ -44,7 +44,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -120,7 +120,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
@@ -194,7 +194,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -271,7 +271,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"

lib/BUCK

@@ -32,7 +32,6 @@ cxx_library(
     srcs=glob(['decompress/zstd*.c']),
     deps=[
         ':common',
-        ':legacy',
     ],
 )
 
@@ -47,6 +46,14 @@ cxx_library(
     deps=[':common'],
 )
 
+# Decoders for formats older than v0.8. Not a dependency of :decompress,
+# because ZSTD_LEGACY_SUPPORT is 0 (see .buckconfig), which makes them
+# unreachable.
+# Enabling legacy formats means two edits, which only make sense together :
+# raise the level in .buckconfig, AND add ':legacy' to :decompress deps.
+# Raising the level alone fails to link, which is easy to notice. Adding the
+# dep alone is silent : these decoders get compiled back in, while
+# ZSTD_isLegacy() remains compiled out, so legacy frames are still rejected.
 cxx_library(
     name='legacy',
     header_namespace='',
@@ -56,9 +63,6 @@ cxx_library(
     ]),
     srcs=glob(['legacy/*.c']),
     deps=[':common'],
-    exported_preprocessor_flags=[
-        '-DZSTD_LEGACY_SUPPORT=0',
-    ],
 )
 
 cxx_library(
@@ -132,7 +136,7 @@ cxx_library(
     exported_headers=[
         'zstd_errors.h',
         'common/error_private.h',
-    ]
+    ],
     srcs=['common/error_private.c'],
 )
 

programs/README.md

@@ -3,8 +3,8 @@ Command Line Interface for Zstandard library
 
 Command Line Interface (CLI) can be created using the `make` command without any additional parameters.
 There are however other Makefile targets that create different variations of CLI:
-- `zstd` : default CLI supporting gzip-like arguments; includes dictionary builder, benchmark, and supports decompression of legacy zstd formats
-- `zstd_nolegacy` : Same as `zstd` but without support for legacy zstd formats
+- `zstd` : default CLI supporting gzip-like arguments; includes dictionary builder and benchmark
+- `zstd_nolegacy` : Same as `zstd` but with support for legacy zstd formats explicitly disabled
 - `zstd-small` : CLI optimized for minimal size; no dictionary builder, no benchmark, and no support for legacy zstd formats
 - `zstd-compress` : version of CLI which can only compress into zstd format
 - `zstd-decompress` : version of CLI which can only decompress zstd format
@@ -23,11 +23,12 @@ There are however other Makefile targets that create different variations of CLI
 - __ZSTD_LEGACY_SUPPORT__ : `zstd` can decompress files compressed by older versions of `zstd`.
   Starting v0.8.0, all versions of `zstd` produce frames compliant with the [specification](../doc/zstd_compression_format.md), and are therefore compatible.
   But older versions (< v0.8.0) produced different, incompatible, frames.
-  By default, `zstd` supports decoding legacy formats >= v0.4.0 (`ZSTD_LEGACY_SUPPORT=4`).
+  Since v1.6.0, legacy format support is disabled by default (`ZSTD_LEGACY_SUPPORT=0`).
   This can be altered by modifying this compilation variable.
   `ZSTD_LEGACY_SUPPORT=1` means "support all formats >= v0.1.0".
   `ZSTD_LEGACY_SUPPORT=2` means "support all formats >= v0.2.0", and so on.
   `ZSTD_LEGACY_SUPPORT=0` means _DO NOT_ support any legacy format.
+  Example : `make zstd ZSTD_LEGACY_SUPPORT=4`
   if `ZSTD_LEGACY_SUPPORT >= 8`, it's the same as `0`, since there is no legacy format after `7`.
   Note : `zstd` only supports decoding older formats, and cannot generate any legacy format.
 

tests/Makefile

@@ -44,8 +44,12 @@ CPPFLAGS   += -I$(LIB_SRCDIR) -I$(LIB_SRCDIR)/common -I$(LIB_SRCDIR)/compress -I
 ZSTDCOMMON_FILES := $(sort $(ZSTD_COMMON_FILES))
 ZSTDCOMP_FILES   := $(sort $(ZSTD_COMPRESS_FILES))
 ZSTDDECOMP_FILES := $(sort $(ZSTD_DECOMPRESS_FILES))
-ZSTDLEGACY_FILES := $(sort $(wildcard $(LIB_SRCDIR)/legacy/*.c))
-ZSTD_FILES  := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) $(ZSTDLEGACY_FILES)
+# every decoder, whatever ZSTD_LEGACY_SUPPORT : only the `legacy` target wants these
+ALL_LEGACY_FILES := $(sort $(wildcard $(LIB_SRCDIR)/legacy/*.c))
+# ZSTD_LEGACY_FILES (from libzstd.mk) is the subset selected for the requested
+# level. It's empty when legacy support is disabled, hence unreachable decoders
+# are not compiled into the test binaries.
+ZSTD_FILES  := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) $(sort $(ZSTD_LEGACY_FILES))
 ZDICT_FILES := $(sort $(ZSTD_DICTBUILDER_FILES))
 
 ZSTD_F1 := $(sort $(wildcard $(ZSTD_FILES)))
@@ -233,8 +237,11 @@ CLEAN += invalidDictionaries
 invalidDictionaries : $(ZSTD_OBJECTS) invalidDictionaries.c
 
 CLEAN += legacy
-legacy : CPPFLAGS += -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=4
-legacy : $(ZSTD_FILES) legacy.c
+# legacy.c adapts to the level it is compiled with : it only tests the frames
+# this level can decode. Default to the oldest format, to cover all decoders.
+LEGACY_TEST_LEVEL ?= 1
+legacy : CPPFLAGS += -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=$(LEGACY_TEST_LEVEL)
+legacy : $(ZSTD_FILES) $(ALL_LEGACY_FILES) legacy.c
 
 CLEAN += decodecorpus
 decodecorpus : LDLIBS += -lm

tests/fuzz/Makefile

@@ -26,6 +26,7 @@ CORPORA_URL_PREFIX:=https://github.com/facebook/zstd/releases/download/fuzz-corp
 
 LIBZSTD_MK_DIR = ../../lib
 DEBUGLEVEL ?= 2
+# fuzzers keep legacy decoders enabled, they are still built on demand
 ZSTD_LEGACY_SUPPORT ?= 1
 
 .PHONY: default
@@ -40,10 +41,14 @@ DEFAULT_SEQ_PROD_DIR = $(CONTRIBDIR)/externalSequenceProducer
 DEFAULT_SEQ_PROD_SRC = $(DEFAULT_SEQ_PROD_DIR)/sequence_producer.c
 THIRD_PARTY_SEQ_PROD_OBJ ?=
 
+# Note : ZSTD_LEGACY_SUPPORT below looks redundant with libzstd.mk, which adds
+# the same define to CPPFLAGS. It is not : fuzz.py passes CPPFLAGS on the make
+# command line, and that overrides the `CPPFLAGS +=` there. Dropping it silently
+# disables the legacy decoders in every OSS-Fuzz and regressiontest build.
 FUZZ_CPPFLAGS := -I$(LIB_SRCDIR) -I$(LIB_SRCDIR)/common -I$(LIB_SRCDIR)/compress \
 	-I$(LIB_SRCDIR)/dictBuilder -I$(LIB_SRCDIR)/deprecated -I$(LIB_SRCDIR)/legacy \
 	-I$(CONTRIBDIR)/seekable_format -I$(PRGDIR) -I$(DEFAULT_SEQ_PROD_DIR) \
-	-DZSTD_MULTITHREAD -DZSTD_LEGACY_SUPPORT=1 $(CPPFLAGS)
+	-DZSTD_MULTITHREAD -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $(CPPFLAGS)
 FUZZ_EXTRA_FLAGS := -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
 	-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
 	-Wstrict-prototypes -Wundef \