fix: validate Content_Checksum on decode, add opt-in encoder emission

CHANGELOG.md

@@ -8,7 +8,24 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 Encoder-side parity work closing several gaps against the libzstd/RFC 8878
 spec — real ratio improvements for dictionary-compressed frames, no wire
-format or public-API changes.
+format or public-API changes (except where noted below, for the
+Content_Checksum fix).
+
+### Fixed
+
+- The decoder now validates a frame's Content_Checksum when
+  `Content_Checksum_Flag` is set: it reads the trailing 4-byte XXH64
+  checksum and throws `ZstdException` on a mismatch against the decoded
+  content, for ANY conformant frame — not just kzstd's own — so a real
+  `libzstd`-produced frame (checksums are on by default in the `zstd` CLI)
+  is no longer accepted with silently-corrupted content.
+
+### Added
+
+- `Zstd.compress` takes an opt-in `checksum: Boolean = false` parameter; when
+  true, the encoder sets `Content_Checksum_Flag` and appends the XXH64
+  checksum of the input. Defaults to false, so every existing call site's
+  frame bytes are unchanged.
 
 ### Changed
 

api/kzstd.api

@@ -1,10 +1,10 @@
 public final class org/meshtastic/kzstd/Zstd {
 	public static final field DEFAULT_LEVEL I
 	public static final field INSTANCE Lorg/meshtastic/kzstd/Zstd;
-	public final fun compress ([BI)[B
-	public final fun compress ([BLorg/meshtastic/kzstd/ZstdDictionary;I)[B
-	public static synthetic fun compress$default (Lorg/meshtastic/kzstd/Zstd;[BIILjava/lang/Object;)[B
-	public static synthetic fun compress$default (Lorg/meshtastic/kzstd/Zstd;[BLorg/meshtastic/kzstd/ZstdDictionary;IILjava/lang/Object;)[B
+	public final fun compress ([BIZ)[B
+	public final fun compress ([BLorg/meshtastic/kzstd/ZstdDictionary;IZ)[B
+	public static synthetic fun compress$default (Lorg/meshtastic/kzstd/Zstd;[BIZILjava/lang/Object;)[B
+	public static synthetic fun compress$default (Lorg/meshtastic/kzstd/Zstd;[BLorg/meshtastic/kzstd/ZstdDictionary;IZILjava/lang/Object;)[B
 	public final fun decompress ([BI)[B
 	public final fun decompress ([BLorg/meshtastic/kzstd/ZstdDictionary;I)[B
 }

api/kzstd.klib.api

@@ -23,8 +23,8 @@ final object org.meshtastic.kzstd/Zstd { // org.meshtastic.kzstd/Zstd|null[0]
     final const val DEFAULT_LEVEL // org.meshtastic.kzstd/Zstd.DEFAULT_LEVEL|{}DEFAULT_LEVEL[0]
         final fun <get-DEFAULT_LEVEL>(): kotlin/Int // org.meshtastic.kzstd/Zstd.DEFAULT_LEVEL.<get-DEFAULT_LEVEL>|<get-DEFAULT_LEVEL>(){}[0]
 
-    final fun compress(kotlin/ByteArray, kotlin/Int = ...): kotlin/ByteArray // org.meshtastic.kzstd/Zstd.compress|compress(kotlin.ByteArray;kotlin.Int){}[0]
-    final fun compress(kotlin/ByteArray, org.meshtastic.kzstd/ZstdDictionary, kotlin/Int = ...): kotlin/ByteArray // org.meshtastic.kzstd/Zstd.compress|compress(kotlin.ByteArray;org.meshtastic.kzstd.ZstdDictionary;kotlin.Int){}[0]
+    final fun compress(kotlin/ByteArray, kotlin/Int = ..., kotlin/Boolean = ...): kotlin/ByteArray // org.meshtastic.kzstd/Zstd.compress|compress(kotlin.ByteArray;kotlin.Int;kotlin.Boolean){}[0]
+    final fun compress(kotlin/ByteArray, org.meshtastic.kzstd/ZstdDictionary, kotlin/Int = ..., kotlin/Boolean = ...): kotlin/ByteArray // org.meshtastic.kzstd/Zstd.compress|compress(kotlin.ByteArray;org.meshtastic.kzstd.ZstdDictionary;kotlin.Int;kotlin.Boolean){}[0]
     final fun decompress(kotlin/ByteArray, kotlin/Int): kotlin/ByteArray // org.meshtastic.kzstd/Zstd.decompress|decompress(kotlin.ByteArray;kotlin.Int){}[0]
     final fun decompress(kotlin/ByteArray, org.meshtastic.kzstd/ZstdDictionary, kotlin/Int): kotlin/ByteArray // org.meshtastic.kzstd/Zstd.decompress|decompress(kotlin.ByteArray;org.meshtastic.kzstd.ZstdDictionary;kotlin.Int){}[0]
 }

src/commonMain/kotlin/org/meshtastic/kzstd/Zstd.kt

@@ -37,17 +37,31 @@ public object Zstd {
      */
     public const val DEFAULT_LEVEL: Int = 19
 
-    /** Compress [data] into a standard zstd frame using [dictionary]. */
+    /**
+     * Compress [data] into a standard zstd frame using [dictionary].
+     *
+     * [checksum], when true, sets Content_Checksum_Flag and appends a
+     * trailing 4-byte XXH64 checksum of [data] (RFC 8878 §3.1.1) that
+     * [decompress] (and any conformant decoder) validates. Defaults to false
+     * — existing callers' frame bytes are unchanged.
+     */
     @Throws(ZstdException::class)
-    public fun compress(data: ByteArray, dictionary: ZstdDictionary, level: Int = DEFAULT_LEVEL): ByteArray =
-        wrapFailures("compression", data.size) {
-            PureZstdEncoder.encode(data, dictionary.parsed, dictionary.matchIndex, level)
-        }
+    public fun compress(
+        data: ByteArray,
+        dictionary: ZstdDictionary,
+        level: Int = DEFAULT_LEVEL,
+        checksum: Boolean = false,
+    ): ByteArray = wrapFailures("compression", data.size) {
+        PureZstdEncoder.encode(data, dictionary.parsed, dictionary.matchIndex, level, checksum)
+    }
 
-    /** Compress [data] into a standard zstd frame with no dictionary. */
+    /**
+     * Compress [data] into a standard zstd frame with no dictionary. See
+     * [compress] (dictionary overload) for [checksum].
+     */
     @Throws(ZstdException::class)
-    public fun compress(data: ByteArray, level: Int = DEFAULT_LEVEL): ByteArray =
-        compress(data, ZstdDictionary.EMPTY, level)
+    public fun compress(data: ByteArray, level: Int = DEFAULT_LEVEL, checksum: Boolean = false): ByteArray =
+        compress(data, ZstdDictionary.EMPTY, level, checksum)
 
     /**
      * Decompress a standard zstd [frame] using [dictionary], rejecting output

src/commonMain/kotlin/org/meshtastic/kzstd/internal/Xxh64.kt

@@ -0,0 +1,108 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+package org.meshtastic.kzstd.internal
+
+/**
+ * Pure-Kotlin XXH64 (64-bit xxHash), the algorithm RFC 8878 §3.1.1 mandates
+ * for a zstd frame's optional Content_Checksum: "the least significant 32
+ * bits of the XXH64() checksum of the original decompressed data, seed 0".
+ *
+ * This is a standalone, common-Kotlin port of the reference XXH64 algorithm
+ * (no external dependency — kzstd is zero-runtime-dependency). It operates on
+ * a whole [ByteArray] (or a `[offset, offset+length)` slice), matching how the
+ * decoder needs it: hash the fully-decoded frame content once, after the
+ * block loop completes.
+ */
+internal object Xxh64 {
+
+    private const val PRIME1 = -0x61c8864e7a143579L // 0x9E3779B185EBCA87
+    private const val PRIME2 = -0x3d4d51c2d82b14b1L // 0xC2B2AE3D27D4EB4F
+    private const val PRIME3 = 0x165667B19E3779F9L
+    private const val PRIME4 = -0x7a1435883d4d519dL // 0x85EBCA77C2B2AE63
+    private const val PRIME5 = 0x27D4EB2F165667C5L
+
+    /** Hash the full [data] array with [seed] (RFC 8878 uses seed 0). */
+    fun hash(data: ByteArray, seed: Long = 0L): Long = hash(data, 0, data.size, seed)
+
+    /** Hash `data[offset until offset+length]` with [seed]. */
+    fun hash(data: ByteArray, offset: Int, length: Int, seed: Long = 0L): Long {
+        var i = offset
+        val end = offset + length
+        var h64: Long
+
+        if (length >= 32) {
+            var v1 = seed + PRIME1 + PRIME2
+            var v2 = seed + PRIME2
+            var v3 = seed
+            var v4 = seed - PRIME1
+            val limit = end - 32
+            while (i <= limit) {
+                v1 = round(v1, readLE64(data, i))
+                v2 = round(v2, readLE64(data, i + 8))
+                v3 = round(v3, readLE64(data, i + 16))
+                v4 = round(v4, readLE64(data, i + 24))
+                i += 32
+            }
+            h64 = rotl(v1, 1) + rotl(v2, 7) + rotl(v3, 12) + rotl(v4, 18)
+            h64 = mergeRound(h64, v1)
+            h64 = mergeRound(h64, v2)
+            h64 = mergeRound(h64, v3)
+            h64 = mergeRound(h64, v4)
+        } else {
+            h64 = seed + PRIME5
+        }
+
+        h64 += length.toLong()
+
+        while (i + 8 <= end) {
+            val k1 = round(0L, readLE64(data, i))
+            h64 = h64 xor k1
+            h64 = rotl(h64, 27) * PRIME1 + PRIME4
+            i += 8
+        }
+
+        if (i + 4 <= end) {
+            h64 = h64 xor ((readLE32(data, i)) * PRIME1)
+            h64 = rotl(h64, 23) * PRIME2 + PRIME3
+            i += 4
+        }
+
+        while (i < end) {
+            h64 = h64 xor ((data[i].toLong() and 0xFF) * PRIME5)
+            h64 = rotl(h64, 11) * PRIME1
+            i++
+        }
+
+        h64 = h64 xor (h64 ushr 33)
+        h64 *= PRIME2
+        h64 = h64 xor (h64 ushr 29)
+        h64 *= PRIME3
+        h64 = h64 xor (h64 ushr 32)
+        return h64
+    }
+
+    private fun round(acc: Long, input: Long): Long {
+        var a = acc + input * PRIME2
+        a = rotl(a, 31)
+        a *= PRIME1
+        return a
+    }
+
+    private fun mergeRound(acc: Long, v: Long): Long {
+        val merged = acc xor round(0L, v)
+        return merged * PRIME1 + PRIME4
+    }
+
+    private fun rotl(x: Long, r: Int): Long = (x shl r) or (x ushr (64 - r))
+
+    private fun readLE64(b: ByteArray, off: Int): Long {
+        var v = 0L
+        for (i in 0 until 8) v = v or ((b[off + i].toLong() and 0xFF) shl (8 * i))
+        return v
+    }
+
+    private fun readLE32(b: ByteArray, off: Int): Long {
+        var v = 0L
+        for (i in 0 until 4) v = v or ((b[off + i].toLong() and 0xFF) shl (8 * i))
+        return v
+    }
+}

src/commonMain/kotlin/org/meshtastic/kzstd/internal/ZstdDecoder.kt

@@ -12,8 +12,11 @@ import org.meshtastic.kzstd.ZstdException
  * frames exercise — and, defensively, the full block/literals/sequences grammar
  * so it stays correct for any conformant small frame:
  *
- *  - Frame header with optional content-size / dictionary-id / checksum (all
- *    OFF in our frames, but parsed if present).
+ *  - Frame header with optional content-size / dictionary-id / checksum
+ *    (content-size and dictionary-id are OFF in our own frames but parsed —
+ *    and, for dictionary-id, validated against the supplied dictionary — if
+ *    present; checksum, when present in ANY frame including real libzstd's,
+ *    is validated against the decoded content — see [decode]).
  *  - Block loop: Raw, RLE, and Compressed blocks; the Last_Block flag.
  *  - Literals section: Raw, RLE, Compressed (fresh Huffman), and Treeless
  *    (repeat the dictionary's / previous block's Huffman table); 1-stream and
@@ -53,8 +56,8 @@ internal object PureZstdDecoder {
         val magic = reader.readLEInt(4)
         if (magic != FRAME_MAGIC) throw ZstdException("bad frame magic ${magic.toString(16)}")
 
-        val frameDictId = parseFrameHeader(reader) // window/content-size/dict-id/checksum descriptors
-        checkDictionaryId(frameDictId, dict.dictionaryId)
+        val frameHeader = parseFrameHeader(reader) // window/content-size/dict-id/checksum descriptors
+        checkDictionaryId(frameHeader.dictionaryId, dict.dictionaryId)
 
         // Output buffer prefixed (conceptually) by the dictionary content: a
         // match offset that reaches before the frame start indexes into the
@@ -98,7 +101,28 @@ internal object PureZstdDecoder {
             if (lastBlock) break
         }
 
-        return out.frameOutput()
+        val output = out.frameOutput()
+
+        // RFC 8878 §3.1.1: when Content_Checksum_Flag is set, a trailing 4-byte
+        // little-endian field holds the low 32 bits of XXH64(seed=0) over the
+        // FULL decoded content. Validate it here against ANY conformant frame
+        // (not just kzstd's own — real libzstd sets this by default), so
+        // corrupted content is caught rather than silently returned.
+        if (frameHeader.contentChecksumFlag) validateContentChecksum(reader, output)
+
+        return output
+    }
+
+    /** Reads the trailing 4-byte checksum and throws if it doesn't match [output]. */
+    private fun validateContentChecksum(reader: ForwardByteReader, output: ByteArray) {
+        val storedChecksum = reader.readLEInt(4)
+        val computedChecksum = (Xxh64.hash(output) and 0xFFFFFFFFL).toInt()
+        if (storedChecksum != computedChecksum) {
+            throw ZstdException(
+                "content checksum mismatch: frame has ${storedChecksum.toUInt()}, " +
+                    "computed ${computedChecksum.toUInt()}",
+            )
+        }
     }
 
     /**
@@ -120,11 +144,20 @@ internal object PureZstdDecoder {
     // ── Frame header ─────────────────────────────────────────────────────────
 
     /**
-     * Parses the frame header and returns its declared Dictionary_ID, or `0` if
-     * the field is absent (Dictionary_ID_Flag == 0) -- `0` also doubles as RFC
-     * 8878's own "no Dictionary_ID" sentinel, so callers can treat it uniformly.
+     * The fields [parseFrameHeader] extracts that later decode steps need. A
+     * struct rather than separate return values so a future field (e.g. an
+     * eventual Frame_Content_Size check) doesn't force every caller and every
+     * other in-flight header field to renegotiate a shared return type again.
      */
-    private fun parseFrameHeader(reader: ForwardByteReader): Int {
+    private class FrameHeader(
+        /** Declared Dictionary_ID, or `0` if absent (Dictionary_ID_Flag == 0) --
+         *  `0` also doubles as RFC 8878's own "no Dictionary_ID" sentinel. */
+        val dictionaryId: Int,
+        /** Whether Content_Checksum_Flag is set. */
+        val contentChecksumFlag: Boolean,
+    )
+
+    private fun parseFrameHeader(reader: ForwardByteReader): FrameHeader {
         val descriptor = reader.readByte()
         val frameContentSizeFlag = (descriptor ushr 6) and 0x3
         val singleSegment = (descriptor ushr 5) and 0x1
@@ -155,12 +188,11 @@ internal object PureZstdDecoder {
         }
         if (fcsBytes > 0) reader.readLELong(fcsBytes)
 
-        // Content checksum (if any) is a trailing 4-byte field consumed by the
-        // block loop's end; our frames set it OFF. Record nothing here.
-        @Suppress("UNUSED_EXPRESSION")
-        contentChecksum
-
-        return frameDictId
+        // Content checksum (if any) is a trailing 4-byte field AFTER the block
+        // loop's end; the caller reads and validates it once the full content
+        // is decoded (a running hash can't be computed until then anyway,
+        // since blocks may repeat-copy from bytes not yet finalised as output).
+        return FrameHeader(dictionaryId = frameDictId, contentChecksumFlag = contentChecksum == 1)
     }
 
     // ── Compressed block ───────────────────────────────────────────────────────

src/commonMain/kotlin/org/meshtastic/kzstd/internal/ZstdEncoder.kt

@@ -73,8 +73,19 @@ internal object PureZstdEncoder {
      * distinct strategies/parameters (windowLog, targetLength, etc.); this
      * encoder uses a single fixed greedy/lazy strategy at every level, just
      * with a shallower or deeper search.
+     *
+     * [checksum], when true, sets Content_Checksum_Flag and appends the
+     * trailing 4-byte XXH64(seed=0) low-32-bits checksum of [data] (RFC 8878
+     * §3.1.1). Defaults to false so every existing call site's wire output is
+     * byte-for-byte unchanged (see [ByteIdenticalRegressionTest]).
      */
-    fun encode(data: ByteArray, dict: ParsedDictionary, index: MatchIndex, level: Int = 19): ByteArray {
+    fun encode(
+        data: ByteArray,
+        dict: ParsedDictionary,
+        index: MatchIndex,
+        level: Int = 19,
+        checksum: Boolean = false,
+    ): ByteArray {
         if (data.size > MAX_BLOCK_SIZE) {
             throw ZstdException(
                 "input ${data.size} exceeds the single-block limit of $MAX_BLOCK_SIZE bytes; " +
@@ -89,9 +100,9 @@ internal object PureZstdEncoder {
         // Encode the single block. If it does not beat raw, fall back to a Raw
         // block (still a valid frame).
         val compressedBlock = encodeCompressedBlock(program, data, dict)
-        val out = ArrayList<Byte>(data.size + 16)
+        val out = ArrayList<Byte>(data.size + 20)
         FRAME_MAGIC.forEach { out.add(it) }
-        out.add(frameHeaderDescriptor())
+        out.add(frameHeaderDescriptor(checksum))
         out.add(windowDescriptor(dict.content.size + data.size))
 
         if (compressedBlock != null && compressedBlock.size < data.size) {
@@ -103,18 +114,27 @@ internal object PureZstdEncoder {
             writeBlockHeader(out, lastBlock = true, blockType = 0, blockSize = data.size)
             data.forEach { out.add(it) }
         }
+
+        if (checksum) {
+            val h = Xxh64.hash(data) and 0xFFFFFFFFL
+            out.add((h and 0xFF).toByte())
+            out.add(((h ushr 8) and 0xFF).toByte())
+            out.add(((h ushr 16) and 0xFF).toByte())
+            out.add(((h ushr 24) and 0xFF).toByte())
+        }
+
         return ByteArray(out.size) { out[it] }
     }
 
     // ── Frame header ───────────────────────────────────────────────────────────
 
     /**
      * Frame_Header_Descriptor: Frame_Content_Size flag 0, Single_Segment 0,
-     * Content_Checksum 0, Dictionary_ID 0. (Single_Segment 0 means a
+     * Content_Checksum [checksum], Dictionary_ID 0. (Single_Segment 0 means a
      * Window_Descriptor byte follows, which is how the SDK's frames are shaped
      * and what its decoder skips.)
      */
-    private fun frameHeaderDescriptor(): Byte = 0
+    private fun frameHeaderDescriptor(checksum: Boolean): Byte = if (checksum) (1 shl 2).toByte() else 0
 
     /**
      * Window_Descriptor byte. The window must cover the full back-reference