change blob storage conditions and conditional ext compression

Author: wassfila

README.md

@@ -97,8 +97,9 @@ the config parameter is optional and do have default values
 * `rootdir` : defaults to current working directory. The path where to find the a `content` directory.
 * `outdir` : defaults to `.structure`. Relative output directory is the location where all output data will be generated, which is relative to the root directory.
 * `folder_single_doc` : defaults to `false`. When `true`, each folder is treated as a single document and the first YAML/YML file contributes overrides plus `meta_data` fields.
-* `blob_external_threshold_bytes` : defaults to `1_048_576` (1 MB). Blobs larger than this size are written to disk under `blobs/<YYYY>/<MM>/<prefix>/<hash>`.
-* `blob_inline_compression_min_bytes` : defaults to `4_096`. Inline blobs bigger than or equal to this size are gzip-compressed before being stored inside the `blob_store` table.
+* `external_storage_kb` : defaults to `512`. Blobs larger than this size (in KB) are written to disk under `blobs/<YYYY>/<MM>/<prefix>/<hash>`.
+* `inline_compression_kb` : defaults to `32`. Inline blobs bigger than or equal to this size are eligible for gzip compression before being stored inside the `blob_store` table.
+* `file_compress_ext` : defaults to `["txt","md","json","csv","tsv","yaml","yml"]`. Inline blobs are compressed only if their source extension (when known) appears in this list.
 
 ## Generated output
 * `gen/document_list.json`

src/blob_manager.js

@@ -1,14 +1,15 @@
 import { createHash } from 'crypto';
 import { readFile, writeFile } from 'fs/promises';
-import { join } from 'path';
+import { extname, join } from 'path';
 import { gzip } from 'zlib';
 import { promisify } from 'util';
 import { check_dir_create, exists_abs } from './utils.js';
 import { get_config } from './collect.js';
 
 const gzipAsync = promisify(gzip);
-const DEFAULT_EXTERNAL_THRESHOLD_BYTES = 1024 * 1024; // 1MB
-const DEFAULT_INLINE_COMPRESSION_MIN_BYTES = 4 * 1024; // 4KB
+const DEFAULT_EXTERNAL_THRESHOLD_BYTES = 512 * 1024;
+const DEFAULT_INLINE_COMPRESSION_MIN_BYTES = 32 * 1024;
+const DEFAULT_COMPRESSIBLE_EXTENSIONS = new Set(['txt', 'md', 'json', 'csv', 'tsv', 'yaml', 'yml']);
 
 function formatMonth(month) {
     return String(month).padStart(2, '0');
@@ -41,25 +42,32 @@ class BlobManager {
     constructor(timestamp) {
         this.timestamp = timestamp ?? new Date().toISOString();
         const config = get_config();
+        const legacyExternalThreshold = config?.blob_external_threshold_bytes;
+        const legacyInlineCompression = config?.blob_inline_compression_min_bytes;
         this.externalThreshold = normalizeThreshold(
-            config?.blob_external_threshold_bytes,
+            Number.isFinite(Number(config?.external_storage_kb))
+                ? Number(config.external_storage_kb) * 1024
+                : legacyExternalThreshold,
             DEFAULT_EXTERNAL_THRESHOLD_BYTES
         );
         this.inlineCompressionMin = normalizeThreshold(
-            config?.blob_inline_compression_min_bytes,
+            Number.isFinite(Number(config?.inline_compression_kb))
+                ? Number(config.inline_compression_kb) * 1024
+                : legacyInlineCompression,
             DEFAULT_INLINE_COMPRESSION_MIN_BYTES
         );
+        this.compressibleExtensions = buildCompressibleExtensionSet(config?.file_compress_ext);
         this.hashIndex = new Map(); // hash -> {relativePath, storageDir, size, payload, compression}
     }
 
-    async ensureFromBuffer(buffer) {
+    async ensureFromBuffer(buffer, options = {}) {
         if (!buffer) {
             return null;
         }
         const hash = createHash('sha512').update(buffer).digest('hex');
         const existing = this.hashIndex.get(hash);
         if (!existing) {
-            const entry = await this.persistBuffer(buffer, hash);
+            const entry = await this.persistBuffer(buffer, hash, options);
             this.hashIndex.set(hash, entry);
             return buildResult(hash, entry);
         }
@@ -71,7 +79,8 @@ class BlobManager {
             return null;
         }
         const buffer = await readFile(absPath);
-        return this.ensureFromBuffer(buffer);
+        const compressionHint = inferCompressionHintFromPath(absPath, this.compressibleExtensions);
+        return this.ensureFromBuffer(buffer, {compressionHint});
     }
 
     async writeBlob(relativePath, buffer) {
@@ -84,7 +93,7 @@ class BlobManager {
         await writeFile(absPath, buffer);
     }
 
-    async persistBuffer(buffer, hash) {
+    async persistBuffer(buffer, hash, options = {}) {
         const size = buffer.length;
         if (size > this.externalThreshold) {
             const {year, month} = getDateParts(this.timestamp);
@@ -102,7 +111,7 @@ class BlobManager {
                 compression: null
             };
         }
-        const inlinePayload = await this.prepareInlinePayload(buffer);
+        const inlinePayload = await this.prepareInlinePayload(buffer, options);
         return {
             relativePath: null,
             storageDir: null,
@@ -112,8 +121,8 @@ class BlobManager {
         };
     }
 
-    async prepareInlinePayload(buffer) {
-        const shouldCompress = buffer.length >= this.inlineCompressionMin;
+    async prepareInlinePayload(buffer, options = {}) {
+        const shouldCompress = this.shouldCompressInlinePayload(buffer.length, options?.compressionHint);
         if (!shouldCompress) {
             return {
                 payload: Buffer.from(buffer),
@@ -126,6 +135,16 @@ class BlobManager {
             compression: true
         };
     }
+
+    shouldCompressInlinePayload(byteLength, compressionHint) {
+        if (byteLength < this.inlineCompressionMin) {
+            return false;
+        }
+        if (!compressionHint) {
+            return true;
+        }
+        return compressionHint.shouldCompress === true;
+    }
 }
 
 function normalizeThreshold(value, fallback) {
@@ -150,6 +169,33 @@ function buildResult(hash, entry) {
     };
 }
 
+function buildCompressibleExtensionSet(value) {
+    if (!value) {
+        return new Set(DEFAULT_COMPRESSIBLE_EXTENSIONS);
+    }
+    const list = Array.isArray(value) ? value : String(value).split(',');
+    const normalized = list
+        .map((entry) => String(entry ?? '').trim().replace(/^\./, '').toLowerCase())
+        .filter(Boolean);
+    if (!normalized.length) {
+        return new Set(DEFAULT_COMPRESSIBLE_EXTENSIONS);
+    }
+    return new Set(normalized);
+}
+
+function inferCompressionHintFromPath(absPath, compressibleExtensions) {
+    if (!absPath) {
+        return null;
+    }
+    const extension = extname(absPath).replace(/^\./, '').toLowerCase();
+    if (!extension) {
+        return null;
+    }
+    return {
+        shouldCompress: compressibleExtensions.has(extension)
+    };
+}
+
 function createBlobManager(timestamp) {
     return new BlobManager(timestamp);
 }

src/collect.js

@@ -13,8 +13,9 @@ let config = {
     outdir: ".structure",
     debug:false,
     folder_single_doc:false,
-    blob_external_threshold_bytes: 1024 * 1024,
-    blob_inline_compression_min_bytes: 4 * 1024
+    external_storage_kb: 512,
+    inline_compression_kb: 32,
+    file_compress_ext: ['txt','md','json','csv','tsv','yaml','yml']
 }
 
 const DOCUMENTS_TABLE_NAME = 'documents'