From bec05bc49465d929fa8be2832cbd688274313288 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Fri, 5 Dec 2025 15:04:06 -0500 Subject: [PATCH 1/7] switch to ESM --- jestconfig.json | 7 ++++++- package.json | 13 ++++++++++--- tsconfig.json | 4 ++-- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/jestconfig.json b/jestconfig.json index 20c25c0..195dc05 100644 --- a/jestconfig.json +++ b/jestconfig.json @@ -1,6 +1,11 @@ { + "preset": "ts-jest/presets/default-esm", + "extensionsToTreatAsEsm": [".ts"], + "moduleNameMapper": { + "^(\\.{1,2}/.*)\\.js$": "$1" + }, "transform": { - "^.+\\.(t|j)sx?$": "ts-jest" + "^.+\\.tsx?$": ["ts-jest", { "useESM": true }] }, "testRegex": "(/__tests__/.*|(\\.|/)(test|spec))\\.(jsx?|tsx?)$", "moduleFileExtensions": ["ts", "tsx", "js", "jsx", "json", "node"] diff --git a/package.json b/package.json index aac17fc..1193b0e 100644 --- a/package.json +++ b/package.json @@ -2,6 +2,7 @@ "name": "isbinaryfile", "description": "Detects if a file is binary in Node.js. Similar to Perl's -B.", "version": "5.0.7", + "type": "module", "keywords": [ "text", "binary", @@ -34,8 +35,14 @@ "lib/**/*" ], "license": "MIT", - "main": "lib/index.js", - "types": "lib/index.d.ts", + "main": "./lib/index.js", + "types": "./lib/index.d.ts", + "exports": { + ".": { + "types": "./lib/index.d.ts", + "default": "./lib/index.js" + } + }, "maintainers": [ { "name": "Garen J. Torikian", @@ -57,7 +64,7 @@ "preversion": "npm run lint", "version": "npm run format && git add -A src", "postversion": "git push && git push --tags", - "test": "jest --config jestconfig.json", + "test": "NODE_OPTIONS='--experimental-vm-modules' jest --config jestconfig.json", "watch": "tsc -w" } } diff --git a/tsconfig.json b/tsconfig.json index ca03321..871bc1c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,8 +1,8 @@ { "compilerOptions": { "target": "ES2022", - "module": "commonjs", - "moduleResolution": "node", + "module": "NodeNext", + "moduleResolution": "NodeNext", "declaration": true, "outDir": "./lib", "strict": true From dbf607d7b428d45c0aa4ae5069afb8e836ce9275 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Fri, 5 Dec 2025 15:04:56 -0500 Subject: [PATCH 2/7] protect against too large bounds --- src/index.ts | 5 +++++ test/index.test.ts | 24 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/src/index.ts b/src/index.ts index 6422356..ef2698d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -32,6 +32,11 @@ class Reader { } public next(len: number): number[] { + // Prevent massive array allocation by checking bounds first + if (len < 0 || len > this.size - this.offset) { + this.error = true; + return []; + } const n = new Array(); for (let i = 0; i < len; i++) { // Stop reading if an error occurred diff --git a/test/index.test.ts b/test/index.test.ts index 1741a94..137d5c3 100644 --- a/test/index.test.ts +++ b/test/index.test.ts @@ -166,6 +166,19 @@ describe('async', () => { expect(result).toBe(true); }); + it('should not crash on malformed protobuf-like data (issue #80)', async () => { + const buff = Buffer.from( + '82ACE2828045E382805FE1828053E7828045E7878045E8838145E2988445E2948545E2828D4CE2828A44E28280418CF7EC2E', + 'hex', + ); + + expect.assertions(1); + + const result = await isBinaryFile(buff); + + expect(typeof result).toBe('boolean'); + }); + it('should return false on a Vai script file', async () => { const file = path.join(FIXTURE_PATH, 'vai_script.txt'); @@ -306,6 +319,17 @@ describe('sync', () => { expect(result).toBe(false); }); + + it('should not crash on malformed protobuf-like data (issue #80)', () => { + const buff = Buffer.from( + '82ACE2828045E382805FE1828053E7828045E7878045E8838145E2988445E2948545E2828D4CE2828A44E28280418CF7EC2E', + 'hex', + ); + + const result = isBinaryFileSync(buff); + + expect(typeof result).toBe('boolean'); + }); }); it('should return false on a UTF-8 file with emoji', () => { From 44c46dd393d9f0045a63296d2fdfb26148b68920 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Fri, 5 Dec 2025 15:16:03 -0500 Subject: [PATCH 3/7] split tests --- test/{index.test.ts => async.test.ts} | 201 +++++------------------- test/sync.test.ts | 214 ++++++++++++++++++++++++++ 2 files changed, 249 insertions(+), 166 deletions(-) rename test/{index.test.ts => async.test.ts} (50%) create mode 100644 test/sync.test.ts diff --git a/test/index.test.ts b/test/async.test.ts similarity index 50% rename from test/index.test.ts rename to test/async.test.ts index 137d5c3..f08ceb0 100644 --- a/test/index.test.ts +++ b/test/async.test.ts @@ -1,4 +1,4 @@ -import { isBinaryFile, isBinaryFileSync } from '../src/index'; +import { isBinaryFile } from '../src/index'; import * as fs from 'fs'; import * as path from 'path'; @@ -188,194 +188,63 @@ describe('async', () => { expect(result).toBe(false); }); -}); - -describe('sync', () => { - it('should require size if bytes are given', () => { - const bytes = fs.readFileSync(path.join(FIXTURE_PATH, 'grep')); - - const result = isBinaryFileSync(bytes); - - expect(result).toBe(true); - }); - - it('should return true on a binary program, accepting path', () => { - const file = path.join(FIXTURE_PATH, 'grep'); - - const result = isBinaryFileSync(file); - - expect(result).toBe(true); - }); - - it('should return true on a binary program, accepting bytes & size', () => { - const bytes = fs.readFileSync(path.join(FIXTURE_PATH, 'grep')); - const size = fs.lstatSync(path.join(FIXTURE_PATH, 'grep')).size; - const result = isBinaryFileSync(bytes, size); + it('should return false for non-UTF8 files', async () => { + const encodingDir = path.join(FIXTURE_PATH, 'encodings'); + const files = fs.readdirSync(encodingDir); - expect(result).toBe(true); + for (const file of files) { + if (!/big5/.test(file) && !/gb/.test(file) && !/kr/.test(file)) { + expect(await isBinaryFile(path.join(encodingDir, file))).toBe(false); + } + } }); - it('should return false on a extensionless script, accepting path', () => { - const file = path.join(FIXTURE_PATH, 'perl_script'); - - const result = isBinaryFileSync(file); - + it('should return false on a UTF-8 file with emoji', async () => { + const file = path.join(FIXTURE_PATH, 'emoji.txt'); + const result = await isBinaryFile(file); expect(result).toBe(false); }); - it('should return false on a extensionless script, accepting bytes & size', () => { - const bytes = fs.readFileSync(path.join(FIXTURE_PATH, 'perl_script')); - const size = fs.lstatSync(path.join(FIXTURE_PATH, 'perl_script')).size; - - const result = isBinaryFileSync(bytes, size); - + it('should return false on UTF-8 file with 4-byte sequence truncated at byte 508', async () => { + const file = path.join(FIXTURE_PATH, '508A-4byte.txt'); + const result = await isBinaryFile(file); expect(result).toBe(false); }); - it('should return false on a russian text', () => { - const file = path.join(FIXTURE_PATH, 'russian_file.rst'); - - const result = isBinaryFileSync(file); - + it('should return false on UTF-8 file with 3-byte sequence truncated at byte 509', async () => { + const file = path.join(FIXTURE_PATH, '509A-3byte.txt'); + const result = await isBinaryFile(file); expect(result).toBe(false); }); - it('should return false on a zero-byte image file', () => { - const file = path.join(FIXTURE_PATH, 'null_file.gif'); - - const result = isBinaryFileSync(file); - + it('should return false on UTF-8 file with 4-byte sequence truncated at byte 509', async () => { + const file = path.join(FIXTURE_PATH, '509A-4byte.txt'); + const result = await isBinaryFile(file); expect(result).toBe(false); }); - it('should return true on a gif', () => { - const file = path.join(FIXTURE_PATH, 'trunks.gif'); - - const result = isBinaryFileSync(file); - - expect(result).toBe(true); - }); - - it('should return false on some UTF8 lua file', () => { - const file = path.join(FIXTURE_PATH, 'no.lua'); - - const result = isBinaryFileSync(file); - + it('should return false on UTF-8 file with 2-byte sequence truncated at byte 510', async () => { + const file = path.join(FIXTURE_PATH, '510A-2byte.txt'); + const result = await isBinaryFile(file); expect(result).toBe(false); }); - it('should boom on a directory', () => { - const file = path.join(FIXTURE_PATH, 'dir'); - - try { - isBinaryFileSync(file); - } catch (e: any) { - expect(e.message).toBe('Path provided was not a file!'); - } - }); - - it('should boom on non-existent file', () => { - const file = path.join(FIXTURE_PATH, 'blahblahblbahhhhhh'); - - try { - isBinaryFileSync(file); - } catch (e: any) { - expect(e.message).toBe("ENOENT: no such file or directory, stat 'test/fixtures/blahblahblbahhhhhh'"); - } - }); - - it('should return true on a PDF', () => { - const file = path.join(FIXTURE_PATH, 'pdf.pdf'); - - const result = isBinaryFileSync(file); - - expect(result).toBe(true); - }); - - it('should return true on a tricky PDF that needs a header check', () => { - const file = path.join(FIXTURE_PATH, 'test.pdf'); - - const result = isBinaryFileSync(file); - - expect(result).toBe(true); - }); - - it('should return false for non-UTF8 files', async () => { - const encodingDir = path.join(FIXTURE_PATH, 'encodings'); - const files = fs.readdirSync(encodingDir); - - files.forEach((file) => { - if (!/big5/.test(file) && !/gb/.test(file) && !/kr/.test(file)) { - expect(isBinaryFileSync(path.join(encodingDir, file))).toBe(false); - } - }); + it('should return false on UTF-8 file with 3-byte sequence truncated at byte 510', async () => { + const file = path.join(FIXTURE_PATH, '510A-3byte.txt'); + const result = await isBinaryFile(file); + expect(result).toBe(false); }); - it('should return false on a Vai script file', () => { - const file = path.join(FIXTURE_PATH, 'vai_script.txt'); - - const result = isBinaryFileSync(file); - + it('should return false on UTF-8 file with 4-byte sequence truncated at byte 510', async () => { + const file = path.join(FIXTURE_PATH, '510A-4byte.txt'); + const result = await isBinaryFile(file); expect(result).toBe(false); }); - it('should not crash on malformed protobuf-like data (issue #80)', () => { - const buff = Buffer.from( - '82ACE2828045E382805FE1828053E7828045E7878045E8838145E2988445E2948545E2828D4CE2828A44E28280418CF7EC2E', - 'hex', - ); - - const result = isBinaryFileSync(buff); - - expect(typeof result).toBe('boolean'); + it('should return false on real-world Python file with UTF-8 at boundary (utf8-boundary-truncation bug case)', async () => { + const file = path.join(FIXTURE_PATH, 'utf8-boundary-truncation_case.py'); + const result = await isBinaryFile(file); + expect(result).toBe(false); }); }); - -it('should return false on a UTF-8 file with emoji', () => { - const file = path.join(FIXTURE_PATH, 'emoji.txt'); - const result = isBinaryFileSync(file); - expect(result).toBe(false); -}); - -it('should return false on UTF-8 file with 4-byte sequence truncated at byte 508', () => { - const file = path.join(FIXTURE_PATH, '508A-4byte.txt'); - const result = isBinaryFileSync(file); - expect(result).toBe(false); -}); - -it('should return false on UTF-8 file with 3-byte sequence truncated at byte 509', () => { - const file = path.join(FIXTURE_PATH, '509A-3byte.txt'); - const result = isBinaryFileSync(file); - expect(result).toBe(false); -}); - -it('should return false on UTF-8 file with 4-byte sequence truncated at byte 509', () => { - const file = path.join(FIXTURE_PATH, '509A-4byte.txt'); - const result = isBinaryFileSync(file); - expect(result).toBe(false); -}); - -it('should return false on UTF-8 file with 2-byte sequence truncated at byte 510', () => { - const file = path.join(FIXTURE_PATH, '510A-2byte.txt'); - const result = isBinaryFileSync(file); - expect(result).toBe(false); -}); - -it('should return false on UTF-8 file with 3-byte sequence truncated at byte 510', () => { - const file = path.join(FIXTURE_PATH, '510A-3byte.txt'); - const result = isBinaryFileSync(file); - expect(result).toBe(false); -}); - -it('should return false on UTF-8 file with 4-byte sequence truncated at byte 510', () => { - const file = path.join(FIXTURE_PATH, '510A-4byte.txt'); - const result = isBinaryFileSync(file); - expect(result).toBe(false); -}); - -it('should return false on real-world Python file with UTF-8 at boundary (utf8-boundary-truncation bug case)', () => { - const file = path.join(FIXTURE_PATH, 'utf8-boundary-truncation_case.py'); - const result = isBinaryFileSync(file); - expect(result).toBe(false); -}); diff --git a/test/sync.test.ts b/test/sync.test.ts new file mode 100644 index 0000000..2cfa41d --- /dev/null +++ b/test/sync.test.ts @@ -0,0 +1,214 @@ +import { isBinaryFileSync } from '../src/index'; + +import * as fs from 'fs'; +import * as path from 'path'; + +const FIXTURE_PATH = './test/fixtures'; + +describe('sync', () => { + it('should require size if bytes are given', () => { + const bytes = fs.readFileSync(path.join(FIXTURE_PATH, 'grep')); + + const result = isBinaryFileSync(bytes); + + expect(result).toBe(true); + }); + + it('should return true on a binary program, accepting path', () => { + const file = path.join(FIXTURE_PATH, 'grep'); + + const result = isBinaryFileSync(file); + + expect(result).toBe(true); + }); + + it('should return true on a binary program, accepting bytes & size', () => { + const bytes = fs.readFileSync(path.join(FIXTURE_PATH, 'grep')); + const size = fs.lstatSync(path.join(FIXTURE_PATH, 'grep')).size; + + const result = isBinaryFileSync(bytes, size); + + expect(result).toBe(true); + }); + + it('should return false on a extensionless script, accepting path', () => { + const file = path.join(FIXTURE_PATH, 'perl_script'); + + const result = isBinaryFileSync(file); + + expect(result).toBe(false); + }); + + it('should return false on a extensionless script, accepting bytes & size', () => { + const bytes = fs.readFileSync(path.join(FIXTURE_PATH, 'perl_script')); + const size = fs.lstatSync(path.join(FIXTURE_PATH, 'perl_script')).size; + + const result = isBinaryFileSync(bytes, size); + + expect(result).toBe(false); + }); + + it('should return false on a russian text', () => { + const file = path.join(FIXTURE_PATH, 'russian_file.rst'); + + const result = isBinaryFileSync(file); + + expect(result).toBe(false); + }); + + it('should return false on a zero-byte image file', () => { + const file = path.join(FIXTURE_PATH, 'null_file.gif'); + + const result = isBinaryFileSync(file); + + expect(result).toBe(false); + }); + + it('should return true on a gif', () => { + const file = path.join(FIXTURE_PATH, 'trunks.gif'); + + const result = isBinaryFileSync(file); + + expect(result).toBe(true); + }); + + it('should return false on some UTF8 lua file', () => { + const file = path.join(FIXTURE_PATH, 'no.lua'); + + const result = isBinaryFileSync(file); + + expect(result).toBe(false); + }); + + it('should boom on a directory', () => { + const file = path.join(FIXTURE_PATH, 'dir'); + + try { + isBinaryFileSync(file); + } catch (e: any) { + expect(e.message).toBe('Path provided was not a file!'); + } + }); + + it('should boom on non-existent file', () => { + const file = path.join(FIXTURE_PATH, 'blahblahblbahhhhhh'); + + try { + isBinaryFileSync(file); + } catch (e: any) { + expect(e.message).toBe("ENOENT: no such file or directory, stat 'test/fixtures/blahblahblbahhhhhh'"); + } + }); + + it('should return true on a PDF', () => { + const file = path.join(FIXTURE_PATH, 'pdf.pdf'); + + const result = isBinaryFileSync(file); + + expect(result).toBe(true); + }); + + it('should return true on a tricky PDF that needs a header check', () => { + const file = path.join(FIXTURE_PATH, 'test.pdf'); + + const result = isBinaryFileSync(file); + + expect(result).toBe(true); + }); + + it('should return false for non-UTF8 files', () => { + const encodingDir = path.join(FIXTURE_PATH, 'encodings'); + const files = fs.readdirSync(encodingDir); + + files.forEach((file) => { + if (!/big5/.test(file) && !/gb/.test(file) && !/kr/.test(file)) { + expect(isBinaryFileSync(path.join(encodingDir, file))).toBe(false); + } + }); + }); + + it('should return false on a protobuf.proto', () => { + const file = path.join(FIXTURE_PATH, 'protobuf.proto'); + const result = isBinaryFileSync(file); + expect(result).toBe(false); + }); + + it('should return false on a protobuf.proto.txt', () => { + const file = path.join(FIXTURE_PATH, 'protobuf.proto.txt'); + const result = isBinaryFileSync(file); + expect(result).toBe(false); + }); + + it('should return true on a protobuf.proto.bin', () => { + const file = path.join(FIXTURE_PATH, 'protobuf.proto.bin'); + const result = isBinaryFileSync(file); + expect(result).toBe(true); + }); + + it('should return false on a Vai script file', () => { + const file = path.join(FIXTURE_PATH, 'vai_script.txt'); + + const result = isBinaryFileSync(file); + + expect(result).toBe(false); + }); + + it('should not crash on malformed protobuf-like data (issue #80)', () => { + const buff = Buffer.from( + '82ACE2828045E382805FE1828053E7828045E7878045E8838145E2988445E2948545E2828D4CE2828A44E28280418CF7EC2E', + 'hex', + ); + + const result = isBinaryFileSync(buff); + + expect(typeof result).toBe('boolean'); + }); + + it('should return false on a UTF-8 file with emoji', () => { + const file = path.join(FIXTURE_PATH, 'emoji.txt'); + const result = isBinaryFileSync(file); + expect(result).toBe(false); + }); + + it('should return false on UTF-8 file with 4-byte sequence truncated at byte 508', () => { + const file = path.join(FIXTURE_PATH, '508A-4byte.txt'); + const result = isBinaryFileSync(file); + expect(result).toBe(false); + }); + + it('should return false on UTF-8 file with 3-byte sequence truncated at byte 509', () => { + const file = path.join(FIXTURE_PATH, '509A-3byte.txt'); + const result = isBinaryFileSync(file); + expect(result).toBe(false); + }); + + it('should return false on UTF-8 file with 4-byte sequence truncated at byte 509', () => { + const file = path.join(FIXTURE_PATH, '509A-4byte.txt'); + const result = isBinaryFileSync(file); + expect(result).toBe(false); + }); + + it('should return false on UTF-8 file with 2-byte sequence truncated at byte 510', () => { + const file = path.join(FIXTURE_PATH, '510A-2byte.txt'); + const result = isBinaryFileSync(file); + expect(result).toBe(false); + }); + + it('should return false on UTF-8 file with 3-byte sequence truncated at byte 510', () => { + const file = path.join(FIXTURE_PATH, '510A-3byte.txt'); + const result = isBinaryFileSync(file); + expect(result).toBe(false); + }); + + it('should return false on UTF-8 file with 4-byte sequence truncated at byte 510', () => { + const file = path.join(FIXTURE_PATH, '510A-4byte.txt'); + const result = isBinaryFileSync(file); + expect(result).toBe(false); + }); + + it('should return false on real-world Python file with UTF-8 at boundary (utf8-boundary-truncation bug case)', () => { + const file = path.join(FIXTURE_PATH, 'utf8-boundary-truncation_case.py'); + const result = isBinaryFileSync(file); + expect(result).toBe(false); + }); +}); From c9d4948387346dc54c8ff32c107e0a03da676ea5 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Fri, 5 Dec 2025 15:23:15 -0500 Subject: [PATCH 4/7] update release workflows --- .github/workflows/publish.yml | 2 +- .github/workflows/release-please.yml | 19 +++++++++++++++ .release-please-manifest.json | 3 +++ README.md | 29 +++++++++++----------- package.json | 5 ---- release-please-config.json | 18 ++++++++++++++ release.md | 36 ++++++++++++++++++++++++++++ 7 files changed, 91 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/release-please.yml create mode 100644 .release-please-manifest.json create mode 100644 release-please-config.json create mode 100644 release.md diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 32fb37a..149ffa5 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -26,4 +26,4 @@ jobs: - run: npm ci - run: npm run build --if-present - run: npm test - - run: npm publish + - run: npm publish --provenance --access public diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml new file mode 100644 index 0000000..68501bc --- /dev/null +++ b/.github/workflows/release-please.yml @@ -0,0 +1,19 @@ +name: Release Please + +on: + push: + branches: + - main + +permissions: + contents: write + pull-requests: write + +jobs: + release-please: + runs-on: ubuntu-latest + steps: + - uses: googleapis/release-please-action@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + release-type: node diff --git a/.release-please-manifest.json b/.release-please-manifest.json new file mode 100644 index 0000000..2e0a9e1 --- /dev/null +++ b/.release-please-manifest.json @@ -0,0 +1,3 @@ +{ + ".": "5.0.7" +} diff --git a/README.md b/README.md index cd057ad..9f08a89 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # isBinaryFile Detects if a file is binary in Node.js using ✨promises✨. Similar to [Perl's `-B` switch](http://stackoverflow.com/questions/899206/how-does-perl-know-a-file-is-binary), in that: + - it reads the first few thousand bytes of a file - checks for a `null` byte; if it's found, it's binary - flags non-ASCII characters. After a certain number of "weird" characters, the file is flagged as binary @@ -21,45 +22,43 @@ Returns `Promise` (or just `boolean` for `*Sync`). `true` if the file i ### isBinaryFile(filepath) -* `filepath` - a `string` indicating the path to the file. +- `filepath` - a `string` indicating the path to the file. ### isBinaryFile(bytes[, size]) -* `bytes` - a `Buffer` of the file's contents. -* `size` - an optional `number` indicating the file size. +- `bytes` - a `Buffer` of the file's contents. +- `size` - an optional `number` indicating the file size. ### isBinaryFileSync(filepath) -* `filepath` - a `string` indicating the path to the file. - +- `filepath` - a `string` indicating the path to the file. ### isBinaryFileSync(bytes[, size]) -* `bytes` - a `Buffer` of the file's contents. -* `size` - an optional `number` indicating the file size. +- `bytes` - a `Buffer` of the file's contents. +- `size` - an optional `number` indicating the file size. ### Examples Here's an arbitrary usage: ```javascript -const isBinaryFile = require("isbinaryfile").isBinaryFile; -const fs = require("fs"); +const isBinaryFile = require('isbinaryfile').isBinaryFile; +const fs = require('fs'); -const filename = "fixtures/pdf.pdf"; +const filename = 'fixtures/pdf.pdf'; const data = fs.readFileSync(filename); const stat = fs.lstatSync(filename); isBinaryFile(data, stat.size).then((result) => { if (result) { - console.log("It is binary!") - } - else { - console.log("No it is not.") + console.log('It is binary!'); + } else { + console.log('No it is not.'); } }); -const isBinaryFileSync = require("isbinaryfile").isBinaryFileSync; +const isBinaryFileSync = require('isbinaryfile').isBinaryFileSync; const bytes = fs.readFileSync(filename); const size = fs.lstatSync(filename).size; console.log(isBinaryFileSync(bytes, size)); // true or false diff --git a/package.json b/package.json index 1193b0e..107febc 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,6 @@ "jest": "^29.7.0", "oxlint": "^0.16", "prettier": "^3", - "release-it": "^19.0.4", "ts-jest": "^29.1.4", "typescript": "^5" }, @@ -59,11 +58,7 @@ "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"", "lint": "oxlint src test", "prepare": "npm run build", - "release": "release-it", "prepublishOnly": "npm test && npm run lint", - "preversion": "npm run lint", - "version": "npm run format && git add -A src", - "postversion": "git push && git push --tags", "test": "NODE_OPTIONS='--experimental-vm-modules' jest --config jestconfig.json", "watch": "tsc -w" } diff --git a/release-please-config.json b/release-please-config.json new file mode 100644 index 0000000..f72ea36 --- /dev/null +++ b/release-please-config.json @@ -0,0 +1,18 @@ +{ + "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json", + "packages": { + ".": { + "release-type": "node", + "changelog-sections": [ + { "type": "feat", "section": "Features" }, + { "type": "fix", "section": "Bug Fixes" }, + { "type": "perf", "section": "Performance Improvements" }, + { "type": "docs", "section": "Documentation", "hidden": true }, + { "type": "chore", "section": "Miscellaneous", "hidden": true }, + { "type": "style", "hidden": true }, + { "type": "refactor", "hidden": true }, + { "type": "test", "hidden": true } + ] + } + } +} diff --git a/release.md b/release.md new file mode 100644 index 0000000..b9b3b53 --- /dev/null +++ b/release.md @@ -0,0 +1,36 @@ +## Release + +This project uses [release-please](https://github.com/googleapis/release-please) for automated releases with [conventional commits](https://www.conventionalcommits.org/). + +### Commit Message Format + +Version bumps are determined automatically from commit messages: + +| Commit Type | Example | Version Bump | +| ------------------------------ | ------------------------------- | --------------------- | +| `feat:` | `feat: add streaming detection` | Minor (5.0.0 → 5.1.0) | +| `fix:` | `fix: handle empty buffers` | Patch (5.0.0 → 5.0.1) | +| `feat!:` or `BREAKING CHANGE:` | `feat!: require Node 24` | Major (5.0.0 → 6.0.0) | +| `docs:`, `chore:`, `test:` | `docs: update examples` | No release | + +### Release Flow + +1. Create a feature branch and make changes +2. Commit using conventional commit format (e.g., `feat: add new feature`) +3. Open a PR and merge to `main` +4. **release-please** automatically creates/updates a Release PR +5. The Release PR accumulates changes and shows the pending changelog +6. When ready to release, merge the Release PR +7. A git tag and GitHub Release are created automatically +8. GitHub Actions publishes to npm + +### How the Workflows Interact + +Two GitHub Actions workflows handle releases: + +| Workflow | Trigger | Purpose | +| -------------------- | ------------------- | -------------------------------------------------------------------- | +| `release-please.yml` | Push to `main` | Creates Release PR, manages versions and changelog, creates git tags | +| `publish.yml` | Git tag `v*` pushed | Builds, tests, and publishes to npm via OIDC | + +When you merge a Release PR, `release-please.yml` creates a git tag (e.g., `v5.1.0`), which triggers `publish.yml` to publish to npm. From f0ac14a39718aaec17266f147efb9265a874e88a Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Fri, 5 Dec 2025 15:31:03 -0500 Subject: [PATCH 5/7] add regression test --- test/async.test.ts | 45 +++++++++++++++++++++++++++++++++++++++++++++ test/sync.test.ts | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+) diff --git a/test/async.test.ts b/test/async.test.ts index f08ceb0..b9c6fe1 100644 --- a/test/async.test.ts +++ b/test/async.test.ts @@ -179,6 +179,51 @@ describe('async', () => { expect(typeof result).toBe('boolean'); }); + it('should not crash on UTF-8 Chinese text buffer (issue #81)', async () => { + const chars = [ + 0xe9, 0xa2, 0x98, 0xe7, 0x9b, 0xae, 0x20, 0x20, 0x0a, 0xe2, 0x80, 0x9c, 0x37, 0x35, 0x35, 0x20, + 0xe5, 0xb9, 0xb4, 0xe6, 0x96, 0xad, 0xe8, 0xa3, 0x82, 0xe8, 0xae, 0xba, 0xe2, 0x80, 0x9d, 0xef, + 0xbc, 0x9a, 0xe4, 0xb8, 0x80, 0xe6, 0xac, 0xa1, 0xe6, 0x8a, 0x8a, 0xe4, 0xb8, 0xad, 0xe5, 0x94, + 0x90, 0xe8, 0xa7, 0x86, 0xe4, 0xb8, 0xba, 0xe4, 0xb8, 0xad, 0xe5, 0x9b, 0xbd, 0xe8, 0xbf, 0x91, + 0xe4, 0xbb, 0xa3, 0xe5, 0x8f, 0xb2, 0xe5, 0xbc, 0x80, 0xe7, 0xab, 0xaf, 0xe7, 0x9a, 0x84, 0xe7, + 0xb3, 0xbb, 0xe7, 0xbb, 0x9f, 0xe8, 0xae, 0xba, 0xe8, 0xaf, 0x81, 0x0a, 0x0a, 0xe5, 0x89, 0xaf, + 0xe6, 0xa0, 0x87, 0xe9, 0xa2, 0x98, 0x20, 0x20, 0x0a, 0xe4, 0xbb, 0x8e, 0xe8, 0xb1, 0xa1, 0xe5, + 0xbe, 0x81, 0xe7, 0xa7, 0xa9, 0xe5, 0xba, 0x8f, 0xe5, 0xb4, 0xa9, 0xe6, 0xba, 0x83, 0xe5, 0x88, + 0xb0, 0xe6, 0x99, 0x9a, 0xe6, 0x9c, 0x9f, 0xe5, 0xb8, 0x9d, 0xe5, 0x9b, 0xbd, 0xe6, 0x8b, 0x9c, + 0xe5, 0x8d, 0xa0, 0xe5, 0xba, 0xad, 0xe5, 0x8c, 0x96, 0xe7, 0x9a, 0x84, 0xe7, 0x90, 0x86, 0xe8, + 0xae, 0xba, 0xe8, 0x80, 0x83, 0xe5, 0x8f, 0xa4, 0x0a, 0x0a, 0xe6, 0x91, 0x98, 0xe8, 0xa6, 0x81, + 0x20, 0x20, 0x0a, 0xe4, 0xbb, 0xa5, 0xe2, 0x80, 0x9c, 0xe7, 0xbb, 0x88, 0xe6, 0x9e, 0x81, 0xe6, + 0x8b, 0x85, 0xe4, 0xbf, 0x9d, 0xe7, 0x9a, 0x84, 0xe8, 0x84, 0xb1, 0xe8, 0x90, 0xbd, 0xe2, 0x80, + 0x9d, 0xe4, 0xb8, 0xba, 0xe6, 0xa0, 0xb8, 0xe5, 0xbf, 0x83, 0xe5, 0x88, 0xa4, 0xe5, 0x87, 0x86, + 0xef, 0xbc, 0x8c, 0xe6, 0x9c, 0xac, 0xe6, 0x96, 0x87, 0xe6, 0x8f, 0x90, 0xe5, 0x87, 0xba, 0xef, + 0xbc, 0x9a, 0x37, 0x35, 0x35, 0x20, 0xe5, 0xb9, 0xb4, 0xe5, 0xae, 0x89, 0xe5, 0x8f, 0xb2, 0xe4, + 0xb9, 0x8b, 0xe4, 0xb9, 0xb1, 0xe5, 0xb9, 0xb6, 0xe9, 0x9d, 0x9e, 0xe4, 0xbc, 0xa0, 0xe7, 0xbb, + 0x9f, 0xe6, 0x84, 0x8f, 0xe4, 0xb9, 0x89, 0xe4, 0xb8, 0x8a, 0xe7, 0x9a, 0x84, 0xe7, 0x8e, 0x8b, + 0xe6, 0x9c, 0x9d, 0xe5, 0x8d, 0xb1, 0xe6, 0x9c, 0xba, 0xef, 0xbc, 0x8c, 0xe8, 0x80, 0x8c, 0xe6, + 0x98, 0xaf, 0xe4, 0xb8, 0xad, 0xe5, 0x9b, 0xbd, 0xe6, 0x96, 0x87, 0xe6, 0x98, 0x8e, 0xe8, 0xb1, + 0xa1, 0xe5, 0xbe, 0x81, 0xe7, 0xa7, 0xa9, 0xe5, 0xba, 0x8f, 0xe7, 0x9a, 0x84, 0xe8, 0x87, 0xaa, + 0xe6, 0x9d, 0x80, 0xe7, 0x82, 0xb9, 0xef, 0xbc, 0x9b, 0xe5, 0xae, 0x8b, 0xe2, 0x80, 0x94, 0xe6, + 0xb8, 0x85, 0x20, 0x31, 0x31, 0x30, 0x30, 0x20, 0xe4, 0xbd, 0x99, 0xe5, 0xb9, 0xb4, 0xe4, 0xb9, + 0x83, 0xe4, 0xb8, 0x80, 0xe5, 0x85, 0xb7, 0xe5, 0x88, 0xb6, 0xe5, 0xba, 0xa6, 0xe4, 0xbb, 0x8d, + 0xe8, 0xbf, 0x90, 0xe4, 0xbd, 0x9c, 0xe3, 0x80, 0x81, 0xe5, 0x8d, 0xb4, 0xe5, 0xb7, 0xb2, 0xe5, + 0xa4, 0xb1, 0xe7, 0x94, 0x9f, 0xe6, 0x88, 0x90, 0xe5, 0x8a, 0x9b, 0xe7, 0x9a, 0x84, 0xe2, 0x80, + 0x9c, 0xe6, 0xb4, 0xbb, 0xe6, 0xad, 0xbb, 0xe4, 0xba, 0xba, 0xe5, 0xb8, 0x9d, 0xe5, 0x9b, 0xbd, + 0xe2, 0x80, 0x9d, 0xe3, 0x80, 0x82, 0xe8, 0xae, 0xba, 0xe6, 0x96, 0x87, 0xe7, 0xbb, 0xbc, 0xe5, + 0x90, 0x88, 0xe5, 0x8e, 0x86, 0xe5, 0x8f, 0xb2, 0xe7, 0xa4, 0xbe, 0xe4, 0xbc, 0x9a, 0xe5, 0xad, + 0xa6, 0xe3, 0x80, 0x81, 0xe7, 0xb2, 0xbe, 0xe7, 0xa5, 0x9e, 0xe5, 0x88, 0x86, 0xe6, 0x9e, 0x90, + 0xe4, 0xba, 0xba, 0xe7, 0xb1, 0xbb, 0xe5, 0xad, 0xa6, 0xe3, 0x80, 0x81, 0xe6, 0x94, 0xbf, 0xe6, + 0xb2, 0xbb, 0xe7, 0xa5, 0x9e, 0xe5, 0xad, 0xa6, 0xe4, 0xb8, 0x8e, 0xe6, 0x96, 0xb0, 0xe5, 0x88, + 0xb6, 0xe5, 0xba, 0xa6, 0xe4, 0xb8, 0xbb, 0xe4, + ]; + const buff = Buffer.from(chars); + + expect.assertions(1); + + const result = await isBinaryFile(buff); + + expect(result).toBe(false); + }); + it('should return false on a Vai script file', async () => { const file = path.join(FIXTURE_PATH, 'vai_script.txt'); diff --git a/test/sync.test.ts b/test/sync.test.ts index 2cfa41d..65351d1 100644 --- a/test/sync.test.ts +++ b/test/sync.test.ts @@ -164,6 +164,49 @@ describe('sync', () => { expect(typeof result).toBe('boolean'); }); + it('should not crash on UTF-8 Chinese text buffer (issue #81)', () => { + const chars = [ + 0xe9, 0xa2, 0x98, 0xe7, 0x9b, 0xae, 0x20, 0x20, 0x0a, 0xe2, 0x80, 0x9c, 0x37, 0x35, 0x35, 0x20, + 0xe5, 0xb9, 0xb4, 0xe6, 0x96, 0xad, 0xe8, 0xa3, 0x82, 0xe8, 0xae, 0xba, 0xe2, 0x80, 0x9d, 0xef, + 0xbc, 0x9a, 0xe4, 0xb8, 0x80, 0xe6, 0xac, 0xa1, 0xe6, 0x8a, 0x8a, 0xe4, 0xb8, 0xad, 0xe5, 0x94, + 0x90, 0xe8, 0xa7, 0x86, 0xe4, 0xb8, 0xba, 0xe4, 0xb8, 0xad, 0xe5, 0x9b, 0xbd, 0xe8, 0xbf, 0x91, + 0xe4, 0xbb, 0xa3, 0xe5, 0x8f, 0xb2, 0xe5, 0xbc, 0x80, 0xe7, 0xab, 0xaf, 0xe7, 0x9a, 0x84, 0xe7, + 0xb3, 0xbb, 0xe7, 0xbb, 0x9f, 0xe8, 0xae, 0xba, 0xe8, 0xaf, 0x81, 0x0a, 0x0a, 0xe5, 0x89, 0xaf, + 0xe6, 0xa0, 0x87, 0xe9, 0xa2, 0x98, 0x20, 0x20, 0x0a, 0xe4, 0xbb, 0x8e, 0xe8, 0xb1, 0xa1, 0xe5, + 0xbe, 0x81, 0xe7, 0xa7, 0xa9, 0xe5, 0xba, 0x8f, 0xe5, 0xb4, 0xa9, 0xe6, 0xba, 0x83, 0xe5, 0x88, + 0xb0, 0xe6, 0x99, 0x9a, 0xe6, 0x9c, 0x9f, 0xe5, 0xb8, 0x9d, 0xe5, 0x9b, 0xbd, 0xe6, 0x8b, 0x9c, + 0xe5, 0x8d, 0xa0, 0xe5, 0xba, 0xad, 0xe5, 0x8c, 0x96, 0xe7, 0x9a, 0x84, 0xe7, 0x90, 0x86, 0xe8, + 0xae, 0xba, 0xe8, 0x80, 0x83, 0xe5, 0x8f, 0xa4, 0x0a, 0x0a, 0xe6, 0x91, 0x98, 0xe8, 0xa6, 0x81, + 0x20, 0x20, 0x0a, 0xe4, 0xbb, 0xa5, 0xe2, 0x80, 0x9c, 0xe7, 0xbb, 0x88, 0xe6, 0x9e, 0x81, 0xe6, + 0x8b, 0x85, 0xe4, 0xbf, 0x9d, 0xe7, 0x9a, 0x84, 0xe8, 0x84, 0xb1, 0xe8, 0x90, 0xbd, 0xe2, 0x80, + 0x9d, 0xe4, 0xb8, 0xba, 0xe6, 0xa0, 0xb8, 0xe5, 0xbf, 0x83, 0xe5, 0x88, 0xa4, 0xe5, 0x87, 0x86, + 0xef, 0xbc, 0x8c, 0xe6, 0x9c, 0xac, 0xe6, 0x96, 0x87, 0xe6, 0x8f, 0x90, 0xe5, 0x87, 0xba, 0xef, + 0xbc, 0x9a, 0x37, 0x35, 0x35, 0x20, 0xe5, 0xb9, 0xb4, 0xe5, 0xae, 0x89, 0xe5, 0x8f, 0xb2, 0xe4, + 0xb9, 0x8b, 0xe4, 0xb9, 0xb1, 0xe5, 0xb9, 0xb6, 0xe9, 0x9d, 0x9e, 0xe4, 0xbc, 0xa0, 0xe7, 0xbb, + 0x9f, 0xe6, 0x84, 0x8f, 0xe4, 0xb9, 0x89, 0xe4, 0xb8, 0x8a, 0xe7, 0x9a, 0x84, 0xe7, 0x8e, 0x8b, + 0xe6, 0x9c, 0x9d, 0xe5, 0x8d, 0xb1, 0xe6, 0x9c, 0xba, 0xef, 0xbc, 0x8c, 0xe8, 0x80, 0x8c, 0xe6, + 0x98, 0xaf, 0xe4, 0xb8, 0xad, 0xe5, 0x9b, 0xbd, 0xe6, 0x96, 0x87, 0xe6, 0x98, 0x8e, 0xe8, 0xb1, + 0xa1, 0xe5, 0xbe, 0x81, 0xe7, 0xa7, 0xa9, 0xe5, 0xba, 0x8f, 0xe7, 0x9a, 0x84, 0xe8, 0x87, 0xaa, + 0xe6, 0x9d, 0x80, 0xe7, 0x82, 0xb9, 0xef, 0xbc, 0x9b, 0xe5, 0xae, 0x8b, 0xe2, 0x80, 0x94, 0xe6, + 0xb8, 0x85, 0x20, 0x31, 0x31, 0x30, 0x30, 0x20, 0xe4, 0xbd, 0x99, 0xe5, 0xb9, 0xb4, 0xe4, 0xb9, + 0x83, 0xe4, 0xb8, 0x80, 0xe5, 0x85, 0xb7, 0xe5, 0x88, 0xb6, 0xe5, 0xba, 0xa6, 0xe4, 0xbb, 0x8d, + 0xe8, 0xbf, 0x90, 0xe4, 0xbd, 0x9c, 0xe3, 0x80, 0x81, 0xe5, 0x8d, 0xb4, 0xe5, 0xb7, 0xb2, 0xe5, + 0xa4, 0xb1, 0xe7, 0x94, 0x9f, 0xe6, 0x88, 0x90, 0xe5, 0x8a, 0x9b, 0xe7, 0x9a, 0x84, 0xe2, 0x80, + 0x9c, 0xe6, 0xb4, 0xbb, 0xe6, 0xad, 0xbb, 0xe4, 0xba, 0xba, 0xe5, 0xb8, 0x9d, 0xe5, 0x9b, 0xbd, + 0xe2, 0x80, 0x9d, 0xe3, 0x80, 0x82, 0xe8, 0xae, 0xba, 0xe6, 0x96, 0x87, 0xe7, 0xbb, 0xbc, 0xe5, + 0x90, 0x88, 0xe5, 0x8e, 0x86, 0xe5, 0x8f, 0xb2, 0xe7, 0xa4, 0xbe, 0xe4, 0xbc, 0x9a, 0xe5, 0xad, + 0xa6, 0xe3, 0x80, 0x81, 0xe7, 0xb2, 0xbe, 0xe7, 0xa5, 0x9e, 0xe5, 0x88, 0x86, 0xe6, 0x9e, 0x90, + 0xe4, 0xba, 0xba, 0xe7, 0xb1, 0xbb, 0xe5, 0xad, 0xa6, 0xe3, 0x80, 0x81, 0xe6, 0x94, 0xbf, 0xe6, + 0xb2, 0xbb, 0xe7, 0xa5, 0x9e, 0xe5, 0xad, 0xa6, 0xe4, 0xb8, 0x8e, 0xe6, 0x96, 0xb0, 0xe5, 0x88, + 0xb6, 0xe5, 0xba, 0xa6, 0xe4, 0xb8, 0xbb, 0xe4, + ]; + const buff = Buffer.from(chars); + + const result = isBinaryFileSync(buff); + + expect(result).toBe(false); + }); + it('should return false on a UTF-8 file with emoji', () => { const file = path.join(FIXTURE_PATH, 'emoji.txt'); const result = isBinaryFileSync(file); From 7171398fa2ff2c09b3a2cc4257120e0d77f43fde Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Fri, 5 Dec 2025 15:58:33 -0500 Subject: [PATCH 6/7] feat!: add encoding hints --- README.md | 99 +++++++++++----- src/encoding.ts | 127 +++++++++++++++++++++ src/index.ts | 66 +++++++---- test/async.test.ts | 6 +- test/encoding-hints.test.ts | 114 ++++++++++++++++++ test/fixtures/encodings/utf16be-no-bom.txt | Bin 0 -> 106 bytes test/fixtures/encodings/utf16le-no-bom.txt | Bin 0 -> 106 bytes test/sync.test.ts | 6 +- 8 files changed, 368 insertions(+), 50 deletions(-) create mode 100644 src/encoding.ts create mode 100644 test/encoding-hints.test.ts create mode 100644 test/fixtures/encodings/utf16be-no-bom.txt create mode 100644 test/fixtures/encodings/utf16le-no-bom.txt diff --git a/README.md b/README.md index 9f08a89..e17c24e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # isBinaryFile -Detects if a file is binary in Node.js using ✨promises✨. Similar to [Perl's `-B` switch](http://stackoverflow.com/questions/899206/how-does-perl-know-a-file-is-binary), in that: +Detects if a file is binary in Node.js. Similar to [Perl's `-B` switch](http://stackoverflow.com/questions/899206/how-does-perl-know-a-file-is-binary), in that: - it reads the first few thousand bytes of a file - checks for a `null` byte; if it's found, it's binary @@ -20,50 +20,97 @@ npm install isbinaryfile Returns `Promise` (or just `boolean` for `*Sync`). `true` if the file is binary, `false` otherwise. -### isBinaryFile(filepath) +### isBinaryFile(filepath[, options]) - `filepath` - a `string` indicating the path to the file. +- `options` - an optional object with the following properties: + - `encoding` - an encoding hint (see [Encoding Hints](#encoding-hints) below) -### isBinaryFile(bytes[, size]) +### isBinaryFile(bytes[, options]) - `bytes` - a `Buffer` of the file's contents. -- `size` - an optional `number` indicating the file size. +- `options` - an optional object with the following properties: + - `size` - the size of the buffer (defaults to `bytes.length`) + - `encoding` - an encoding hint (see [Encoding Hints](#encoding-hints) below) -### isBinaryFileSync(filepath) +### isBinaryFileSync(filepath[, options]) -- `filepath` - a `string` indicating the path to the file. +Synchronous version of `isBinaryFile`. -### isBinaryFileSync(bytes[, size]) +### isBinaryFileSync(bytes[, options]) -- `bytes` - a `Buffer` of the file's contents. -- `size` - an optional `number` indicating the file size. +Synchronous version of `isBinaryFile` for buffers. ### Examples Here's an arbitrary usage: ```javascript -const isBinaryFile = require('isbinaryfile').isBinaryFile; -const fs = require('fs'); +import { isBinaryFile, isBinaryFileSync } from 'isbinaryfile'; +import fs from 'fs'; const filename = 'fixtures/pdf.pdf'; -const data = fs.readFileSync(filename); -const stat = fs.lstatSync(filename); - -isBinaryFile(data, stat.size).then((result) => { - if (result) { - console.log('It is binary!'); - } else { - console.log('No it is not.'); - } -}); - -const isBinaryFileSync = require('isbinaryfile').isBinaryFileSync; + +// Async with file path +const result = await isBinaryFile(filename); +if (result) { + console.log('It is binary!'); +} else { + console.log('No it is not.'); +} + +// Sync with buffer const bytes = fs.readFileSync(filename); -const size = fs.lstatSync(filename).size; -console.log(isBinaryFileSync(bytes, size)); // true or false +console.log(isBinaryFileSync(bytes)); // true or false + +// With explicit size option +const partialBuffer = Buffer.alloc(100); +fs.readSync(fs.openSync(filename, 'r'), partialBuffer, 0, 100, 0); +console.log(isBinaryFileSync(partialBuffer, { size: 100 })); +``` + +### Encoding Hints + +For files that use non-UTF-8 encodings, you can provide encoding hints to improve detection accuracy: + +```javascript +import { isBinaryFile, isBinaryFileSync } from 'isbinaryfile'; + +// UTF-16 files without BOM are auto-detected in most cases +const result1 = await isBinaryFile('utf16-file.txt'); + +// Or provide explicit encoding hint +const result2 = await isBinaryFile('utf16-file.txt', { encoding: 'utf-16' }); + +// ISO-8859-1 / Latin-1 encoded files +const result3 = isBinaryFileSync('german-text.txt', { encoding: 'latin1' }); + +// CJK encoded files (Big5, GB2312, EUC-KR, etc.) +const result4 = isBinaryFileSync('chinese-big5.txt', { encoding: 'big5' }); +const result5 = isBinaryFileSync('korean-text.txt', { encoding: 'euc-kr' }); + +// Generic CJK hint when exact encoding is unknown +const result6 = isBinaryFileSync('asian-text.txt', { encoding: 'cjk' }); ``` +#### Supported Encoding Hints + +| Hint | Description | +| ------------ | ------------------------------------------ | +| `utf-16` | UTF-16 (auto-detect endianness) | +| `utf-16le` | UTF-16 Little Endian | +| `utf-16be` | UTF-16 Big Endian | +| `latin1` | ISO-8859-1 / Latin-1 | +| `iso-8859-1` | Alias for latin1 | +| `cjk` | Generic CJK (use when encoding is unknown) | +| `big5` | Traditional Chinese | +| `gb2312` | Simplified Chinese | +| `gbk` | Extended GB2312 | +| `euc-kr` | Korean | +| `shift-jis` | Japanese | + +**Note:** UTF-16 without BOM is automatically detected in most cases without needing a hint. + ## Testing -Run `npm install`, then run `npm test`. +Run `npm test`. diff --git a/src/encoding.ts b/src/encoding.ts new file mode 100644 index 0000000..e8f3981 --- /dev/null +++ b/src/encoding.ts @@ -0,0 +1,127 @@ +/** + * Encoding hints to improve text detection accuracy for non-UTF-8 files. + */ +export type EncodingHint = + | 'utf-16' // UTF-16 (auto-detect endianness) + | 'utf-16le' // UTF-16 Little Endian + | 'utf-16be' // UTF-16 Big Endian + | 'latin1' // ISO-8859-1 / Latin-1 + | 'iso-8859-1' // Alias for latin1 + | 'cjk' // Generic CJK (Big5, GB2312, GBK, EUC-KR, Shift-JIS) + | 'big5' // Traditional Chinese + | 'gb2312' // Simplified Chinese + | 'gbk' // Extended GB2312 + | 'euc-kr' // Korean + | 'shift-jis'; // Japanese + +const MAX_BYTES = 512; + +/** + * Detect UTF-16 without BOM by analyzing null byte patterns. + * UTF-16LE: ASCII chars have nulls at odd positions (e.g., 't\0e\0s\0t\0') + * UTF-16BE: ASCII chars have nulls at even positions (e.g., '\0t\0e\0s\0t') + */ +export function detectUtf16NoBom(fileBuffer: Buffer, bytesRead: number): 'utf-16le' | 'utf-16be' | null { + if (bytesRead < 4) return null; + + const scanLength = Math.min(bytesRead, MAX_BYTES); + let nullsAtEven = 0; + let nullsAtOdd = 0; + + for (let i = 0; i < scanLength; i++) { + if (fileBuffer[i] === 0x00) { + if (i % 2 === 0) nullsAtEven++; + else nullsAtOdd++; + } + } + + const totalNulls = nullsAtEven + nullsAtOdd; + + // For UTF-16 ASCII text, roughly 30-70% of total bytes should be nulls + // (pure ASCII = 50%, mixed with non-ASCII = less) + if (totalNulls > scanLength * 0.3 && totalNulls < scanLength * 0.7) { + // UTF-16LE: nulls at odd positions (high bytes of little-endian chars) + if (nullsAtOdd > nullsAtEven * 3) return 'utf-16le'; + // UTF-16BE: nulls at even positions (high bytes of big-endian chars) + if (nullsAtEven > nullsAtOdd * 3) return 'utf-16be'; + } + + return null; +} + +/** + * Check if the buffer is valid text for the given encoding hint. + * Returns true if it's valid text (not binary), false if binary. + */ +export function isTextWithEncodingHint( + fileBuffer: Buffer, + bytesRead: number, + encoding: EncodingHint +): boolean { + const scanLength = Math.min(bytesRead, MAX_BYTES); + + // Handle UTF-16 hints + if (encoding === 'utf-16' || encoding === 'utf-16le' || encoding === 'utf-16be') { + // For UTF-16, null bytes are expected, so we just check for control characters + for (let i = 0; i < scanLength; i += 2) { + const byte1 = fileBuffer[i]; + const byte2 = i + 1 < scanLength ? fileBuffer[i + 1] : 0; + + // In UTF-16, check for problematic control characters + // Allow common ones: tab (0x09), newline (0x0A), carriage return (0x0D) + if (encoding === 'utf-16le' || encoding === 'utf-16') { + // LE: low byte first + if (byte2 === 0 && byte1 < 0x20 && byte1 !== 0x09 && byte1 !== 0x0A && byte1 !== 0x0D && byte1 !== 0x00) { + return false; + } + } + if (encoding === 'utf-16be' || encoding === 'utf-16') { + // BE: high byte first + if (byte1 === 0 && byte2 < 0x20 && byte2 !== 0x09 && byte2 !== 0x0A && byte2 !== 0x0D && byte2 !== 0x00) { + return false; + } + } + } + return true; + } + + // Handle Latin-1 / ISO-8859-1 hints + if (encoding === 'latin1' || encoding === 'iso-8859-1') { + for (let i = 0; i < scanLength; i++) { + const byte = fileBuffer[i]; + // Null byte is never valid in Latin-1 text files + if (byte === 0x00) return false; + // Control characters except TAB, LF, CR are suspicious + if (byte < 0x20 && byte !== 0x09 && byte !== 0x0A && byte !== 0x0D) { + return false; + } + // Bytes 0x80-0xFF are all valid Latin-1 characters, so no check needed + } + return true; + } + + // Handle CJK encodings + if ( + encoding === 'cjk' || + encoding === 'big5' || + encoding === 'gb2312' || + encoding === 'gbk' || + encoding === 'euc-kr' || + encoding === 'shift-jis' + ) { + // For CJK, we trust the user's hint and only check for definitive binary indicators + for (let i = 0; i < scanLength; i++) { + const byte = fileBuffer[i]; + // Null byte is binary indicator even in CJK + if (byte === 0x00) return false; + // Control characters (except TAB, LF, CR) suggest binary + if (byte < 0x20 && byte !== 0x09 && byte !== 0x0A && byte !== 0x0D) { + return false; + } + } + return true; + } + + // Unknown encoding, fall through to default behavior + return false; +} diff --git a/src/index.ts b/src/index.ts index ef2698d..5d8c019 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,9 +1,27 @@ -import * as fs from 'fs'; +import { statSync, openSync, readSync, closeSync, type Stats } from 'node:fs'; import { open, stat } from 'node:fs/promises'; +import { detectUtf16NoBom, isTextWithEncodingHint } from './encoding.js'; +import type { EncodingHint } from './encoding.js'; + +export { EncodingHint } from './encoding.js'; const MAX_BYTES: number = 512; const UTF8_BOUNDARY_RESERVE: number = 3; +/** + * Options for binary file detection. + */ +export interface IsBinaryOptions { + /** + * Hint about expected encoding to avoid false positives. + */ + encoding?: EncodingHint; + /** + * Size of the buffer (only used when file is a Buffer). + */ + size?: number; +} + // A very basic non-exception raising reader. Read bytes and // at the end use hasError() to check whether this worked. class Reader { @@ -114,7 +132,7 @@ function isBinaryProto(fileBuffer: Buffer, totalBytes: number): boolean { return numMessages > 0; } -export async function isBinaryFile(file: string | Buffer, size?: number): Promise { +export async function isBinaryFile(file: string | Buffer, options?: IsBinaryOptions): Promise { if (isString(file)) { const fileStat = await stat(file); isStatFile(fileStat); @@ -123,41 +141,37 @@ export async function isBinaryFile(file: string | Buffer, size?: number): Promis try { const allocBuffer = Buffer.alloc(MAX_BYTES + UTF8_BOUNDARY_RESERVE); const { bytesRead } = await fileHandle.read(allocBuffer, 0, MAX_BYTES + UTF8_BOUNDARY_RESERVE, 0); - return isBinaryCheck(allocBuffer, bytesRead); + return isBinaryCheck(allocBuffer, bytesRead, options); } finally { await fileHandle.close(); } } else { - if (size === undefined) { - size = file.length; - } - return isBinaryCheck(file, size); + const size = options?.size !== undefined ? options.size : file.length; + return isBinaryCheck(file, size, options); } } -export function isBinaryFileSync(file: string | Buffer, size?: number): boolean { +export function isBinaryFileSync(file: string | Buffer, options?: IsBinaryOptions): boolean { if (isString(file)) { - const stat = fs.statSync(file); + const fileStat = statSync(file); - isStatFile(stat); + isStatFile(fileStat); - const fileDescriptor = fs.openSync(file, 'r'); + const fileDescriptor = openSync(file, 'r'); const allocBuffer = Buffer.alloc(MAX_BYTES + UTF8_BOUNDARY_RESERVE); - const bytesRead = fs.readSync(fileDescriptor, allocBuffer, 0, MAX_BYTES + UTF8_BOUNDARY_RESERVE, 0); - fs.closeSync(fileDescriptor); + const bytesRead = readSync(fileDescriptor, allocBuffer, 0, MAX_BYTES + UTF8_BOUNDARY_RESERVE, 0); + closeSync(fileDescriptor); - return isBinaryCheck(allocBuffer, bytesRead); + return isBinaryCheck(allocBuffer, bytesRead, options); } else { - if (size === undefined) { - size = file.length; - } - return isBinaryCheck(file, size); + const size = options?.size !== undefined ? options.size : file.length; + return isBinaryCheck(file, size, options); } } -function isBinaryCheck(fileBuffer: Buffer, bytesRead: number): boolean { +function isBinaryCheck(fileBuffer: Buffer, bytesRead: number, options?: IsBinaryOptions): boolean { // empty file. no clue what it is. if (bytesRead === 0) { return false; @@ -220,6 +234,18 @@ function isBinaryCheck(fileBuffer: Buffer, bytesRead: number): boolean { return false; } + // Handle encoding hints - if provided, use specialized validation + if (options?.encoding) { + return !isTextWithEncodingHint(fileBuffer, bytesRead, options.encoding); + } + + // Auto-detect UTF-16 without BOM by analyzing null byte patterns + const utf16Detected = detectUtf16NoBom(fileBuffer, bytesRead); + if (utf16Detected) { + // Detected UTF-16 pattern, validate as text + return !isTextWithEncodingHint(fileBuffer, bytesRead, utf16Detected); + } + for (let i = 0; i < scanBytes; i++) { if (fileBuffer[i] === 0) { // NULL byte--it's binary! @@ -275,7 +301,7 @@ function isString(x: any): x is string { return typeof x === 'string'; } -function isStatFile(stat: fs.Stats): void { +function isStatFile(stat: Stats): void { if (!stat.isFile()) { throw new Error(`Path provided was not a file!`); } diff --git a/test/async.test.ts b/test/async.test.ts index b9c6fe1..8a9f794 100644 --- a/test/async.test.ts +++ b/test/async.test.ts @@ -32,7 +32,7 @@ describe('async', () => { expect.assertions(1); - const result = await isBinaryFile(bytes, size); + const result = await isBinaryFile(bytes, { size }); expect(result).toBe(true); }); @@ -53,7 +53,7 @@ describe('async', () => { expect.assertions(1); - const result = await isBinaryFile(bytes, size); + const result = await isBinaryFile(bytes, { size }); expect(result).toBe(false); }); @@ -239,6 +239,8 @@ describe('async', () => { const files = fs.readdirSync(encodingDir); for (const file of files) { + // Big5, GB, and Korean encodings require encoding hints to be detected as text. + // See encoding-hints.test.ts for tests with encoding hints. if (!/big5/.test(file) && !/gb/.test(file) && !/kr/.test(file)) { expect(await isBinaryFile(path.join(encodingDir, file))).toBe(false); } diff --git a/test/encoding-hints.test.ts b/test/encoding-hints.test.ts new file mode 100644 index 0000000..e9cdfef --- /dev/null +++ b/test/encoding-hints.test.ts @@ -0,0 +1,114 @@ +import { isBinaryFile, isBinaryFileSync } from '../src/index'; +import * as fs from 'fs'; +import * as path from 'path'; + +const FIXTURE_PATH = './test/fixtures'; +const ENCODING_PATH = path.join(FIXTURE_PATH, 'encodings'); + +describe('encoding hints', () => { + describe('options API', () => { + it('should work with no options', () => { + const result = isBinaryFileSync(path.join(FIXTURE_PATH, 'grep')); + expect(result).toBe(true); + }); + + it('should work with options object containing size', () => { + const bytes = fs.readFileSync(path.join(FIXTURE_PATH, 'grep')); + const result = isBinaryFileSync(bytes, { size: bytes.length }); + expect(result).toBe(true); + }); + }); + + describe('UTF-16 without BOM', () => { + it('should auto-detect UTF-16LE without BOM as text', () => { + const result = isBinaryFileSync(path.join(ENCODING_PATH, 'utf16le-no-bom.txt')); + expect(result).toBe(false); + }); + + it('should auto-detect UTF-16BE without BOM as text', () => { + const result = isBinaryFileSync(path.join(ENCODING_PATH, 'utf16be-no-bom.txt')); + expect(result).toBe(false); + }); + + it('should detect UTF-16LE text with utf-16le hint', () => { + const result = isBinaryFileSync(path.join(ENCODING_PATH, 'utf16le-no-bom.txt'), { encoding: 'utf-16le' }); + expect(result).toBe(false); + }); + + it('should detect UTF-16BE text with utf-16be hint', () => { + const result = isBinaryFileSync(path.join(ENCODING_PATH, 'utf16be-no-bom.txt'), { encoding: 'utf-16be' }); + expect(result).toBe(false); + }); + + it('should detect UTF-16 text with generic utf-16 hint', () => { + const result = isBinaryFileSync(path.join(ENCODING_PATH, 'utf16le-no-bom.txt'), { encoding: 'utf-16' }); + expect(result).toBe(false); + }); + }); + + describe('Latin-1 / ISO-8859-1', () => { + it('should detect test-latin.txt as text with latin1 hint', () => { + const result = isBinaryFileSync(path.join(ENCODING_PATH, 'test-latin.txt'), { encoding: 'latin1' }); + expect(result).toBe(false); + }); + + it('should detect test-latin.txt as text with iso-8859-1 hint', () => { + const result = isBinaryFileSync(path.join(ENCODING_PATH, 'test-latin.txt'), { encoding: 'iso-8859-1' }); + expect(result).toBe(false); + }); + + it('should still detect binary files as binary even with latin1 hint', () => { + const result = isBinaryFileSync(path.join(FIXTURE_PATH, 'grep'), { encoding: 'latin1' }); + expect(result).toBe(true); + }); + }); + + describe('CJK encodings', () => { + it('should detect big5.txt as text with big5 hint', () => { + const result = isBinaryFileSync(path.join(ENCODING_PATH, 'big5.txt'), { encoding: 'big5' }); + expect(result).toBe(false); + }); + + it('should detect big5_B.txt as text with big5 hint', () => { + const result = isBinaryFileSync(path.join(ENCODING_PATH, 'big5_B.txt'), { encoding: 'big5' }); + expect(result).toBe(false); + }); + + it('should detect test-gb.txt as text with gb2312 hint', () => { + const result = isBinaryFileSync(path.join(ENCODING_PATH, 'test-gb.txt'), { encoding: 'gb2312' }); + expect(result).toBe(false); + }); + + it('should detect test-gb2.txt as text with gbk hint', () => { + const result = isBinaryFileSync(path.join(ENCODING_PATH, 'test-gb2.txt'), { encoding: 'gbk' }); + expect(result).toBe(false); + }); + + it('should detect test-kr.txt as text with euc-kr hint', () => { + const result = isBinaryFileSync(path.join(ENCODING_PATH, 'test-kr.txt'), { encoding: 'euc-kr' }); + expect(result).toBe(false); + }); + + it('should detect CJK files as text with generic cjk hint', () => { + const result = isBinaryFileSync(path.join(ENCODING_PATH, 'big5.txt'), { encoding: 'cjk' }); + expect(result).toBe(false); + }); + + it('should still detect binary files as binary even with cjk hint', () => { + const result = isBinaryFileSync(path.join(FIXTURE_PATH, 'grep'), { encoding: 'cjk' }); + expect(result).toBe(true); + }); + }); + + describe('async API with encoding hints', () => { + it('should work with async API and encoding hints', async () => { + const result = await isBinaryFile(path.join(ENCODING_PATH, 'big5.txt'), { encoding: 'big5' }); + expect(result).toBe(false); + }); + + it('should auto-detect UTF-16 with async API', async () => { + const result = await isBinaryFile(path.join(ENCODING_PATH, 'utf16le-no-bom.txt')); + expect(result).toBe(false); + }); + }); +}); diff --git a/test/fixtures/encodings/utf16be-no-bom.txt b/test/fixtures/encodings/utf16be-no-bom.txt new file mode 100644 index 0000000000000000000000000000000000000000..e1f4e40a59f7af8bb12926f7e109293718456d30 GIT binary patch literal 106 zcmXAh%?dzZ5Jk_UuxDZ80TkKDhHNZ-qLlKJ*LR|+x%ZyA4K;zy*-3~P)Wb9}5}P{~ jR!-f&dr3t~MovK~f4p3_c5XG2wlwPH-}580jg9aEk){!L literal 0 HcmV?d00001 diff --git a/test/fixtures/encodings/utf16le-no-bom.txt b/test/fixtures/encodings/utf16le-no-bom.txt new file mode 100644 index 0000000000000000000000000000000000000000..f2bf5d2fc815c1b0eef8138fb12831201fbb116f GIT binary patch literal 106 zcmXAh%L+gM5Jk_YuxDZ80~A@14cS;CN-2;0z7tK&x%bX(N+vdECnBJihi+sb)VD0G ioT`6!CIvAGDHXZ- { const bytes = fs.readFileSync(path.join(FIXTURE_PATH, 'grep')); const size = fs.lstatSync(path.join(FIXTURE_PATH, 'grep')).size; - const result = isBinaryFileSync(bytes, size); + const result = isBinaryFileSync(bytes, { size }); expect(result).toBe(true); }); @@ -43,7 +43,7 @@ describe('sync', () => { const bytes = fs.readFileSync(path.join(FIXTURE_PATH, 'perl_script')); const size = fs.lstatSync(path.join(FIXTURE_PATH, 'perl_script')).size; - const result = isBinaryFileSync(bytes, size); + const result = isBinaryFileSync(bytes, { size }); expect(result).toBe(false); }); @@ -121,6 +121,8 @@ describe('sync', () => { const files = fs.readdirSync(encodingDir); files.forEach((file) => { + // Big5, GB, and Korean encodings require encoding hints to be detected as text. + // See encoding-hints.test.ts for tests with encoding hints. if (!/big5/.test(file) && !/gb/.test(file) && !/kr/.test(file)) { expect(isBinaryFileSync(path.join(encodingDir, file))).toBe(false); } From 20e95cdbc5b1aa389ffab55410310038e0a72148 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Fri, 5 Dec 2025 16:01:09 -0500 Subject: [PATCH 7/7] add `"esModuleInterop": true` --- tsconfig.json | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tsconfig.json b/tsconfig.json index 871bc1c..08a1d33 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -5,8 +5,14 @@ "moduleResolution": "NodeNext", "declaration": true, "outDir": "./lib", - "strict": true + "strict": true, + "esModuleInterop": true }, - "include": ["src"], - "exclude": ["node_modules", "**/__tests__/*"] + "include": [ + "src" + ], + "exclude": [ + "node_modules", + "**/__tests__/*" + ] }