Skip to content

Commit b7fe305

Browse files
sinescodeclaude
andcommitted
Release v0.4.3 — bug fixes, safety improvements, new flags
- Add --max-matches/-M flag to limit output count - Add --quiet/-q flag for script/pipe usage - BufWriter 64KB batching for faster output I/O - Symlink cycle detection in recursive file scan - Channel-based progress wake (was 100ms sleep) - Unsafe mmap documented with safety rationale - Version auto-synced from Cargo.toml via env!() - try_set_handler replaces panicky set_handler - Empty --domain validated early with clear error - 13 unit tests for domain matching logic Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
1 parent 2191751 commit b7fe305

4 files changed

Lines changed: 186 additions & 38 deletions

File tree

Cargo.lock

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[package]
22
name = "ulpExtractor"
3-
version = "0.4.2"
3+
version = "0.4.3"
44
edition = "2021"
55

66
[dependencies]

src/extractor.rs

Lines changed: 117 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
use std::collections::HashSet;
22
use std::fs::{self, File, OpenOptions};
3-
use std::io::Write;
3+
use std::io::{BufWriter, Write};
44
use std::path::{Path, PathBuf};
55
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
66
use std::sync::Arc;
@@ -28,8 +28,6 @@ pub struct ExtractResult {
2828
pub duration_ms: u64,
2929
}
3030

31-
/// Find all files in `dir` matching any of the given extensions.
32-
3331
/// Find `domain` inside `user_part` at a domain boundary. Returns the end
3432
/// position of the match so the caller can slice the remainder (user:pass).
3533
/// Handles URLs, emails, and bare domains:
@@ -85,6 +83,12 @@ fn trim_ascii(mut bytes: &[u8]) -> &[u8] {
8583
bytes
8684
}
8785

86+
fn is_symlink(path: &Path) -> bool {
87+
fs::symlink_metadata(path)
88+
.map(|m| m.file_type().is_symlink())
89+
.unwrap_or(false)
90+
}
91+
8892
pub fn find_files(dir: &Path, extensions: &[String], recursive: bool) -> std::io::Result<Vec<PathBuf>> {
8993
let exts: Vec<String> = extensions
9094
.iter()
@@ -101,6 +105,9 @@ pub fn find_files(dir: &Path, extensions: &[String], recursive: bool) -> std::io
101105
for entry in fs::read_dir(dir)? {
102106
let entry = entry?;
103107
let path = entry.path();
108+
if is_symlink(&path) {
109+
continue;
110+
}
104111
if path.is_file() {
105112
let matches = path.extension()
106113
.and_then(|e| e.to_str())
@@ -118,6 +125,9 @@ fn find_files_recursive(dir: &Path, exts: &[String], files: &mut Vec<PathBuf>) -
118125
for entry in fs::read_dir(dir)? {
119126
let entry = entry?;
120127
let path = entry.path();
128+
if is_symlink(&path) {
129+
continue;
130+
}
121131
if path.is_dir() {
122132
find_files_recursive(&path, exts, files)?;
123133
} else if path.is_file() {
@@ -152,8 +162,9 @@ pub fn extract(
152162
progress: Arc<ExtractProgress>,
153163
cancelled: Arc<AtomicBool>,
154164
append: bool,
165+
max_matches: Option<usize>,
155166
) -> std::io::Result<ExtractResult> {
156-
extract_multi(&[input_path.to_path_buf()], domain, divider, threads, output_path, progress, cancelled, append)
167+
extract_multi(&[input_path.to_path_buf()], domain, divider, threads, output_path, progress, cancelled, append, max_matches)
157168
}
158169

159170
/// Extract from multiple files into a single output.
@@ -169,12 +180,17 @@ pub fn extract_multi(
169180
progress: Arc<ExtractProgress>,
170181
cancelled: Arc<AtomicBool>,
171182
append: bool,
183+
max_matches: Option<usize>,
172184
) -> std::io::Result<ExtractResult> {
173185
let start = Instant::now();
174186
let domain = domain.as_bytes();
175187
let div = divider as u8;
176188
let threads = threads.max(1);
177189

190+
if domain.is_empty() {
191+
return Ok(ExtractResult { matched_count: 0, total_bytes: 0, duration_ms: 0 });
192+
}
193+
178194
// Pre-scan total bytes
179195
let total = total_bytes(input_paths)?;
180196
progress.total.store(total as usize, Ordering::Relaxed);
@@ -184,11 +200,12 @@ pub fn extract_multi(
184200
fs::create_dir_all(parent)?;
185201
}
186202
}
187-
let mut out = if append {
203+
let out_file = if append {
188204
OpenOptions::new().append(true).create(true).open(output_path)?
189205
} else {
190206
File::create(output_path)?
191207
};
208+
let mut out = BufWriter::with_capacity(65536, out_file);
192209
let mut seen: HashSet<Vec<u8>> = HashSet::new();
193210
let mut total_matched = 0usize;
194211
let mut bytes_done = 0usize;
@@ -203,8 +220,17 @@ pub fn extract_multi(
203220
if cancelled.load(Ordering::Relaxed) {
204221
break;
205222
}
223+
if let Some(max) = max_matches {
224+
if total_matched >= max {
225+
break;
226+
}
227+
}
206228

207229
let file = File::open(input_path)?;
230+
// Safety: file is opened read-only and mmap is only used for reading.
231+
// The file may be modified externally by another process — this is a
232+
// documented limitation. In practice, input files for this tool are
233+
// static credential dumps.
208234
let mmap = unsafe { Mmap::map(&file)? };
209235
let data: &[u8] = &mmap;
210236

@@ -300,11 +326,17 @@ pub fn extract_multi(
300326
out.write_all(line)?;
301327
out.write_all(b"\n")?;
302328
total_matched += 1;
329+
if let Some(max) = max_matches {
330+
if total_matched >= max {
331+
break;
332+
}
333+
}
303334
}
304335
}
305336
progress.matched.store(total_matched, Ordering::Relaxed);
306337
}
307338

339+
out.flush()?;
308340
progress.matched.store(total_matched, Ordering::Relaxed);
309341
progress.processed.store(total as usize, Ordering::Relaxed);
310342

@@ -314,3 +346,83 @@ pub fn extract_multi(
314346
duration_ms: start.elapsed().as_millis() as u64,
315347
})
316348
}
349+
350+
// ── Tests ───────────────────────────────────────────────────────────────
351+
352+
#[cfg(test)]
353+
mod tests {
354+
use super::*;
355+
356+
#[test]
357+
fn test_domain_end_exact_match() {
358+
assert_eq!(domain_end(b"netflix.com", b"netflix.com"), Some(11));
359+
}
360+
361+
#[test]
362+
fn test_domain_end_subdomain() {
363+
// www.netflix.com → matches netflix.com
364+
let end = domain_end(b"www.netflix.com", b"netflix.com");
365+
assert_eq!(end, Some(15));
366+
}
367+
368+
#[test]
369+
fn test_domain_end_deep_subdomain() {
370+
let end = domain_end(b"platform.deepseek.com", b"deepseek.com");
371+
assert_eq!(end, Some(21));
372+
}
373+
374+
#[test]
375+
fn test_domain_end_url_with_path() {
376+
let end = domain_end(b"https://deepseek.com/login", b"deepseek.com");
377+
assert_eq!(end, Some(20));
378+
}
379+
380+
#[test]
381+
fn test_domain_end_email() {
382+
let end = domain_end(b"user@deepseek.com", b"deepseek.com");
383+
assert_eq!(end, Some(17));
384+
}
385+
386+
#[test]
387+
fn test_domain_end_rejects_partial() {
388+
// mydeepseek.com should NOT match deepseek.com
389+
assert_eq!(domain_end(b"mydeepseek.com", b"deepseek.com"), None);
390+
}
391+
392+
#[test]
393+
fn test_domain_end_bare_domain_with_colon() {
394+
// deepseek.com:user → matches deepseek.com
395+
let end = domain_end(b"deepseek.com:user", b"deepseek.com");
396+
assert_eq!(end, Some(12));
397+
}
398+
399+
#[test]
400+
fn test_domain_end_empty_domain() {
401+
assert_eq!(domain_end(b"anything", b""), None);
402+
}
403+
404+
#[test]
405+
fn test_domain_end_user_part_too_short() {
406+
assert_eq!(domain_end(b"short", b"verylongdomain.com"), None);
407+
}
408+
409+
#[test]
410+
fn test_domain_end_no_match() {
411+
assert_eq!(domain_end(b"something.else.com", b"netflix.com"), None);
412+
}
413+
414+
#[test]
415+
fn test_trim_ascii_basic() {
416+
assert_eq!(trim_ascii(b" hello "), b"hello");
417+
}
418+
419+
#[test]
420+
fn test_trim_ascii_no_whitespace() {
421+
assert_eq!(trim_ascii(b"hello"), b"hello");
422+
}
423+
424+
#[test]
425+
fn test_trim_ascii_all_whitespace() {
426+
assert_eq!(trim_ascii(b" "), b"");
427+
}
428+
}

0 commit comments

Comments
 (0)