diff --git a/CHANGELOG.md b/CHANGELOG.md index 49f6da1..4bbdbd0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,24 @@ # Changelog All notable changes to this project follow Keep a Changelog and Semantic Versioning. +## [Unreleased] +### Added +- Evidence fidelity: normalizer tags each event's source; `process_creation` rules now + only accept genuine process-creation telemetry (Sysmon/EventChannel eventdata, auditd + execve). Low-fidelity `full_log`/decoder scrapes can no longer produce a false `DETECTED`. +- Sigma matcher coverage: `re` (regex), numeric `lt`/`lte`/`gt`/`gte`, and keyword + (full-text search) identifiers. +- Wazuh collector date pre-filter: queries read only the day(s) a window spans instead of + the whole archive; scanner buffer enlarged so long archive events are no longer truncated. +- Unit tests for the previously untested `canary` and `report` packages. +### Changed +- Canary now executes once and polls telemetry until a deadline (configurable via `Checker`) + instead of re-firing on fixed-interval retries. +- Techniques whose collected events are all low-fidelity report `NO_TELEMETRY` (collection + gap) rather than `MISSED` (proven detection miss). +- Dry-run / synthetic pipeline prints an unmistakable banner so its output cannot be mistaken + for real telemetry. + ## [1.2.0] — 2026-07-04 ### Added - Pipeline canary (positive control): per-run marker, gating logic, `make canary` diff --git a/README.md b/README.md index 2205711..4435c6d 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,17 @@ go test ./internal/evaluator/ -v -run Regression ## Supported Sigma subset The native Go matcher supports the Sigma specification subset needed for process-creation rules: -field modifiers (`contains`, `startswith`, `endswith`, `|all`), condition grammar (`and`/`or`/`not`/parens/`1 of them`/`all of them`), and case-insensitive matching. Not yet supported: regex, aggregation expressions, `near`, correlated rules. +field modifiers (`contains`, `startswith`, `endswith`, `|all`, `re`, numeric `lt`/`lte`/`gt`/`gte`), +`*` wildcards in values, keyword (full-text) search identifiers, condition grammar +(`and`/`or`/`not`/parens/`1 of them`/`all of them`), and case-insensitive matching. Not yet +supported: aggregation expressions, `near`, correlated rules, `base64`/`cidr` modifiers. + +**Evidence fidelity.** For `process_creation` rules the evaluator only accepts genuine +process-creation telemetry (Sysmon/EventChannel `eventdata`, auditd `execve`). Command-output and +metadata scrapes (`full_log`, decoder name) are tagged low-fidelity and can never satisfy a +process-creation rule — so a log line that merely mentions a binary cannot produce a false +`DETECTED`. When a technique collects only low-fidelity events, the verdict is `NO_TELEMETRY` +(a collection gap), not `MISSED`. ## Limitations diff --git a/cmd/purpleloop/main.go b/cmd/purpleloop/main.go index a9c63fa..8a84846 100644 --- a/cmd/purpleloop/main.go +++ b/cmd/purpleloop/main.go @@ -71,6 +71,15 @@ func main() { ctx := context.Background() + // Warn loudly whenever any pipeline stage is synthetic, so a dry-run report + // can never be mistaken for real detection evidence. + if *dryRun || *victim == "" || *manager == "" { + fmt.Fprintln(os.Stderr, "┌─────────────────────────────────────────────────────────────┐") + fmt.Fprintln(os.Stderr, "│ DRY-RUN / SYNTHETIC PIPELINE — results are NOT real telemetry │") + fmt.Fprintln(os.Stderr, "│ Missing --victim-container or --manager-container. │") + fmt.Fprintln(os.Stderr, "└─────────────────────────────────────────────────────────────┘") + } + switch { case *arbiterFile != "": if err := runArbiter(ctx, *arbiterFile, *output, *dryRun, *victim, *manager); err != nil { diff --git a/internal/canary/canary.go b/internal/canary/canary.go index b211a94..e760764 100644 --- a/internal/canary/canary.go +++ b/internal/canary/canary.go @@ -32,62 +32,91 @@ func NewMarker() string { return "purpleloop-canary-" + hex.EncodeToString(b) } -// Check executes the canary on one platform and verifies the marker was detected. -func Check(ctx context.Context, marker string, exec model.Executor, - coll model.Collector, platform string, target model.Target, dryRun bool) Result { +// Checker verifies the canary marker on one platform. Zero-value fields fall +// back to production defaults; tests override the timing and rule paths. +type Checker struct { + RulePath string // canary Sigma rule (default: rulePath const) + RulesDir string // rules root passed to the evaluator (default: "detections") + PollInterval time.Duration // gap between telemetry polls (default: 5s) + Timeout time.Duration // total time to wait for the marker (default: 90s) + WindowPad time.Duration // padding around the run window (default: 2m) +} - r := Result{Platform: platform, Marker: marker} +func (ck Checker) withDefaults() Checker { + if ck.RulePath == "" { + ck.RulePath = rulePath + } + if ck.RulesDir == "" { + ck.RulesDir = "detections" + } + if ck.PollInterval == 0 { + ck.PollInterval = 5 * time.Second + } + if ck.Timeout == 0 { + ck.Timeout = 90 * time.Second + } + if ck.WindowPad == 0 { + ck.WindowPad = 2 * time.Minute + } + return ck +} - // Build the canary atomic - atomic := atomicFor(platform, marker) +// Run executes the canary once, then polls telemetry until the marker is +// detected or the timeout elapses. Executing a single time (rather than +// re-firing on every retry) keeps the run window tight and the marker unique. +func (ck Checker) Run(ctx context.Context, marker string, exec model.Executor, + coll model.Collector, platform string, target model.Target) Result { - // Execute with 3-try bounded retry for ingestion lag - for attempt := 0; attempt < 3; attempt++ { - if attempt > 0 { - select { - case <-ctx.Done(): - r.Err = ctx.Err() - return r - case <-time.After(time.Duration(attempt+1) * 5 * time.Second): - } - } + ck = ck.withDefaults() + r := Result{Platform: platform, Marker: marker} + atomic := atomicFor(platform, marker) - run, err := exec.Run(ctx, atomic, target) - if err != nil { - r.Err = fmt.Errorf("canary execute: %w", err) - continue - } - _ = exec.Cleanup(ctx, atomic, target) + run, err := exec.Run(ctx, atomic, target) + if err != nil { + r.Err = fmt.Errorf("canary execute: %w", err) + return r + } + defer func() { _ = exec.Cleanup(ctx, atomic, target) }() - time.Sleep(10 * time.Second) // ingest delay + eval := evaluator.RuleMatcherEvaluator{RulesDir: ck.RulesDir} + rule := model.SigmaRule{Path: ck.RulePath, Title: "Pipeline Canary"} + window := run.Window(ck.WindowPad) - events, err := coll.Query(ctx, run.Window(2*time.Minute), target.Host) + deadline := time.Now().Add(ck.Timeout) + for { + events, err := coll.Query(ctx, window, target.Host) if err != nil { r.Err = fmt.Errorf("canary collect: %w", err) - continue - } - if len(events) == 0 { - continue + } else if len(events) > 0 { + verdict, evidence, err := eval.Evaluate(rule, events) + switch { + case err != nil: + r.Err = fmt.Errorf("canary evaluate: %w", err) + case verdict == model.Detected && evidenceContainsMarker(evidence, marker): + r.Healthy = true + r.Evidence = evidence + r.Err = nil + return r + } } - // Evaluate canary rule - eval := evaluator.RuleMatcherEvaluator{RulesDir: "detections"} - rule := model.SigmaRule{Path: rulePath, Title: "Pipeline Canary"} - verdict, evidence, err := eval.Evaluate(rule, events) - if err != nil { - r.Err = fmt.Errorf("canary evaluate: %w", err) - continue + if time.Now().After(deadline) { + return r } - - // Must be DETECTED AND contain exact marker - if verdict == model.Detected && evidenceContainsMarker(evidence, marker) { - r.Healthy = true - r.Evidence = evidence + select { + case <-ctx.Done(): + r.Err = ctx.Err() return r + case <-time.After(ck.PollInterval): } } +} - return r +// Check executes the canary on one platform and verifies the marker was +// detected, using production defaults. +func Check(ctx context.Context, marker string, exec model.Executor, + coll model.Collector, platform string, target model.Target, dryRun bool) Result { + return Checker{}.Run(ctx, marker, exec, coll, platform, target) } func atomicFor(platform, marker string) model.AtomicTest { diff --git a/internal/canary/canary_test.go b/internal/canary/canary_test.go new file mode 100644 index 0000000..7be77cb --- /dev/null +++ b/internal/canary/canary_test.go @@ -0,0 +1,86 @@ +package canary + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/jayelbotvibe-web/purple-loop/internal/model" +) + +type fakeExecutor struct{ ran bool } + +func (f *fakeExecutor) Run(context.Context, model.AtomicTest, model.Target) (model.RunResult, error) { + f.ran = true + now := time.Now().UTC() + return model.RunResult{StartedAt: now, FinishedAt: now}, nil +} +func (f *fakeExecutor) Cleanup(context.Context, model.AtomicTest, model.Target) error { return nil } + +// fakeCollector returns whatever events it is configured with. +type fakeCollector struct{ events []model.Event } + +func (f fakeCollector) Query(context.Context, model.TimeWindow, string) ([]model.Event, error) { + return f.events, nil +} + +func testChecker() Checker { + return Checker{ + RulePath: "../../detections/canary/pipeline_canary.yml", + RulesDir: "../../detections", + PollInterval: time.Millisecond, + Timeout: 20 * time.Millisecond, + } +} + +func TestChecker_HealthyOnProcessTelemetry(t *testing.T) { + marker := NewMarker() + raw, _ := json.Marshal(map[string]string{ + "Image": "C:\\Windows\\System32\\cmd.exe", + "CommandLine": "cmd.exe /c echo " + marker, + }) + coll := fakeCollector{events: []model.Event{{ID: "e1", Raw: raw}}} + + r := testChecker().Run(context.Background(), marker, &fakeExecutor{}, coll, "windows", model.Target{Host: "win"}) + if !r.Healthy { + t.Fatalf("expected healthy canary, got err=%v", r.Err) + } + if len(r.Evidence) == 0 { + t.Error("healthy canary should carry evidence") + } +} + +func TestChecker_NotHealthyWithoutTelemetry(t *testing.T) { + marker := NewMarker() + coll := fakeCollector{events: nil} + + r := testChecker().Run(context.Background(), marker, &fakeExecutor{}, coll, "linux", model.Target{Host: "victim"}) + if r.Healthy { + t.Fatal("no telemetry must not report healthy") + } +} + +// A command-output log echoing the marker is NOT process-creation evidence, +// so the canary must not report the pipeline healthy on it. +func TestChecker_LowFidelityLogIsNotHealthy(t *testing.T) { + marker := NewMarker() + raw := json.RawMessage(`{"full_log":"ossec: output: 'echo ` + marker + `': ` + marker + `"}`) + coll := fakeCollector{events: []model.Event{{ID: "log1", Raw: raw}}} + + r := testChecker().Run(context.Background(), marker, &fakeExecutor{}, coll, "linux", model.Target{Host: "victim"}) + if r.Healthy { + t.Fatal("low-fidelity command-output must not report the pipeline healthy") + } +} + +func TestNewMarker_Unique(t *testing.T) { + a, b := NewMarker(), NewMarker() + if a == b { + t.Error("markers should be unique") + } + if !strings.HasPrefix(a, "purpleloop-canary-") { + t.Errorf("unexpected marker format: %q", a) + } +} diff --git a/internal/collector/prefilter_test.go b/internal/collector/prefilter_test.go new file mode 100644 index 0000000..c5a4fee --- /dev/null +++ b/internal/collector/prefilter_test.go @@ -0,0 +1,44 @@ +package collector + +import ( + "testing" + "time" + + "github.com/jayelbotvibe-web/purple-loop/internal/model" +) + +func TestDateAlternation(t *testing.T) { + mk := func(s, e string) model.TimeWindow { + start, _ := time.Parse(time.RFC3339, s) + end, _ := time.Parse(time.RFC3339, e) + return model.TimeWindow{Start: start, End: end} + } + + cases := []struct { + name string + w model.TimeWindow + want string + }{ + {"same day", mk("2026-07-04T09:00:00Z", "2026-07-04T09:05:00Z"), "2026-07-04"}, + {"spans midnight", mk("2026-07-04T23:59:00Z", "2026-07-05T00:02:00Z"), "2026-07-04|2026-07-05"}, + {"zero window", model.TimeWindow{}, ""}, + {"reversed", mk("2026-07-05T00:00:00Z", "2026-07-04T00:00:00Z"), ""}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := dateAlternation(c.w); got != c.want { + t.Errorf("dateAlternation = %q, want %q", got, c.want) + } + }) + } +} + +func TestShQuote(t *testing.T) { + if got := shQuote("victim01"); got != "'victim01'" { + t.Errorf("shQuote = %q", got) + } + // An embedded single quote must be escaped so it cannot break out. + if got := shQuote("a'b"); got != `'a'\''b'` { + t.Errorf("shQuote escaping = %q", got) + } +} diff --git a/internal/collector/wazuh.go b/internal/collector/wazuh.go index efa9194..f7264cd 100644 --- a/internal/collector/wazuh.go +++ b/internal/collector/wazuh.go @@ -5,6 +5,7 @@ package collector import ( "bufio" + "bytes" "context" "encoding/json" "fmt" @@ -62,7 +63,7 @@ func (c *WazuhCollector) Query(ctx context.Context, w model.TimeWindow, host str return []model.Event{{ID: "dry-0001", Timestamp: time.Now().UTC(), Raw: raw}}, nil } - lines, err := c.readAlerts(ctx, host) + lines, err := c.readAlerts(ctx, host, w) if err != nil { return nil, fmt.Errorf("wazuh collector: %w", err) } @@ -96,47 +97,79 @@ func (c *WazuhCollector) Query(ctx context.Context, w model.TimeWindow, host str return events, nil } +const archivesPath = "/var/ossec/logs/archives/archives.json" + // readAlerts gets lines from alerts.json matching host, either via docker -// exec or from a local fixture file (test mode). -func (c *WazuhCollector) readAlerts(ctx context.Context, host string) ([]string, error) { +// exec or from a local fixture file (test mode). For the docker path it adds a +// coarse date pre-filter derived from the window, so a single query only reads +// the day(s) it actually spans instead of the entire archive history. +func (c *WazuhCollector) readAlerts(ctx context.Context, host string, w model.TimeWindow) ([]string, error) { if c.alertsPath != "" { - return readAlertsFile(c.alertsPath, host) + return runGrep(exec.Command("grep", "-F", "--", host, c.alertsPath)) + } + + dateRe := dateAlternation(w) + var cmd *exec.Cmd + if dateRe == "" { + cmd = exec.CommandContext(ctx, "docker", "exec", c.ManagerContainer, + "grep", "-F", "--", host, archivesPath) + } else { + pipeline := fmt.Sprintf("grep -F -- %s %s | grep -E -- %s", + shQuote(host), shQuote(archivesPath), shQuote(dateRe)) + cmd = exec.CommandContext(ctx, "docker", "exec", c.ManagerContainer, "sh", "-c", pipeline) } - cmd := exec.CommandContext(ctx, "docker", "exec", c.ManagerContainer, - "grep", host, "/var/ossec/logs/archives/archives.json") + return runGrep(cmd) +} + +// runGrep executes a grep command, treating exit status 1 (no matches) as an +// empty result rather than an error, and scans the output into lines. +func runGrep(cmd *exec.Cmd) ([]string, error) { out, err := cmd.Output() if err != nil { - // grep returns 1 on no matches — that's not an error for us if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 1 { return nil, nil } - return nil, fmt.Errorf("docker exec grep: %w", err) + return nil, fmt.Errorf("grep: %w", err) } + return scanLines(out), nil +} + +// scanLines splits grep output into trimmed, non-empty lines. The buffer is +// enlarged because a single Wazuh archive event can exceed bufio's default +// 64 KB token limit, which would otherwise silently drop long events. +func scanLines(out []byte) []string { var lines []string - sc := bufio.NewScanner(strings.NewReader(string(out))) + sc := bufio.NewScanner(bytes.NewReader(out)) + sc.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) for sc.Scan() { if t := strings.TrimSpace(sc.Text()); t != "" { lines = append(lines, t) } } - return lines, nil + return lines } -func readAlertsFile(path, host string) ([]string, error) { - cmd := exec.Command("grep", host, path) - out, err := cmd.Output() - if err != nil { - if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 1 { - return nil, nil - } - return nil, fmt.Errorf("grep fixture: %w", err) +// dateAlternation builds a grep -E alternation of the YYYY-MM-DD stamps the +// window spans (e.g. "2026-07-04|2026-07-05"). Returns "" to skip the +// pre-filter when the window is unset or spans an implausibly large range. +func dateAlternation(w model.TimeWindow) string { + if w.Start.IsZero() || w.End.IsZero() || w.End.Before(w.Start) { + return "" } - var lines []string - sc := bufio.NewScanner(strings.NewReader(string(out))) - for sc.Scan() { - if t := strings.TrimSpace(sc.Text()); t != "" { - lines = append(lines, t) + const maxDays = 31 + start := w.Start.UTC().Truncate(24 * time.Hour) + end := w.End.UTC().Truncate(24 * time.Hour) + var stamps []string + for d := start; !d.After(end); d = d.Add(24 * time.Hour) { + stamps = append(stamps, d.Format("2006-01-02")) + if len(stamps) > maxDays { + return "" // range too wide to be a useful pre-filter } } - return lines, nil + return strings.Join(stamps, "|") +} + +// shQuote single-quotes a string for safe use inside an sh -c command. +func shQuote(s string) string { + return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" } diff --git a/internal/evaluator/coverage_test.go b/internal/evaluator/coverage_test.go new file mode 100644 index 0000000..e9ba6eb --- /dev/null +++ b/internal/evaluator/coverage_test.go @@ -0,0 +1,76 @@ +package evaluator + +import ( + "os" + "testing" +) + +// mustParse parses a Sigma rule from an in-memory YAML string via a temp file, +// exercising the real parser path. +func mustParse(t *testing.T, yaml string) *Rule { + t.Helper() + path := t.TempDir() + "/rule.yml" + if err := os.WriteFile(path, []byte(yaml), 0o600); err != nil { + t.Fatalf("write rule: %v", err) + } + r, err := RuleParser{}.Parse(path) + if err != nil { + t.Fatalf("parse: %v", err) + } + return r +} + +func TestMatcher_Regex(t *testing.T) { + rule := mustParse(t, ` +title: regex +logsource: + category: process_creation +detection: + selection: + CommandLine|re: 'nc\s+-l\s+\d+' + condition: selection +`) + m := Matcher{} + if !m.Match(rule, map[string]string{"CommandLine": "nc -l 4444"}) { + t.Error("regex should match 'nc -l 4444'") + } + if m.Match(rule, map[string]string{"CommandLine": "netcat listening"}) { + t.Error("regex should not match 'netcat listening'") + } +} + +func TestMatcher_Numeric(t *testing.T) { + rule := mustParse(t, ` +title: numeric +detection: + selection: + EventID: 4688 + Count|gt: 5 + condition: selection +`) + m := Matcher{} + if !m.Match(rule, map[string]string{"EventID": "4688", "Count": "9"}) { + t.Error("Count 9 > 5 should match") + } + if m.Match(rule, map[string]string{"EventID": "4688", "Count": "3"}) { + t.Error("Count 3 > 5 should not match") + } +} + +func TestMatcher_Keywords(t *testing.T) { + rule := mustParse(t, ` +title: keywords +detection: + keywords: + - 'mimikatz' + - 'sekurlsa' + condition: keywords +`) + m := Matcher{} + if !m.Match(rule, map[string]string{"CommandLine": "invoke-mimikatz -dumpcreds"}) { + t.Error("keyword 'mimikatz' should match in CommandLine") + } + if m.Match(rule, map[string]string{"CommandLine": "whoami /all"}) { + t.Error("no keyword present should not match") + } +} diff --git a/internal/evaluator/fidelity_test.go b/internal/evaluator/fidelity_test.go new file mode 100644 index 0000000..d963dc5 --- /dev/null +++ b/internal/evaluator/fidelity_test.go @@ -0,0 +1,71 @@ +package evaluator + +import ( + "encoding/json" + "testing" + + "github.com/jayelbotvibe-web/purple-loop/internal/model" +) + +// A command-output log line that merely mentions a binary must NOT count as +// process-creation evidence. Before fidelity gating this produced a false +// DETECTED because the full_log scrape populated CommandLine/Image. +func TestEvaluate_LowFidelityLogDoesNotDetect(t *testing.T) { + eval := RuleMatcherEvaluator{RulesDir: "../../detections/linux"} + rule := model.SigmaRule{Path: "../../detections/canary/pipeline_canary.yml"} + + // full_log command output that echoes the canary marker — not a process event. + raw := json.RawMessage(`{"full_log":"ossec: output: 'echo purpleloop-canary-deadbeef': purpleloop-canary-deadbeef","decoder":{"name":"ossec"}}`) + events := []model.Event{{ID: "log-1", Raw: raw}} + + verdict, evidence, err := eval.Evaluate(rule, events) + if err != nil { + t.Fatalf("evaluate: %v", err) + } + if verdict == model.Detected { + t.Fatalf("low-fidelity log event must not DETECT a process_creation rule; got %s with %d evidence", verdict, len(evidence)) + } + if verdict != model.NoTelemetry { + t.Errorf("expected NO_TELEMETRY when only low-fidelity events exist, got %s", verdict) + } +} + +// Genuine process-creation telemetry (Windows eventdata) must still DETECT. +func TestEvaluate_HighFidelityProcessDetects(t *testing.T) { + eval := RuleMatcherEvaluator{RulesDir: "../../detections/windows"} + rule := model.SigmaRule{Path: "../../detections/canary/pipeline_canary.yml"} + + raw := json.RawMessage(`{"data":{"win":{"eventdata":{"commandLine":"cmd.exe /c echo purpleloop-canary-deadbeef","image":"C:\\Windows\\System32\\cmd.exe"}}}}`) + events := []model.Event{{ID: "win-1", Raw: raw}} + + verdict, evidence, err := eval.Evaluate(rule, events) + if err != nil { + t.Fatalf("evaluate: %v", err) + } + if verdict != model.Detected { + t.Fatalf("high-fidelity process event should DETECT, got %s (%d evidence)", verdict, len(evidence)) + } +} + +// The normalizer must tag sources so the evaluator can distinguish them. +func TestNormalizer_FidelityTagging(t *testing.T) { + n := Normalizer{} + cases := []struct { + name string + raw string + want string + }{ + {"windows eventdata", `{"data":{"win":{"eventdata":{"image":"C:\\a.exe"}}}}`, FidelityProcess}, + {"auditd execve", `{"data":{"audit":{"execve":{"a0":"id"}}}}`, FidelityProcess}, + {"top-level synthetic", `{"Image":"/usr/bin/id","CommandLine":"id"}`, FidelityProcess}, + {"full_log scrape", `{"full_log":"ossec: output: 'id': uid=0"}`, FidelityLog}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + out := n.Normalize(json.RawMessage(c.raw)) + if out[FidelityKey] != c.want { + t.Errorf("fidelity = %q, want %q (out=%v)", out[FidelityKey], c.want, out) + } + }) + } +} diff --git a/internal/evaluator/matcher.go b/internal/evaluator/matcher.go index 99b6fb8..22955d2 100644 --- a/internal/evaluator/matcher.go +++ b/internal/evaluator/matcher.go @@ -1,6 +1,10 @@ package evaluator -import "strings" +import ( + "regexp" + "strconv" + "strings" +) // Matcher evaluates parsed Sigma rules against normalized events. type Matcher struct{} @@ -40,6 +44,11 @@ func evalIdent(name string, detections map[string]FieldMap, event map[string]str if !ok { return false } + // A keyword identifier (bare string list) is a full-text search: any value + // appearing anywhere in the event's fields matches. + if kw, ok := fm[keywordField]; ok { + return matchKeywords(kw.Values, event) + } // All field entries in the identifier must match (AND) for field, entry := range fm { val, exists := event[field] @@ -53,6 +62,27 @@ func evalIdent(name string, detections map[string]FieldMap, event map[string]str return true } +// matchKeywords returns true if any keyword appears (case-insensitive) in any +// canonical field value of the event. +func matchKeywords(keywords []string, event map[string]string) bool { + for _, kw := range keywords { + needle := strings.ToLower(kw) + for field, val := range event { + if field == FidelityKey { + continue + } + if strings.Contains(strings.ToLower(val), needle) { + return true + } + } + } + return false +} + +// keywordField is the reserved field name under which a keyword (full-text) +// identifier's bare-string list is stored. +const keywordField = "__keywords__" + func matchField(eventValue string, entry FieldEntry) bool { if len(entry.Values) == 0 { return false @@ -62,6 +92,8 @@ func matchField(eventValue string, entry FieldEntry) bool { hasEndsWith := false hasStartsWith := false hasContains := false + hasRe := false + numOp := "" for _, m := range entry.Modifiers { switch m { case "all": @@ -72,12 +104,29 @@ func matchField(eventValue string, entry FieldEntry) bool { hasStartsWith = true case "contains": hasContains = true + case "re": + hasRe = true + case "lt", "lte", "gt", "gte": + numOp = m } } // Build the matcher function based on modifiers matchOne := func(candidate string) bool { - // Strip leading slash/drive for path matching + // Regex: match against the raw value, case-sensitive per Sigma default. + if hasRe { + re, err := regexp.Compile(candidate) + if err != nil { + return false + } + return re.MatchString(eventValue) + } + + // Numeric comparison: both sides must parse as numbers. + if numOp != "" { + return matchNumeric(eventValue, candidate, numOp) + } + v := strings.ToLower(eventValue) c := strings.ToLower(candidate) @@ -115,6 +164,27 @@ func matchField(eventValue string, entry FieldEntry) bool { return false } +// matchNumeric compares two numeric strings under the given operator +// (lt/lte/gt/gte). Returns false if either side is not a number. +func matchNumeric(eventValue, candidate, op string) bool { + ev, err1 := strconv.ParseFloat(strings.TrimSpace(eventValue), 64) + cv, err2 := strconv.ParseFloat(strings.TrimSpace(candidate), 64) + if err1 != nil || err2 != nil { + return false + } + switch op { + case "lt": + return ev < cv + case "lte": + return ev <= cv + case "gt": + return ev > cv + case "gte": + return ev >= cv + } + return false +} + // matchWildcard performs case-insensitive glob matching where * matches any // sequence of characters. It splits the pattern on * and matches each literal // segment in order within the value string. diff --git a/internal/evaluator/normalizer.go b/internal/evaluator/normalizer.go index 476c9ea..0916a22 100644 --- a/internal/evaluator/normalizer.go +++ b/internal/evaluator/normalizer.go @@ -10,19 +10,47 @@ import ( // Field mappings are derived from real captured events. type Normalizer struct{} +// Reserved keys and fidelity levels attached to a normalized event. +// +// FidelityKey records where the canonical fields came from, so the evaluator +// can refuse to treat command-output or metadata scraping as genuine +// process-creation evidence. This is what keeps a DETECTED verdict honest: +// a log line that merely mentions a binary name is not proof that the process +// actually ran. +const ( + FidelityKey = "__source_fidelity__" + + // FidelityProcess marks fields sourced from real process-creation + // telemetry (Sysmon/EventChannel eventdata, auditd execve, or a synthetic + // process event with top-level fields). + FidelityProcess = "process_creation" + + // FidelityLog marks fields scraped from command-output or metadata + // (full_log, decoder name). Usable for text/keyword rules, but NOT + // accepted as process-creation evidence. + FidelityLog = "log" +) + // Normalize converts a raw Wazuh event into a flat map of canonical fields. +// The reserved FidelityKey entry records the highest-fidelity source that +// contributed a field (see the Fidelity* constants). func (Normalizer) Normalize(raw json.RawMessage) map[string]string { var event map[string]any if err := json.Unmarshal(raw, &event); err != nil { return nil } out := make(map[string]string) + highFidelity := false + lowFidelity := false - // Top-level canonical fields (dry-run, fixtures) + // Top-level canonical fields (dry-run, fixtures, synthetic process events) getString(event, "Image", &out, "Image") getString(event, "ParentImage", &out, "ParentImage") getString(event, "CommandLine", &out, "CommandLine") getString(event, "User", &out, "User") + if out["Image"] != "" || out["CommandLine"] != "" { + highFidelity = true + } // Try Windows Sysmon / EventChannel paths if data, ok := event["data"].(map[string]any); ok { @@ -38,6 +66,7 @@ func (Normalizer) Normalize(raw json.RawMessage) map[string]string { getString(ed, "callerProcessName", &out, "Image") getString(ed, "subjectUserName", &out, "User") getString(ed, "processName", &out, "Image") + highFidelity = true } } } @@ -61,33 +90,48 @@ func (Normalizer) Normalize(raw json.RawMessage) map[string]string { } if len(parts) > 0 { out["CommandLine"] = strings.Join(parts, " ") + highFidelity = true } } } } - // Fallback: extract from full_log (command output events) + // Fallback: extract from full_log (command-output events). This is a text + // scrape, not a process event, so it is tagged low fidelity. if fl, ok := event["full_log"].(string); ok { // "ossec: output: 'df -P': ..." → extract command if idx := strings.Index(fl, "output: '"); idx >= 0 { rest := fl[idx+9:] if end := strings.Index(rest, "'"); end > 0 { cmd := rest[:end] - out["Image"] = cmd - out["CommandLine"] = cmd + if out["Image"] == "" { + out["Image"] = cmd + } + if out["CommandLine"] == "" { + out["CommandLine"] = cmd + } + lowFidelity = true } } } - // Use decoder name as fallback Image for SCA events + // Use decoder name as fallback Image for SCA events (metadata, low fidelity) if out["Image"] == "" { if dec, ok := event["decoder"].(map[string]any); ok { if name, ok := dec["name"].(string); ok { out["Image"] = name + lowFidelity = true } } } + switch { + case highFidelity: + out[FidelityKey] = FidelityProcess + case lowFidelity: + out[FidelityKey] = FidelityLog + } + return out } diff --git a/internal/evaluator/rule.go b/internal/evaluator/rule.go index e201cd1..0d35b4b 100644 --- a/internal/evaluator/rule.go +++ b/internal/evaluator/rule.go @@ -4,6 +4,7 @@ package evaluator import ( "os" + "strconv" "strings" "gopkg.in/yaml.v3" @@ -13,6 +14,7 @@ import ( type Rule struct { Path string Title string + Category string // logsource.category, e.g. "process_creation" Detections map[string]FieldMap // search-identifier → field conditions Condition Expr // parsed condition tree } @@ -68,6 +70,9 @@ func (RuleParser) Parse(path string) (*Rule, error) { } var raw struct { Title string `yaml:"title"` + Logsource struct { + Category string `yaml:"category"` + } `yaml:"logsource"` Detection struct { Condition string `yaml:"condition"` Fields map[string]any `yaml:",inline"` @@ -77,7 +82,7 @@ func (RuleParser) Parse(path string) (*Rule, error) { return nil, err } - rule := &Rule{Path: path, Title: raw.Title} + rule := &Rule{Path: path, Title: raw.Title, Category: raw.Logsource.Category} // Parse search-identifiers (all fields except "condition") rule.Detections = make(map[string]FieldMap) @@ -101,8 +106,43 @@ func (RuleParser) Parse(path string) (*Rule, error) { return rule, nil } +// scalarToString coerces a YAML scalar (string, int, float, bool) into the +// string form the matcher compares against. Returns "" for unsupported types. +func scalarToString(v any) string { + switch t := v.(type) { + case string: + return t + case bool: + return strconv.FormatBool(t) + case int: + return strconv.Itoa(t) + case int64: + return strconv.FormatInt(t, 10) + case float64: + return strconv.FormatFloat(t, 'f', -1, 64) + default: + return "" + } +} + func parseFieldMap(val any) (FieldMap, error) { fm := make(FieldMap) + + // A bare list of strings is a keyword (full-text) identifier, not a set of + // field conditions. Store it under the reserved keyword field. + if list, ok := val.([]any); ok { + var kw []string + for _, item := range list { + if s, ok := item.(string); ok { + kw = append(kw, s) + } + } + if len(kw) > 0 { + fm[keywordField] = FieldEntry{Values: kw} + } + return fm, nil + } + m, ok := val.(map[string]any) if !ok { return fm, nil @@ -113,16 +153,21 @@ func parseFieldMap(val any) (FieldMap, error) { parts := strings.Split(key, "|") fieldName := parts[0] entry.Modifiers = parts[1:] - // Parse value(s) + // Parse value(s). Scalars may be strings, numbers, or booleans + // (e.g. `EventID: 4688`, `Count|gt: 5`); coerce each to its string form. switch vv := v.(type) { case string: entry.Values = []string{vv} case []any: for _, item := range vv { - if s, ok := item.(string); ok { + if s := scalarToString(item); s != "" { entry.Values = append(entry.Values, s) } } + default: + if s := scalarToString(vv); s != "" { + entry.Values = []string{s} + } } fm[fieldName] = entry } diff --git a/internal/evaluator/sigma.go b/internal/evaluator/sigma.go index 172a61b..8f63284 100644 --- a/internal/evaluator/sigma.go +++ b/internal/evaluator/sigma.go @@ -48,13 +48,24 @@ func (e RuleMatcherEvaluator) Evaluate(rule model.SigmaRule, events []model.Even return model.Errored, nil, fmt.Errorf("parse rule %s: %w", rule.Path, err) } - // Evaluate each event + // For process-creation rules, only genuine process-creation telemetry can + // justify a verdict. Command-output/metadata scrapes (full_log, decoder + // name) are tagged low fidelity by the normalizer and are excluded here, so + // a log line that merely mentions a binary can never produce a false + // DETECTED. See normalizer.go and the Fidelity* constants. + requireProcess := parsedRule.Category == FidelityProcess + var matchedEvents []model.Event + usableEvents := 0 for _, ev := range events { normalized := normalizer.Normalize(ev.Raw) if len(normalized) == 0 { continue } + if requireProcess && normalized[FidelityKey] != FidelityProcess { + continue // low-fidelity event cannot be process-creation evidence + } + usableEvents++ if matcher.Match(parsedRule, normalized) { matchedEvents = append(matchedEvents, ev) } @@ -63,5 +74,10 @@ func (e RuleMatcherEvaluator) Evaluate(rule model.SigmaRule, events []model.Even if len(matchedEvents) > 0 { return model.Detected, matchedEvents, nil } + // No telemetry of the kind this rule needs was collected — that is a + // collection gap, not a proven detection miss. + if requireProcess && usableEvents == 0 { + return model.NoTelemetry, nil, nil + } return model.Missed, nil, nil } diff --git a/internal/feed/emulation.go b/internal/feed/emulation.go index a8dc4fa..ce72ae2 100644 --- a/internal/feed/emulation.go +++ b/internal/feed/emulation.go @@ -12,9 +12,9 @@ import ( // EmulationPlan is a multi-stage actor emulation plan. type EmulationPlan struct { - Name string `yaml:"name"` - Description string `yaml:"description"` - Stages []EmulationStage `yaml:"stages"` + Name string `yaml:"name"` + Description string `yaml:"description"` + Stages []EmulationStage `yaml:"stages"` } // EmulationStage is one tactical phase in an emulation plan. diff --git a/internal/report/report.go b/internal/report/report.go index 2b84b02..f918174 100644 --- a/internal/report/report.go +++ b/internal/report/report.go @@ -53,7 +53,7 @@ h1{color:#fff}table{width:100%;border-collapse:collapse;margin-top:1em} th,td{padding:.5em .75em;text-align:left;border-bottom:1px solid #333} th{background:#1a1a2e;color:#ccc} tr:hover{background:#1a1a2e} -.DETECTED{color:#4f4}.MISSED{color:#f44}.ERROR{color:#f0f} +.DETECTED{color:#4f4}.MISSED{color:#f44}.ERROR{color:#f0f}.NO_TELEMETRY{color:#fb0}.INCONCLUSIVE{color:#aaa} .summary{display:flex;gap:1.5em;margin:1em 0} .summary span{font-size:1.2em} @@ -61,7 +61,7 @@ tr:hover{background:#1a1a2e}

` + html.EscapeString(run.StartedAt.Format("2006-01-02 15:04:05 UTC")) + `

`) - for _, v := range []model.Verdict{model.Detected, model.Missed, model.Errored} { + for _, v := range []model.Verdict{model.Detected, model.Missed, model.NoTelemetry, model.Inconclusive, model.Errored} { if n, ok := counts[v]; ok && n > 0 { f.WriteString(fmt.Sprintf(`%s: %d`, v, v, n)) } diff --git a/internal/report/report_test.go b/internal/report/report_test.go new file mode 100644 index 0000000..0a79b6e --- /dev/null +++ b/internal/report/report_test.go @@ -0,0 +1,76 @@ +package report + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/jayelbotvibe-web/purple-loop/internal/model" +) + +func sampleRun() model.CampaignResult { + return model.CampaignResult{ + StartedAt: time.Date(2026, 7, 4, 9, 0, 0, 0, time.UTC), + Chains: []model.ProofChain{ + {TechniqueID: "T1059.004", SourceCVE: "CVE-2026-0001", ArbiterPriority: 0.91, + Verdict: model.Detected, EventsCollected: 3, RuleMatched: "win_proc_create.yml"}, + {TechniqueID: "T1082", Verdict: model.NoTelemetry, EventsCollected: 0}, + {TechniqueID: "T1016", Verdict: model.Missed, EventsCollected: 5}, + }, + } +} + +func TestJSONReporter(t *testing.T) { + var buf bytes.Buffer + if err := (JSONReporter{Out: &buf}).Write(sampleRun()); err != nil { + t.Fatalf("write: %v", err) + } + var back model.CampaignResult + if err := json.Unmarshal(buf.Bytes(), &back); err != nil { + t.Fatalf("output is not valid JSON: %v", err) + } + if len(back.Chains) != 3 || back.Chains[0].TechniqueID != "T1059.004" { + t.Errorf("round-trip mismatch: %+v", back.Chains) + } +} + +func TestHTMLReporter(t *testing.T) { + path := filepath.Join(t.TempDir(), "report.html") + if err := (HTMLReporter{Path: path}).Write(sampleRun()); err != nil { + t.Fatalf("write: %v", err) + } + data, _ := os.ReadFile(path) + html := string(data) + for _, want := range []string{"T1059.004", "DETECTED", "NO_TELEMETRY", "