diff --git a/Cargo.lock b/Cargo.lock index aeb49d04..b0d377ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -907,10 +907,13 @@ dependencies = [ "tree-sitter-bash", "tree-sitter-c", "tree-sitter-c-sharp", + "tree-sitter-clojure-orchard", "tree-sitter-containerfile", + "tree-sitter-dart", "tree-sitter-elixir", "tree-sitter-go", "tree-sitter-hcl", + "tree-sitter-html", "tree-sitter-java", "tree-sitter-javascript", "tree-sitter-json", @@ -921,9 +924,11 @@ dependencies = [ "tree-sitter-ruby", "tree-sitter-rust", "tree-sitter-scala", + "tree-sitter-sfapex", "tree-sitter-solidity", "tree-sitter-swift", "tree-sitter-typescript", + "tree-sitter-xml", "tree-sitter-yaml", "unicode-segmentation", "unicode-width", @@ -3356,6 +3361,16 @@ dependencies = [ "tree-sitter-language", ] +[[package]] +name = "tree-sitter-clojure-orchard" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e2db28a1ab22649790656936325bdc69e992c38006258694ea39a7620e784d" +dependencies = [ + "cc", + "tree-sitter-language", +] + [[package]] name = "tree-sitter-containerfile" version = "0.8.0" @@ -3366,6 +3381,16 @@ dependencies = [ "tree-sitter-language", ] +[[package]] +name = "tree-sitter-dart" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "325dd1e24ee9ee21111e9c43680ae7d6010aaa9f282b048a99b9c7163c1cf553" +dependencies = [ + "cc", + "tree-sitter-language", +] + [[package]] name = "tree-sitter-elixir" version = "0.3.5" @@ -3396,6 +3421,16 @@ dependencies = [ "tree-sitter-language", ] +[[package]] +name = "tree-sitter-html" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "261b708e5d92061ede329babaaa427b819329a9d427a1d710abb0f67bbef63ee" +dependencies = [ + "cc", + "tree-sitter-language", +] + [[package]] name = "tree-sitter-java" version = "0.23.5" @@ -3502,6 +3537,16 @@ dependencies = [ "tree-sitter-language", ] +[[package]] +name = "tree-sitter-sfapex" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ac630633dad003566212db2adc29a2a75c3e581a7288713dde28f42a428b016" +dependencies = [ + "cc", + "tree-sitter-language", +] + [[package]] name = "tree-sitter-solidity" version = "1.2.13" @@ -3532,6 +3577,16 @@ dependencies = [ "tree-sitter-language", ] +[[package]] +name = "tree-sitter-xml" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e670041f591d994f54d597ddcd8f4ebc930e282c4c76a42268743b71f0c8b6b3" +dependencies = [ + "cc", + "tree-sitter-language", +] + [[package]] name = "tree-sitter-yaml" version = "0.7.2" diff --git a/Cargo.toml b/Cargo.toml index b46d0f83..ecf3aa2f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,6 +70,11 @@ tree-sitter-ocaml = "0.25.0" tree-sitter-scala = "0.26.0" tree-sitter-elixir = "0.3.5" tree-sitter-json = "0.24.8" +tree-sitter-html = "0.23" +tree-sitter-xml = "0.7" +tree-sitter-dart = "0.2" +tree-sitter-clojure-orchard = "0.2.5" +tree-sitter-sfapex = "3.0.0" [features] default = [] diff --git a/docs/parity/registry-coverage.md b/docs/parity/registry-coverage.md index 2b9c305e..2641528e 100644 --- a/docs/parity/registry-coverage.md +++ b/docs/parity/registry-coverage.md @@ -12,10 +12,10 @@ Measures how well foxguard's existing Semgrep-compat YAML loader (`src/rules/sem | Rule files scanned | 2070 | | Files with YAML parse errors | 0 | | Total rules | 2144 | -| Rules loaded OK | 1982 (92.4%) | -| Rules skipped | 162 (7.6%) | +| Rules loaded OK | 2002 (93.4%) | +| Rules skipped | 142 (6.6%) | -**Headline load rate: 92.4%** (1982 / 2144 rules). +**Headline load rate: 93.4%** (2002 / 2144 rules). ## Skip-reason histogram @@ -23,16 +23,11 @@ Sorted by frequency. The reason names the operator/key that blocks the rule toda | Skip reason | Rules | % of skipped | % of all rules | |---|---:|---:|---:| -| `mode: taint (unsupported shape)` | 126 | 77.8% | 5.9% | -| `unsupported language: apex` | 9 | 5.6% | 0.4% | -| `generic mode (languages: [generic])` | 7 | 4.3% | 0.3% | -| `taint: pattern-propagators` | 5 | 3.1% | 0.2% | -| `unsupported language: clojure` | 5 | 3.1% | 0.2% | -| `unsupported language: html` | 4 | 2.5% | 0.2% | -| `mode: taint (unsupported language: apex)` | 3 | 1.9% | 0.1% | -| `mode: taint (unsupported language: swift)` | 1 | 0.6% | 0.0% | -| `unsupported language: dart` | 1 | 0.6% | 0.0% | -| `unsupported language: xml` | 1 | 0.6% | 0.0% | +| `mode: taint (unsupported shape)` | 126 | 88.7% | 5.9% | +| `generic mode (languages: [generic])` | 7 | 4.9% | 0.3% | +| `taint: pattern-propagators` | 5 | 3.5% | 0.2% | +| `mode: taint (unsupported language: apex)` | 3 | 2.1% | 0.1% | +| `mode: taint (unsupported language: swift)` | 1 | 0.7% | 0.0% | ## Priority order — operator/feature backlog @@ -51,16 +46,11 @@ Rules foxguard cannot run because it has no tree-sitter grammar for the target l | Rank | Language to add | Rules unlocked | |---:|---|---:| -| 1 | `unsupported language: apex` | 9 | -| 2 | `generic mode (languages: [generic])` | 7 | -| 3 | `unsupported language: clojure` | 5 | -| 4 | `unsupported language: html` | 4 | -| 5 | `mode: taint (unsupported language: apex)` | 3 | -| 6 | `mode: taint (unsupported language: swift)` | 1 | -| 7 | `unsupported language: dart` | 1 | -| 8 | `unsupported language: xml` | 1 | +| 1 | `generic mode (languages: [generic])` | 7 | +| 2 | `mode: taint (unsupported language: apex)` | 3 | +| 3 | `mode: taint (unsupported language: swift)` | 1 | -Missing-grammar gaps account for **31 rules** (1.4% of all rules). +Missing-grammar gaps account for **11 rules** (0.5% of all rules). ## Per-language breakdown @@ -86,26 +76,23 @@ Language is the rule's first declared language (js/ts/jsx/tsx collapsed to `java | c | 16 | 16 | 0 | 100.0% | | kotlin | 15 | 15 | 0 | 100.0% | | bash | 13 | 13 | 0 | 100.0% | -| apex | 12 | 0 | 12 | 0.0% | +| apex | 12 | 9 | 3 | 75.0% | | rust | 10 | 10 | 0 | 100.0% | | elixir | 7 | 7 | 0 | 100.0% | | json | 7 | 7 | 0 | 100.0% | | swift | 6 | 5 | 1 | 83.3% | -| clojure | 5 | 0 | 5 | 0.0% | +| clojure | 5 | 5 | 0 | 100.0% | | terraform | 5 | 5 | 0 | 100.0% | -| html | 4 | 0 | 4 | 0.0% | -| dart | 1 | 0 | 1 | 0.0% | -| xml | 1 | 0 | 1 | 0.0% | +| html | 4 | 4 | 0 | 100.0% | +| dart | 1 | 1 | 0 | 100.0% | +| xml | 1 | 1 | 0 | 100.0% | ## Top skip reasons per language -- **apex**: `unsupported language: apex` (9), `mode: taint (unsupported language: apex)` (3) -- **clojure**: `unsupported language: clojure` (5) +- **apex**: `mode: taint (unsupported language: apex)` (3) - **csharp**: `mode: taint (unsupported shape)` (9), `taint: pattern-propagators` (1) -- **dart**: `unsupported language: dart` (1) - **generic**: `generic mode (languages: [generic])` (7) - **go**: `mode: taint (unsupported shape)` (13) -- **html**: `unsupported language: html` (4) - **java**: `mode: taint (unsupported shape)` (17), `taint: pattern-propagators` (3) - **javascript**: `mode: taint (unsupported shape)` (35) - **php**: `mode: taint (unsupported shape)` (14) @@ -113,7 +100,6 @@ Language is the rule's first declared language (js/ts/jsx/tsx collapsed to `java - **ruby**: `mode: taint (unsupported shape)` (15), `taint: pattern-propagators` (1) - **solidity**: `mode: taint (unsupported shape)` (1) - **swift**: `mode: taint (unsupported language: swift)` (1) -- **xml**: `unsupported language: xml` (1) --- diff --git a/src/bin/gen_rules_ts.rs b/src/bin/gen_rules_ts.rs index 4d21290d..fd70fcb9 100644 --- a/src/bin/gen_rules_ts.rs +++ b/src/bin/gen_rules_ts.rs @@ -58,6 +58,11 @@ fn language_slug(language: Language) -> &'static str { Language::Scala => "scala", Language::Elixir => "elixir", Language::Json => "json", + Language::Apex => "apex", + Language::Clojure => "clojure", + Language::Html => "html", + Language::Xml => "xml", + Language::Dart => "dart", // Regex-mode rules fan out per language; this binary generates rules // for concrete languages only. Language::Regex => "regex", @@ -90,6 +95,11 @@ fn language_display_name(language: Language) -> &'static str { Language::Scala => "Scala", Language::Elixir => "Elixir", Language::Json => "JSON", + Language::Apex => "Apex", + Language::Clojure => "Clojure", + Language::Html => "HTML", + Language::Xml => "XML", + Language::Dart => "Dart", Language::Regex => "Regex", } } @@ -120,6 +130,11 @@ fn language_array_name(language: Language) -> &'static str { Language::Scala => "scalaRules", Language::Elixir => "elixirRules", Language::Json => "jsonRules", + Language::Apex => "apexRules", + Language::Clojure => "clojureRules", + Language::Html => "htmlRules", + Language::Xml => "xmlRules", + Language::Dart => "dartRules", Language::Regex => "regexRules", } } diff --git a/src/bin/registry_coverage.rs b/src/bin/registry_coverage.rs index b8f3e66d..fb1b7e6a 100644 --- a/src/bin/registry_coverage.rs +++ b/src/bin/registry_coverage.rs @@ -145,6 +145,20 @@ fn language_supported(lang: &str) -> bool { | "exs" // JSON grammar (tree-sitter-json). | "json" + // Apex grammar (tree-sitter-sfapex). + | "apex" + // Clojure grammar (tree-sitter-clojure-orchard). + | "clojure" + | "clj" + | "cljs" + | "cljc" + // HTML grammar (tree-sitter-html). + | "html" + | "htm" + // XML grammar (tree-sitter-xml). + | "xml" + // Dart grammar (tree-sitter-dart). + | "dart" ) } @@ -915,23 +929,40 @@ rules: #[test] fn unsupported_language_is_skipped() { - // Apex has no supported grammar — it must be skipped. + // COBOL has no supported grammar — it must be skipped. let r = rule( r#" rules: - - id: apex-rule + - id: cobol-rule pattern: foo message: m severity: INFO - languages: [apex] + languages: [cobol] "#, ); match classify_rule(&r) { - Outcome::Skipped(reason) => assert_eq!(reason, "unsupported language: apex"), + Outcome::Skipped(reason) => assert_eq!(reason, "unsupported language: cobol"), other => panic!("expected skip, got {other:?}"), } } + #[test] + fn apex_language_loads() { + // Apex now has a real grammar (tree-sitter-sfapex), so search-mode + // Apex rules must load. + let r = rule( + r#" +rules: + - id: apex-rule + pattern-regex: 'System\.debug' + message: m + severity: INFO + languages: [apex] +"#, + ); + assert!(matches!(classify_rule(&r), Outcome::Loaded)); + } + #[test] fn yaml_language_loads() { let r = rule( diff --git a/src/engine/parser.rs b/src/engine/parser.rs index 6774ead8..d19cd7de 100644 --- a/src/engine/parser.rs +++ b/src/engine/parser.rs @@ -42,6 +42,11 @@ fn parse_source_for_path( Language::Scala => tree_sitter_scala::LANGUAGE.into(), Language::Elixir => tree_sitter_elixir::LANGUAGE.into(), Language::Json => tree_sitter_json::LANGUAGE.into(), + Language::Apex => tree_sitter_sfapex::apex::LANGUAGE.into(), + Language::Clojure => tree_sitter_clojure_orchard::LANGUAGE.into(), + Language::Html => tree_sitter_html::LANGUAGE.into(), + Language::Xml => tree_sitter_xml::LANGUAGE_XML.into(), + Language::Dart => tree_sitter_dart::LANGUAGE.into(), // Regex-mode rules never use a tree-sitter parser — they match raw text // only. Return `None` immediately so the scanner skips the tree build. Language::Regex => return None, @@ -147,6 +152,52 @@ contract Token { assert!(!tree.root_node().has_error()); } + #[test] + fn parses_apex_without_errors() { + let source = + "public class Hello {\n public void greet() {\n String x = 'hi';\n }\n}\n"; + let tree = parse_path(source, Language::Apex, Path::new("Hello.cls")) + .expect("Apex parser should produce a tree"); + + assert!(!tree.root_node().has_error()); + } + + #[test] + fn parses_clojure_without_errors() { + let source = "(defn greet [name]\n (println \"hello\" name))\n"; + let tree = parse_path(source, Language::Clojure, Path::new("core.clj")) + .expect("Clojure parser should produce a tree"); + + assert!(!tree.root_node().has_error()); + } + + #[test] + fn parses_html_without_errors() { + let source = "\n
\n link\n \n\n"; + let tree = parse_path(source, Language::Html, Path::new("index.html")) + .expect("HTML parser should produce a tree"); + + assert!(!tree.root_node().has_error()); + } + + #[test] + fn parses_xml_without_errors() { + let source = "\n