Add support for API Security Custom Data Classification

dd-java-agent/appsec/build.gradle

@@ -15,7 +15,7 @@ dependencies {
   implementation project(':internal-api')
   implementation project(':communication')
   implementation project(':telemetry')
-  implementation group: 'io.sqreen', name: 'libsqreen', version: '17.1.0'
+  implementation group: 'io.sqreen', name: 'libsqreen', version: '17.2.0'
   implementation libs.moshi
 
   testImplementation libs.bytebuddy

dd-java-agent/appsec/src/main/java/com/datadog/appsec/config/AppSecConfigServiceImpl.java

@@ -4,6 +4,7 @@
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_ACTIVATION;
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_AUTO_USER_INSTRUM_MODE;
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_CUSTOM_BLOCKING_RESPONSE;
+import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_CUSTOM_DATA_SCANNERS;
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_CUSTOM_RULES;
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_DD_MULTICONFIG;
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_DD_RULES;
@@ -13,6 +14,7 @@
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_HEADER_FINGERPRINT;
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_IP_BLOCKING;
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_NETWORK_FINGERPRINT;
+import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_PROCESSOR_OVERRIDES;
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_RASP_CMDI;
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_RASP_LFI;
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_RASP_SHI;
@@ -147,6 +149,8 @@ private long getRulesAndDataCapabilities() {
             | CAPABILITY_ASM_CUSTOM_RULES
             | CAPABILITY_ASM_CUSTOM_BLOCKING_RESPONSE
             | CAPABILITY_ASM_TRUSTED_IPS
+            | CAPABILITY_ASM_PROCESSOR_OVERRIDES
+            | CAPABILITY_ASM_CUSTOM_DATA_SCANNERS
             | CAPABILITY_ENDPOINT_FINGERPRINT
             | CAPABILITY_ASM_SESSION_FINGERPRINT
             | CAPABILITY_ASM_NETWORK_FINGERPRINT
@@ -525,6 +529,8 @@ public void close() {
             | CAPABILITY_ASM_CUSTOM_RULES
             | CAPABILITY_ASM_CUSTOM_BLOCKING_RESPONSE
             | CAPABILITY_ASM_TRUSTED_IPS
+            | CAPABILITY_ASM_PROCESSOR_OVERRIDES
+            | CAPABILITY_ASM_CUSTOM_DATA_SCANNERS
             | CAPABILITY_ASM_RASP_SQLI
             | CAPABILITY_ASM_RASP_SSRF
             | CAPABILITY_ASM_RASP_LFI

dd-java-agent/appsec/src/test/groovy/com/datadog/appsec/config/AppSecConfigServiceImplSpecification.groovy

@@ -24,25 +24,27 @@ import java.nio.file.Paths
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_ACTIVATION
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_AUTO_USER_INSTRUM_MODE
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_CUSTOM_BLOCKING_RESPONSE
+import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_CUSTOM_DATA_SCANNERS
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_CUSTOM_RULES
+import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_DD_MULTICONFIG
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_DD_RULES
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_EXCLUSIONS
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_EXCLUSION_DATA
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_HEADER_FINGERPRINT
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_IP_BLOCKING
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_NETWORK_FINGERPRINT
+import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_PROCESSOR_OVERRIDES
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_RASP_CMDI
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_RASP_LFI
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_RASP_SHI
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_RASP_SQLI
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_RASP_SSRF
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_REQUEST_BLOCKING
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_SESSION_FINGERPRINT
+import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_TRACE_TAGGING_RULES
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_TRUSTED_IPS
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_USER_BLOCKING
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ENDPOINT_FINGERPRINT
-import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_DD_MULTICONFIG
-import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_TRACE_TAGGING_RULES
 import static datadog.remoteconfig.PollingHinterNoop.NOOP
 import static datadog.trace.api.UserIdCollectionMode.ANONYMIZATION
 import static datadog.trace.api.UserIdCollectionMode.DISABLED
@@ -282,6 +284,8 @@ class AppSecConfigServiceImplSpecification extends DDSpecification {
       | CAPABILITY_ASM_CUSTOM_RULES
       | CAPABILITY_ASM_CUSTOM_BLOCKING_RESPONSE
       | CAPABILITY_ASM_TRUSTED_IPS
+      | CAPABILITY_ASM_PROCESSOR_OVERRIDES
+      | CAPABILITY_ASM_CUSTOM_DATA_SCANNERS
       | CAPABILITY_ASM_RASP_SQLI
       | CAPABILITY_ASM_RASP_SSRF
       | CAPABILITY_ASM_RASP_CMDI
@@ -442,6 +446,8 @@ class AppSecConfigServiceImplSpecification extends DDSpecification {
       | CAPABILITY_ASM_CUSTOM_RULES
       | CAPABILITY_ASM_CUSTOM_BLOCKING_RESPONSE
       | CAPABILITY_ASM_TRUSTED_IPS
+      | CAPABILITY_ASM_PROCESSOR_OVERRIDES
+      | CAPABILITY_ASM_CUSTOM_DATA_SCANNERS
       | CAPABILITY_ENDPOINT_FINGERPRINT
       | CAPABILITY_ASM_SESSION_FINGERPRINT
       | CAPABILITY_ASM_NETWORK_FINGERPRINT
@@ -533,6 +539,8 @@ class AppSecConfigServiceImplSpecification extends DDSpecification {
       | CAPABILITY_ASM_CUSTOM_RULES
       | CAPABILITY_ASM_CUSTOM_BLOCKING_RESPONSE
       | CAPABILITY_ASM_TRUSTED_IPS
+      | CAPABILITY_ASM_PROCESSOR_OVERRIDES
+      | CAPABILITY_ASM_CUSTOM_DATA_SCANNERS
       | CAPABILITY_ASM_RASP_SQLI
       | CAPABILITY_ASM_RASP_SSRF
       | CAPABILITY_ASM_RASP_CMDI

dd-smoke-tests/appsec/springboot/src/test/groovy/datadog/smoketest/appsec/ProcessorOverridesSmokeTest.groovy

@@ -0,0 +1,202 @@
+package datadog.smoketest.appsec
+
+import datadog.trace.agent.test.utils.OkHttpUtils
+import okhttp3.FormBody
+import okhttp3.Request
+import spock.lang.Shared
+
+/**
+ * Smoke test to verify that processor_overrides and scanners configuration keys
+ * are forwarded to libddwaf without alteration.
+ *
+ * This test creates a custom configuration with:
+ * - A custom scanner that detects a specific pattern ("CUSTOM_TEST_PATTERN_12345")
+ * - Processor overrides to include the custom scanner in schema extraction
+ *
+ * By verifying that the custom scanner triggers detection, we confirm that both
+ * processor_overrides and scanners were correctly forwarded to libddwaf.
+ */
+class ProcessorOverridesSmokeTest extends AbstractAppSecServerSmokeTest {
+
+  @Shared
+  String buildDir = new File(System.getProperty("datadog.smoketest.builddir")).absolutePath
+  @Shared
+  String customRulesPath = "${buildDir}/appsec_processor_overrides_rules.json"
+
+  def prepareCustomConfiguration() {
+    // Prepare ruleset with custom scanners and processor overrides
+    // This configuration will be passed to libddwaf and should be forwarded without modification
+
+    def customConfig = [
+      version: "2.2",
+      metadata: [
+        rules_version: "1.0.0"
+      ],
+      rules: [
+        // Add a simple rule that triggers on custom scanner detection
+        [
+          id: 'custom-scanner-detection-rule',
+          name: 'Custom Scanner Detection Test Rule',
+          tags: [
+            type: 'security_scanner',
+            category: 'attack_attempt'
+          ],
+          conditions: [
+            [
+              parameters: [
+                inputs: [
+                  [
+                    address: 'server.request.body'
+                  ]
+                ],
+                regex: 'CUSTOM_TEST_PATTERN_12345'
+              ],
+              operator: 'match_regex'
+            ]
+          ],
+          transformers: [],
+          on_match: []
+        ]
+      ],
+      // Define custom scanners - this should be forwarded to libddwaf
+      scanners: [
+        [
+          id: 'custom-test-scanner',
+          name: 'Custom Test Scanner',
+          key: [
+            operator: 'match_regex',
+            parameters: [
+              regex: '.*'
+            ]
+          ],
+          value: [
+            operator: 'match_regex',
+            parameters: [
+              regex: 'CUSTOM_TEST_PATTERN_12345'
+            ]
+          ],
+          tags: [
+            type: 'custom_pattern',
+            category: 'test'
+          ]
+        ]
+      ],
+      // Define processor overrides - this should be forwarded to libddwaf
+      processor_overrides: [
+        [
+          target: [[id: 'extract-schema']],
+          scanners: [
+            include: [[id: 'custom-test-scanner']],
+            exclude: []
+          ]
+        ]
+      ]
+    ]
+
+    // Write the custom configuration to a file
+    def gen = new groovy.json.JsonGenerator.Options().build()
+    def configFile = new File(customRulesPath)
+    configFile.text = gen.toJson(customConfig)
+
+    // Point the agent to use this custom configuration
+    defaultAppSecProperties += "-Ddd.appsec.rules=${customRulesPath}" as String
+  }
+
+  @Override
+  ProcessBuilder createProcessBuilder() {
+    // Prepare custom configuration before starting the process
+    prepareCustomConfiguration()
+
+    String springBootShadowJar = System.getProperty("datadog.smoketest.appsec.springboot.shadowJar.path")
+
+    List<String> command = new ArrayList<>()
+    command.add(javaPath())
+    command.addAll(defaultJavaProperties)
+    command.addAll(defaultAppSecProperties)
+    command.addAll((String[]) ["-jar", springBootShadowJar, "--server.port=${httpPort}"])
+
+    ProcessBuilder processBuilder = new ProcessBuilder(command)
+    processBuilder.directory(new File(buildDirectory))
+  }
+
+  void 'test that custom scanner with processor_overrides is detected'() {
+    given: 'a request containing the custom pattern'
+    def url = "http://localhost:${httpPort}/greeting"
+    def client = OkHttpUtils.clientBuilder().build()
+    def formBuilder = new FormBody.Builder()
+    formBuilder.add('test', 'CUSTOM_TEST_PATTERN_12345')
+    def body = formBuilder.build()
+    def request = new Request.Builder()
+      .url(url)
+      .post(body)
+      .build()
+
+    when: 'the request is sent'
+    def response = client.newCall(request).execute()
+
+    then: 'the response is successful'
+    response.code() == 200
+
+    when: 'waiting for traces'
+    waitForTraceCount(1)
+
+    then: 'the custom scanner rule is triggered'
+    def rootSpans = this.rootSpans.toList()
+    rootSpans.size() == 1
+    def rootSpan = rootSpans[0]
+
+    // Verify that the custom rule was triggered, which confirms that:
+    // 1. The 'scanners' configuration was forwarded to libddwaf
+    // 2. The 'processor_overrides' configuration was forwarded to libddwaf
+    // 3. Libddwaf correctly processed both configurations
+    def trigger = null
+    for (t in rootSpan.triggers) {
+      if (t['rule']['id'] == 'custom-scanner-detection-rule') {
+        trigger = t
+        break
+      }
+    }
+    assert trigger != null, 'Custom scanner rule was not triggered - configuration may not have been forwarded correctly'
+
+    // Verify the trigger contains expected information
+    trigger['rule']['id'] == 'custom-scanner-detection-rule'
+    trigger['rule']['tags']['type'] == 'security_scanner'
+    trigger['rule']['tags']['category'] == 'attack_attempt'
+  }
+
+  void 'test that custom scanner does not trigger without pattern'() {
+    given: 'a request without the custom pattern'
+    def url = "http://localhost:${httpPort}/greeting"
+    def client = OkHttpUtils.clientBuilder().build()
+    def formBuilder = new FormBody.Builder()
+    formBuilder.add('test', 'normal_text_without_pattern')
+    def body = formBuilder.build()
+    def request = new Request.Builder()
+      .url(url)
+      .post(body)
+      .build()
+
+    when: 'the request is sent'
+    def response = client.newCall(request).execute()
+
+    then: 'the response is successful'
+    response.code() == 200
+
+    when: 'waiting for traces'
+    waitForTraceCount(1)
+
+    then: 'the custom scanner rule is NOT triggered'
+    def rootSpans = this.rootSpans.toList()
+    rootSpans.size() == 1
+    def rootSpan = rootSpans[0]
+
+    def trigger = null
+    for (t in rootSpan.triggers) {
+      if (t['rule']['id'] == 'custom-scanner-detection-rule') {
+        trigger = t
+        break
+      }
+    }
+    assert trigger == null, 'Custom scanner rule should not trigger without the pattern'
+  }
+}