Make recommended keywords an opt-out option in terraform (#2667)

* Make recommended keywords an opt-out option in terraform * trigger ci * Run make docs * move error block up * Remove logs-app as codeowner of sds * Add cassette --------- Co-authored-by: HantingZhang2 <[email protected]>
DataDog · Nov 14, 2024 · 91276c1 · 91276c1
1 parent bc863b7
commit 91276c1
Show file tree

Hide file tree

Showing 8 changed files with 308 additions and 150 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -38,7 +38,7 @@ datadog/**/*datadog_integration_confluent*      @DataDog/api-clients @DataDog/sa
 datadog/**/*datadog_integration_fastly*         @DataDog/api-clients @DataDog/saas-integrations
 datadog/**/*datadog_integration_gcp*            @DataDog/api-clients @DataDog/gcp-integrations
 datadog/**/*datadog_restriction_policy*         @DataDog/api-clients @DataDog/aaa-granular-access
-datadog/**/*datadog_sensitive_data_scanner*     @DataDog/api-clients @DataDog/logs-app @DataDog/sensitive-data-scanner
+datadog/**/*datadog_sensitive_data_scanner*     @DataDog/api-clients @DataDog/sensitive-data-scanner
 datadog/**/*datadog_service_account*            @DataDog/api-clients @DataDog/team-aaa
 datadog/**/*datadog_spans_metric*               @DataDog/api-clients @DataDog/apm-trace-intake
 datadog/**/*datadog_synthetics_concurrency_cap* @DataDog/api-clients @DataDog/synthetics-app @DataDog/synthetics-ct

diff --git a/datadog/resource_datadog_sensitive_data_scanner_rule.go b/datadog/resource_datadog_sensitive_data_scanner_rule.go
@@ -80,7 +80,7 @@ func resourceDatadogSensitiveDataScannerRule() *schema.Resource {
 					Optional:    true,
 					MaxItems:    1,
 					ForceNew:    true, // If the attribute is removed, we need to recreate the rule.
-					Description: "Object defining a set of keywords and a number of characters that help reduce noise. You can provide a list of keywords you would like to check within a defined proximity of the matching pattern. If any of the keywords are found within the proximity check then the match is kept. If none are found, the match is discarded. Setting the `create_before_destroy` lifecycle Meta-argument to `true` is highly recommended if modifying this field to avoid unexpectedly disabling Sensitive Data Scanner groups.",
+					Description: "Object defining a set of keywords and a number of characters that help reduce noise. You can provide a list of keywords you would like to check within a defined proximity of the matching pattern. If any of the keywords are found within the proximity check then the match is kept. If none are found, the match is discarded. If the rule has the `standard_pattern_id` field, then discarding this field will apply the recommended keywords. Setting the `create_before_destroy` lifecycle Meta-argument to `true` is highly recommended if modifying this field to avoid unexpectedly disabling Sensitive Data Scanner groups.",
 					Elem: &schema.Resource{
 						Schema: map[string]*schema.Schema{
 							"keywords": {
@@ -303,9 +303,11 @@ func buildSensitiveDataScannerRuleAttributes(d *schema.ResourceData) *datadogV2.
 
 	attributes.SetTextReplacement(textReplacement)
 
-	if _, ok := d.GetOk("included_keyword_configuration"); ok {
-		var includedKeywordConfiguration datadogV2.SensitiveDataScannerIncludedKeywordConfiguration
+	var includedKeywordConfiguration datadogV2.SensitiveDataScannerIncludedKeywordConfiguration
 
+	_, hasSp := d.GetOk("standard_pattern_id")
+	if _, ok := d.GetOk("included_keyword_configuration"); ok {
+		// The user is creating a rule with an included keyword configuration specified. Let's simply build that object
 		keywords := []string{}
 		for _, kw := range d.Get("included_keyword_configuration.0.keywords").([]interface{}) {
 			keywords = append(keywords, kw.(string))
@@ -315,6 +317,18 @@ func buildSensitiveDataScannerRuleAttributes(d *schema.ResourceData) *datadogV2.
 		if characterCount, ok := d.GetOk("included_keyword_configuration.0.character_count"); ok {
 			includedKeywordConfiguration.SetCharacterCount(int64(characterCount.(int)))
 		}
+		if hasSp {
+			// If the user creates a rule derived from a standard rule, let's add that the rule is not using the recommended keywords.
+			includedKeywordConfiguration.SetUseRecommendedKeywords(false)
+		}
+		attributes.SetIncludedKeywordConfiguration(includedKeywordConfiguration)
+	} else if hasSp {
+		// The user is creating / updating a rule derived from a standard rule, without specifying an included keyword configuration.
+		// Let's use the recommended keywords here by default.
+		keywords := make([]string, 0)
+		includedKeywordConfiguration.SetKeywords(keywords)
+		includedKeywordConfiguration.SetCharacterCount(int64(30))
+		includedKeywordConfiguration.SetUseRecommendedKeywords(true)
 
 		attributes.SetIncludedKeywordConfiguration(includedKeywordConfiguration)
 	}
@@ -418,18 +432,24 @@ func updateSensitiveDataScannerRuleState(d *schema.ResourceData, ruleAttributes
 	}
 
 	if incKw, ok := ruleAttributes.GetIncludedKeywordConfigurationOk(); ok && incKw != nil {
-		includedKeywordConfig := make(map[string]interface{})
-		includedKeywordConfigList := make([]map[string]interface{}, 0, 1)
+		if _, hasSp := d.GetOk("standard_pattern_id"); hasSp && incKw.GetUseRecommendedKeywords() {
+			// This situation occurs when the rule is derived from a standard pattern, and that uses the recommended keywords.
+			// In that case, we shouldn't do anything because it means the user has chosen the default option.
+		} else {
+			includedKeywordConfig := make(map[string]interface{})
+			includedKeywordConfigList := make([]map[string]interface{}, 0, 1)
 
-		if keywords, ok := incKw.GetKeywordsOk(); ok {
-			includedKeywordConfig["keywords"] = keywords
-		}
-		if characterCount, ok := incKw.GetCharacterCountOk(); ok {
-			includedKeywordConfig["character_count"] = characterCount
-		}
-		includedKeywordConfigList = append(includedKeywordConfigList, includedKeywordConfig)
-		if err := d.Set("included_keyword_configuration", includedKeywordConfigList); err != nil {
-			return diag.FromErr(err)
+			if keywords, ok := incKw.GetKeywordsOk(); ok {
+				includedKeywordConfig["keywords"] = keywords
+			}
+			if characterCount, ok := incKw.GetCharacterCountOk(); ok {
+				includedKeywordConfig["character_count"] = characterCount
+			}
+			includedKeywordConfigList = append(includedKeywordConfigList, includedKeywordConfig)
+			if err := d.Set("included_keyword_configuration", includedKeywordConfigList); err != nil {
+				return diag.FromErr(err)
+
+			}
 		}
 	}
 

diff --git a/datadog/tests/cassettes/TestAccSensitiveDataScannerRuleBasic.freeze b/datadog/tests/cassettes/TestAccSensitiveDataScannerRuleBasic.freeze
@@ -1 +1 @@
-2024-04-22T11:53:12.603353+02:00
+2024-11-13T15:46:07.246591+01:00
diff --git a/datadog/tests/cassettes/TestAccSensitiveDataScannerRuleBasic.yaml b/datadog/tests/cassettes/TestAccSensitiveDataScannerRuleBasic.yaml
diff --git a/datadog/tests/resource_datadog_sensitive_data_scanner_rule_test.go b/datadog/tests/resource_datadog_sensitive_data_scanner_rule_test.go
@@ -71,6 +71,7 @@ func TestAccSensitiveDataScannerRuleBasic(t *testing.T) {
 						resource_name, "included_keyword_configuration.0.character_count", "20"),
 					resource.TestCheckResourceAttr(
 						resource_name, "priority", "1"),
+					testAccCheckDatadogSensitiveDataScannerRuleRecommendedKeywords(accProvider, resource_name, nil),
 				),
 			},
 			{
@@ -132,34 +133,53 @@ func TestAccSensitiveDataScannerRuleWithStandardPattern(t *testing.T) {
 	}
 
 	ctx, accProviders := testAccProviders(context.Background(), t)
-	uniq := uniqueEntityName(ctx, t)
+	uniq1 := uniqueEntityName(ctx, t)
+	uniq2 := uniqueEntityName(ctx, t)
 	accProvider := testAccProvider(t, accProviders)
 
-	resource_name := "datadog_sensitive_data_scanner_rule.another_rule"
+	resource_name_1 := "datadog_sensitive_data_scanner_rule.sp_rule_1"
+	resource_name_2 := "datadog_sensitive_data_scanner_rule.sp_rule_2"
+
+	value_true := true
+	value_false := false
 
 	resource.Test(t, resource.TestCase{
 		PreCheck:          func() { testAccPreCheck(t) },
 		ProviderFactories: accProviders,
 		CheckDestroy:      testAccCheckDatadogSensitiveDataScannerRuleDestroy(accProvider),
 		Steps: []resource.TestStep{
 			{
-				Config: testAccCheckDatadogSensitiveDataScannerRuleWithStandardPattern(uniq),
+				Config: testAccCheckDatadogSensitiveDataScannerRuleWithStandardPattern(uniq1, uniq2),
 				Check: resource.ComposeTestCheckFunc(
-					testAccCheckDatadogSensitiveDataScannerRuleExists(accProvider, resource_name),
+					testAccCheckDatadogSensitiveDataScannerRuleExists(accProvider, resource_name_1),
 					resource.TestCheckResourceAttr(
-						resource_name, "description", "a description"),
+						resource_name_1, "description", "a description"),
 					resource.TestCheckResourceAttr(
-						resource_name, "is_enabled", "true"),
+						resource_name_1, "is_enabled", "true"),
 					resource.TestCheckResourceAttr(
-						resource_name, "name", uniq),
+						resource_name_1, "name", uniq1),
 					resource.TestCheckResourceAttr(
-						resource_name, "excluded_namespaces.0", "username"),
+						resource_name_1, "excluded_namespaces.0", "username"),
 					resource.TestCheckResourceAttr(
-						resource_name, "text_replacement.0.number_of_chars", "10"),
+						resource_name_1, "text_replacement.0.number_of_chars", "10"),
 					resource.TestCheckResourceAttr(
-						resource_name, "text_replacement.0.type", "partial_replacement_from_beginning"),
+						resource_name_1, "text_replacement.0.type", "partial_replacement_from_beginning"),
 					resource.TestCheckResourceAttr(
-						resource_name, "text_replacement.0.replacement_string", ""),
+						resource_name_1, "text_replacement.0.replacement_string", ""),
+					resource.TestCheckResourceAttr(
+						resource_name_1, "included_keyword_configuration.0.keywords.0", "credit"),
+					resource.TestCheckResourceAttr(
+						resource_name_1, "included_keyword_configuration.0.character_count", "20"),
+					testAccCheckDatadogSensitiveDataScannerRuleRecommendedKeywords(accProvider, resource_name_1, &value_false),
+					// assertions on resource 2
+					testAccCheckDatadogSensitiveDataScannerRuleExists(accProvider, resource_name_2),
+					resource.TestCheckResourceAttr(
+						resource_name_2, "description", "a description"),
+					resource.TestCheckResourceAttr(
+						resource_name_2, "is_enabled", "true"),
+					resource.TestCheckResourceAttr(
+						resource_name_2, "name", uniq2),
+					testAccCheckDatadogSensitiveDataScannerRuleRecommendedKeywords(accProvider, resource_name_2, &value_true),
 				),
 			},
 		}})
@@ -295,7 +315,7 @@ resource "datadog_sensitive_data_scanner_rule" "sample_rule" {
 `, name)
 }
 
-func testAccCheckDatadogSensitiveDataScannerRuleWithStandardPattern(name string) string {
+func testAccCheckDatadogSensitiveDataScannerRuleWithStandardPattern(name1, name2 string) string {
 	return fmt.Sprintf(`
 resource "datadog_sensitive_data_scanner_group" "sample_group" {
 	name = "my group"
@@ -310,7 +330,7 @@ data "datadog_sensitive_data_scanner_standard_pattern" "sample_sp" {
 	filter = "AWS Access Key ID Scanner"
 }
 
-resource "datadog_sensitive_data_scanner_rule" "another_rule" {
+resource "datadog_sensitive_data_scanner_rule" "sp_rule_1" {
 	name = "%s"
 	description = "a description"
 	excluded_namespaces = ["username"]
@@ -322,8 +342,21 @@ resource "datadog_sensitive_data_scanner_rule" "another_rule" {
 		replacement_string = ""
 		type = "partial_replacement_from_beginning"
 	}
+	included_keyword_configuration {
+		keywords = ["credit"]
+		character_count = 20
+	}
 }
-`, name)
+
+resource "datadog_sensitive_data_scanner_rule" "sp_rule_2" {
+	name = "%s"
+	description = "a description"
+	excluded_namespaces = ["username"]
+	is_enabled = true
+	group_id = datadog_sensitive_data_scanner_group.sample_group.id
+	standard_pattern_id = data.datadog_sensitive_data_scanner_standard_pattern.sample_sp.id
+}
+`, name1, name2)
 }
 
 func testAccCheckDatadogSensitiveDataScannerRuleDestroy(accProvider func() (*schema.Provider, error)) func(*terraform.State) error {
@@ -336,10 +369,10 @@ func testAccCheckDatadogSensitiveDataScannerRuleDestroy(accProvider func() (*sch
 		for _, resource := range s.RootModule().Resources {
 			if resource.Type == "datadog_sensitive_data_scanner_rule" {
 				resp, _, err := apiInstances.GetSensitiveDataScannerApiV2().ListScanningGroups(auth)
+				if err != nil {
+					return fmt.Errorf("received an error retrieving all scanning groups: %s", err)
+				}
 				if ruleFound := findSensitiveDataScannerRuleHelper(resource.Primary.ID, resp); ruleFound == nil {
-					if err != nil {
-						return fmt.Errorf("received an error retrieving all scanning groups: %s", err)
-					}
 					return nil
 				}
 				return fmt.Errorf("scanning rule still exists")
@@ -379,3 +412,33 @@ func testAccCheckDatadogSensitiveDataScannerRuleExists(accProvider func() (*sche
 		return nil
 	}
 }
+
+func testAccCheckDatadogSensitiveDataScannerRuleRecommendedKeywords(accProvider func() (*schema.Provider, error), name string, expectedUseRecommendedKeywords *bool) resource.TestCheckFunc {
+	return func(s *terraform.State) error {
+		provider, _ := accProvider()
+		providerConf := provider.Meta().(*datadog.ProviderConfiguration)
+		apiInstances := providerConf.DatadogApiInstances
+		auth := providerConf.Auth
+
+		ruleId := s.RootModule().Resources[name].Primary.ID
+		resp, _, err := apiInstances.GetSensitiveDataScannerApiV2().ListScanningGroups(auth)
+		if err != nil {
+			return fmt.Errorf("received an error retrieving the list of scanning groups, %s", err)
+		}
+
+		ruleFound := findSensitiveDataScannerRuleHelper(ruleId, resp)
+		if ruleFound == nil {
+			return fmt.Errorf("received an error retrieving scanning group")
+		}
+
+		actualUseRecommendedKeywords := ruleFound.Attributes.IncludedKeywordConfiguration.UseRecommendedKeywords
+		if expectedUseRecommendedKeywords == nil && actualUseRecommendedKeywords == nil {
+			return nil
+		}
+		if *actualUseRecommendedKeywords != *expectedUseRecommendedKeywords {
+			return fmt.Errorf("actual use_recommended_keywords: %v. expected: %v", actualUseRecommendedKeywords, expectedUseRecommendedKeywords)
+		}
+
+		return nil
+	}
+}
diff --git a/docs/resources/sensitive_data_scanner_rule.md b/docs/resources/sensitive_data_scanner_rule.md
@@ -54,7 +54,7 @@ data "datadog_sensitive_data_scanner_standard_pattern" "aws_sp" {
   filter = "AWS Access Key ID Scanner"
 }
 
-resource "datadog_sensitive_data_scanner_rule" "mylibraryrule" {
+resource "datadog_sensitive_data_scanner_rule" "mylibraryrule_with_custom_included_keywords" {
   name        = "My library rule"
   description = "A description"
   group_id    = datadog_sensitive_data_scanner_group.mygroup.id
@@ -63,6 +63,26 @@ resource "datadog_sensitive_data_scanner_rule" "mylibraryrule" {
   excluded_namespaces = ["username"]
   is_enabled          = true
   tags                = ["sensitive_data:true"]
+
+  // SDS will set the recommended keywords by default. If the user doesn't want to use the recommended keywords,
+  // they have to create an empty included keyword configuration (with empty keywords)
+  included_keyword_configuration {
+    keywords        = ["cc", "credit card"]
+    character_count = 30
+  }
+}
+
+resource "datadog_sensitive_data_scanner_rule" "mylibraryrule_with_recommended_keywords" {
+  name        = "My library rule"
+  description = "A description"
+  group_id    = datadog_sensitive_data_scanner_group.mygroup.id
+  // As standard_pattern_id is provided, the resource MUST NOT contain the "pattern" attribute
+  standard_pattern_id = data.datadog_sensitive_data_scanner_standard_pattern.aws_sp.id
+  excluded_namespaces = ["username"]
+  is_enabled          = true
+  tags                = ["sensitive_data:true"]
+
+  // SDS will set the recommended keywords by default.
 }
 ```
 
@@ -77,7 +97,7 @@ resource "datadog_sensitive_data_scanner_rule" "mylibraryrule" {
 
 - `description` (String) Description of the rule.
 - `excluded_namespaces` (List of String) Attributes excluded from the scan. If namespaces is provided, it has to be a sub-path of the namespaces array.
-- `included_keyword_configuration` (Block List, Max: 1) Object defining a set of keywords and a number of characters that help reduce noise. You can provide a list of keywords you would like to check within a defined proximity of the matching pattern. If any of the keywords are found within the proximity check then the match is kept. If none are found, the match is discarded. Setting the `create_before_destroy` lifecycle Meta-argument to `true` is highly recommended if modifying this field to avoid unexpectedly disabling Sensitive Data Scanner groups. (see [below for nested schema](#nestedblock--included_keyword_configuration))
+- `included_keyword_configuration` (Block List, Max: 1) Object defining a set of keywords and a number of characters that help reduce noise. You can provide a list of keywords you would like to check within a defined proximity of the matching pattern. If any of the keywords are found within the proximity check then the match is kept. If none are found, the match is discarded. If the rule has the `standard_pattern_id` field, then discarding this field will apply the recommended keywords. Setting the `create_before_destroy` lifecycle Meta-argument to `true` is highly recommended if modifying this field to avoid unexpectedly disabling Sensitive Data Scanner groups. (see [below for nested schema](#nestedblock--included_keyword_configuration))
 - `is_enabled` (Boolean) Whether or not the rule is enabled.
 - `name` (String) Name of the rule.
 - `namespaces` (List of String) Attributes included in the scan. If namespaces is empty or missing, all attributes except excluded_namespaces are scanned. If both are missing the whole event is scanned.

diff --git a/examples/resources/datadog_sensitive_data_scanner_rule/resource.tf b/examples/resources/datadog_sensitive_data_scanner_rule/resource.tf
@@ -39,7 +39,7 @@ data "datadog_sensitive_data_scanner_standard_pattern" "aws_sp" {
   filter = "AWS Access Key ID Scanner"
 }
 
-resource "datadog_sensitive_data_scanner_rule" "mylibraryrule" {
+resource "datadog_sensitive_data_scanner_rule" "mylibraryrule_with_custom_included_keywords" {
   name        = "My library rule"
   description = "A description"
   group_id    = datadog_sensitive_data_scanner_group.mygroup.id
@@ -48,4 +48,24 @@ resource "datadog_sensitive_data_scanner_rule" "mylibraryrule" {
   excluded_namespaces = ["username"]
   is_enabled          = true
   tags                = ["sensitive_data:true"]
+
+  // SDS will set the recommended keywords by default. If the user doesn't want to use the recommended keywords,
+  // they have to create an empty included keyword configuration (with empty keywords)
+  included_keyword_configuration {
+    keywords        = ["cc", "credit card"]
+    character_count = 30
+  }
+}
+
+resource "datadog_sensitive_data_scanner_rule" "mylibraryrule_with_recommended_keywords" {
+  name        = "My library rule"
+  description = "A description"
+  group_id    = datadog_sensitive_data_scanner_group.mygroup.id
+  // As standard_pattern_id is provided, the resource MUST NOT contain the "pattern" attribute
+  standard_pattern_id = data.datadog_sensitive_data_scanner_standard_pattern.aws_sp.id
+  excluded_namespaces = ["username"]
+  is_enabled          = true
+  tags                = ["sensitive_data:true"]
+
+  // SDS will set the recommended keywords by default.
 }
diff --git a/go.mod b/go.mod
@@ -96,4 +96,4 @@ require (
 	google.golang.org/protobuf v1.33.0 // indirect
 )
 
-go 1.23
+go 1.23.0