Adding support for global settings (#1003)

[comment]: # (Note that your PR title should follow the conventional
commit format: https://conventionalcommits.org/en/v1.0.0/#summary)
# PR Description
Adding support for global settings

[comment]: # (The below checklist is for PRs adding new features. If a
box is not checked, add a reason why it's not needed.)
# New Feature Checklist

- [ ] List telemetry added about the feature.
- [ ] Link to the one-pager about the feature.
- [ ] List any tasks necessary for release (3P docs, AKS RP chart
changes, etc.) after merging the PR.
- [ ] Attach results of scale and perf testing.

[comment]: # (The below checklist is for code changes. Not all boxes
necessarily need to be checked. Build, doc, and template changes do not
need to fill out the checklist.)
# Tests Checklist

- [ ] Have end-to-end Ginkgo tests been run on your cluster and passed?
To bootstrap your cluster to run the tests, follow [these
instructions](/otelcollector/test/README.md#bootstrap-a-dev-cluster-to-run-ginkgo-tests).
  - Labels used when running the tests on your cluster:
    - [ ] `operator`
    - [ ] `windows`
    - [ ] `arm64`
    - [ ] `arc-extension`
    - [ ] `fips`
- [ ] Have new tests been added? For features, have tests been added for
this feature? For fixes, is there a test that could have caught this
issue and could validate that the fix works?
  - [ ] Is a new scrape job needed?
- [ ] The scrape job was added to the folder
[test-cluster-yamls](/otelcollector/test/test-cluster-yamls/) in the
correct configmap or as a CR.
  - [ ] Was a new test label added?
- [ ] A string constant for the label was added to
[constants.go](/otelcollector/test/utils/constants.go).
- [ ] The label and description was added to the [test
README](/otelcollector/test/README.md).
- [ ] The label was added to this [PR
checklist](/.github/pull_request_template).
- [ ] The label was added as needed to
[testkube-test-crs.yaml](/otelcollector/test/testkube/testkube-test-crs.yaml).
  - [ ] Are additional API server permissions needed for the new tests?
- [ ] These permissions have been added to
[api-server-permissions.yaml](/otelcollector/test/testkube/api-server-permissions.yaml).
  - [ ] Was a new test suite (a new folder under `/tests`) added?
- [ ] The new test suite is included in
[testkube-test-crs.yaml](/otelcollector/test/testkube/testkube-test-crs.yaml).

otelcollector/main/main.go

@@ -147,6 +147,7 @@ func main() {
 			collectorConfig = "/opt/microsoft/otelcollector/ccp-collector-config-replicaset.yml"
 		} else {
 			collectorConfig = "/opt/microsoft/otelcollector/collector-config-replicaset.yml"
+			configmapsettings.SetGlobalSettingsInCollectorConfig()
 		}
 	} else if azmonUseDefaultPrometheusConfig == "true" {
 		fmt.Println("Starting otelcollector with only default scrape configs enabled")

otelcollector/prom-config-validator-builder/main.go

@@ -71,15 +71,15 @@ func setFatalErrorMessageAsEnvVar(message string) {
 	truncatedMessage = re.ReplaceAllString(truncatedMessage, "")
 
 	// Write env var to a file so it can be used by other processes
-	file, err := os.Create("/opt/microsoft/prom_config_validator_env_var")
+	file, err := os.OpenFile("/opt/microsoft/prom_config_validator_env_var", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
 	if err != nil {
-		log.Println("prom-config-validator::Unable to create file for prom_config_validator_env_var")
+		log.Println("prom-config-validator::Unable to open file - prom_config_validator_env_var")
 	}
 	setEnvVarString := fmt.Sprintf("export INVALID_CONFIG_FATAL_ERROR=\"%s\"\n", truncatedMessage)
 	if os.Getenv("OS_TYPE") != "linux" {
 		setEnvVarString = fmt.Sprintf("INVALID_CONFIG_FATAL_ERROR=%s\n", truncatedMessage)
 	}
-	_, err = file.WriteString(setEnvVarString)
+	_, err = file.Write([]byte(setEnvVarString))
 	if err != nil {
 		log.Println("prom-config-validator::Unable to write to the file prom_config_validator_env_var")
 	}
@@ -215,6 +215,30 @@ func generateOtelConfig(promFilePath string, outputFilePath string, otelConfigTe
 		return fmt.Errorf("unsupported features:\n\t%s", strings.Join(unsupportedFeatures, "\n\t"))
 	}
 
+	globalSettingsFromMergedOtelConfig := prometheusConfig["global"]
+
+	if globalSettingsFromMergedOtelConfig != nil {
+		globalSettings := globalSettingsFromMergedOtelConfig.(map[interface{}]interface{})
+		scrapeInterval := globalSettings["scrape_interval"]
+		if (len(globalSettings) > 1) || (len(globalSettings) == 1 && scrapeInterval != "15s") {
+			setEnvVarString := fmt.Sprintf("AZMON_GLOBAL_SETTINGS_CONFIGURED=true\n")
+			file, err := os.OpenFile("/opt/microsoft/prom_config_validator_env_var", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+			if err != nil {
+				log.Println("prom-config-validator::Unable to open file - prom_config_validator_env_var")
+			}
+			_, err = file.Write([]byte(setEnvVarString))
+			if err != nil {
+				log.Println("prom-config-validator::Unable to write to the file prom_config_validator_env_var")
+			}
+			file.Close()
+			if err != nil {
+				log.Println("prom-config-validator::Unable to close file prom_config_validator_env_var", err)
+			} else {
+				log.Printf("prom-config-validator::Successfully set env variables for global config in file prom_config_validator_env_var\n")
+			}
+		}
+	}
+
 	otelConfig.Receivers.Prometheus.Config = prometheusConfig
 
 	if os.Getenv("DEBUG_MODE_ENABLED") == "true" {

otelcollector/shared/configmap/mp/configmapparser.go

@@ -183,6 +183,8 @@ func Configmapparser() {
 					}
 				}
 				shared.SetEnvAndSourceBashrcOrPowershell("AZMON_USE_DEFAULT_PROMETHEUS_CONFIG", "true", true)
+			} else {
+				shared.SetEnvAndSourceBashrcOrPowershell("AZMON_SET_GLOBAL_SETTINGS", "true", true)
 			}
 		}
 	} else if _, err := os.Stat(mergedDefaultConfigPath); err == nil {
@@ -225,7 +227,7 @@ func Configmapparser() {
 			if len(parts) == 2 {
 				key := parts[0]
 				value := parts[1]
-				os.Setenv(key, value)
+				shared.SetEnvAndSourceBashrcOrPowershell(key, value, true)
 
 				// Write to envvars.env
 				fmt.Fprintf(envFile, "%s=%s\n", key, value)

otelcollector/shared/configmap/mp/tomlparser-set-global-config.go

@@ -0,0 +1,87 @@
+package configmapsettings
+
+import (
+	"fmt"
+	"log"
+	"os"
+
+	yaml "gopkg.in/yaml.v2"
+)
+
+type OtelConfig struct {
+	Exporters  interface{} `yaml:"exporters"`
+	Processors interface{} `yaml:"processors"`
+	Extensions interface{} `yaml:"extensions"`
+	Receivers  struct {
+		Prometheus struct {
+			Config          map[string]interface{} `yaml:"config"`
+			TargetAllocator interface{}            `yaml:"target_allocator"`
+		} `yaml:"prometheus"`
+	} `yaml:"receivers"`
+	Service struct {
+		Extensions interface{} `yaml:"extensions"`
+		Pipelines  struct {
+			Metrics struct {
+				Exporters  interface{} `yaml:"exporters"`
+				Processors interface{} `yaml:"processors"`
+				Receivers  interface{} `yaml:"receivers"`
+			} `yaml:"metrics"`
+		} `yaml:"pipelines"`
+		Telemetry struct {
+			Logs struct {
+				Level    interface{} `yaml:"level"`
+				Encoding interface{} `yaml:"encoding"`
+			} `yaml:"logs"`
+		} `yaml:"telemetry"`
+	} `yaml:"service"`
+}
+
+func SetGlobalSettingsInCollectorConfig() {
+	azmonSetGlobalSettings := os.Getenv("AZMON_SET_GLOBAL_SETTINGS")
+
+	if azmonSetGlobalSettings == "true" {
+		mergedCollectorConfigPath := "/opt/microsoft/otelcollector/collector-config.yml"
+		mergedCollectorConfigFileContents, err := os.ReadFile(mergedCollectorConfigPath)
+		if err != nil {
+			fmt.Printf("Unable to read file contents from: %s - %v\n", mergedCollectorConfigPath, err)
+			return
+		}
+		var promScrapeConfig map[string]interface{}
+		var otelConfig OtelConfig
+		err = yaml.Unmarshal([]byte(mergedCollectorConfigFileContents), &otelConfig)
+		if err != nil {
+			fmt.Printf("Unable to unmarshal merged otel configuration from: %s - %v\n", mergedCollectorConfigFileContents, err)
+			return
+		}
+
+		promScrapeConfig = otelConfig.Receivers.Prometheus.Config
+		globalSettingsFromMergedOtelConfig := promScrapeConfig["global"]
+
+		if globalSettingsFromMergedOtelConfig != nil {
+			fmt.Println("Found global settings in merged otel config, triyng to replace replicaset collector config")
+			collectorConfigReplicasetPath := "/opt/microsoft/otelcollector/collector-config-replicaset.yml"
+			replicasetCollectorConfigFileContents, err := os.ReadFile(collectorConfigReplicasetPath)
+			if err != nil {
+				fmt.Printf("Unable to read file contents from: %s - %v\n", replicasetCollectorConfigFileContents, err)
+				return
+			}
+			var otelConfigReplicaset OtelConfig
+			err = yaml.Unmarshal([]byte(replicasetCollectorConfigFileContents), &otelConfigReplicaset)
+			if err != nil {
+				fmt.Printf("Unable to unmarshal merged otel configuration from: %s - %v\n", replicasetCollectorConfigFileContents, err)
+				return
+			}
+			otelConfigReplicaset.Receivers.Prometheus.Config = map[string]interface{}{"global": ""}
+			otelConfigReplicaset.Receivers.Prometheus.Config["global"] = globalSettingsFromMergedOtelConfig
+			otelReplacedConfigYaml, _ := yaml.Marshal(otelConfigReplicaset)
+			if err := os.WriteFile(collectorConfigReplicasetPath, otelReplacedConfigYaml, 0644); err != nil {
+				fmt.Printf("Unable to write to: %s - %v\n", collectorConfigReplicasetPath, err)
+				return
+			}
+
+			log.Println("Updated file with global settings", collectorConfigReplicasetPath)
+			return
+		}
+		fmt.Println("Global settings are empty in custom config map, making no replacement")
+	}
+}

otelcollector/telegraf/telegraf-prometheus-collector-ds.conf

@@ -33,6 +33,8 @@
   podname = "$POD_NAME"
   ostype = "$OS_TYPE"
   mip="$MINIMAL_INGESTION_PROFILE"
+  setGlobalSettings = "$AZMON_SET_GLOBAL_SETTINGS"
+  globalSettingsConfigured = "$AZMON_GLOBAL_SETTINGS_CONFIGURED"
 
 # Configuration for telegraf agent
 [agent]

otelcollector/telegraf/telegraf-prometheus-collector-ta-enabled.conf

@@ -211,6 +211,8 @@
     httpproxyenabled = "$HTTP_PROXY_ENABLED"
     tadapterh="$tokenadapterHealthyAfterSecs"
     tadapterf="$tokenadapterUnhealthyAfterSecs"
+    setGlobalSettings="$AZMON_SET_GLOBAL_SETTINGS"
+    globalSettingsConfigured="$AZMON_GLOBAL_SETTINGS_CONFIGURED"
 
 [[inputs.procstat]]
    exe = "MetricsExtension"

otelcollector/telegraf/telegraf-prometheus-collector-windows.conf

@@ -35,6 +35,9 @@
   ostype = "$OS_TYPE"
   debugmodeenabled = "$DEBUG_MODE_ENABLED"
   windowsVersion = "$windowsVersion"
+  setGlobalSettings = "$AZMON_SET_GLOBAL_SETTINGS"
+  globalSettingsConfigured = "$AZMON_GLOBAL_SETTINGS_CONFIGURED"
+
 
 # Configuration for telegraf agent
 [agent]