Redact credentials from debug config logs

config.go

@@ -7,6 +7,7 @@ import (
 	"crypto/x509"
 	"fmt"
 	"net"
+	"net/url"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -212,6 +213,32 @@ func (c Config) Validate() error {
 	return nil
 }
 
+// Redacted returns a copy of the config safe for logging. It redacts inline
+// passwords without changing the config used to connect to monitored databases.
+func (c Config) Redacted() Config {
+	redacted := c
+	redacted.HTTP = c.HTTP.Redacted()
+	redacted.MySQL.Password = redactPassword(redacted.MySQL.Password)
+	redacted.Sinks = c.Sinks.redacted()
+
+	// ConfigMonitor and ConfigPlans can refer to each other through
+	// ConfigPlans.Monitor. Preserve shared references and cycles while making
+	// copies so redaction never changes the live config.
+	seen := map[*ConfigMonitor]*ConfigMonitor{}
+	if c.Monitors != nil {
+		redacted.Monitors = make([]ConfigMonitor, len(c.Monitors))
+		for i := range c.Monitors {
+			seen[&c.Monitors[i]] = &redacted.Monitors[i]
+		}
+		for i := range c.Monitors {
+			redacted.Monitors[i] = c.Monitors[i].redacted(seen)
+		}
+	}
+	redacted.Plans = c.Plans.redacted(seen)
+
+	return redacted
+}
+
 func (c *Config) InterpolateEnvVars() {
 	// Blip server
 	c.API.InterpolateEnvVars()
@@ -297,6 +324,21 @@ func (c *ConfigHTTP) InterpolateEnvVars() {
 	c.Proxy = interpolateEnv(c.Proxy)
 }
 
+// Redacted returns a copy of the HTTP config safe for logging.
+func (c ConfigHTTP) Redacted() ConfigHTTP {
+	if c.Proxy == "" {
+		return c
+	}
+
+	proxyURL, err := url.Parse(c.Proxy)
+	if err != nil {
+		c.Proxy = "..."
+		return c
+	}
+	c.Proxy = proxyURL.Redacted()
+	return c
+}
+
 func (c *ConfigHTTP) ApplyDefaults(b Config) {
 	if c.Proxy == "" {
 		c.Proxy = b.HTTP.Proxy
@@ -409,6 +451,18 @@ func (c ConfigMonitor) Validate() error {
 	return nil
 }
 
+// Redacted returns a copy of the monitor config safe for logging.
+func (c ConfigMonitor) Redacted() ConfigMonitor {
+	return c.redacted(map[*ConfigMonitor]*ConfigMonitor{})
+}
+
+func (c ConfigMonitor) redacted(seen map[*ConfigMonitor]*ConfigMonitor) ConfigMonitor {
+	c.Password = redactPassword(c.Password)
+	c.Sinks = c.Sinks.redacted()
+	c.Plans = c.Plans.redacted(seen)
+	return c
+}
+
 func (c *ConfigMonitor) ApplyDefaults(b Config) {
 	if c.Socket == "" {
 		c.Socket = b.MySQL.Socket
@@ -810,12 +864,17 @@ func (c *ConfigMySQL) InterpolateMonitor(m *ConfigMonitor) {
 }
 
 func (c ConfigMySQL) Redacted() string {
-	if c.Password != "" {
-		c.Password = "..."
-	}
+	c.Password = redactPassword(c.Password)
 	return fmt.Sprintf("%+v", c)
 }
 
+func redactPassword(password string) string {
+	if password == "" {
+		return ""
+	}
+	return "..."
+}
+
 // --------------------------------------------------------------------------
 
 type ConfigPlans struct {
@@ -838,6 +897,22 @@ func (c ConfigPlans) Validate() error {
 	return nil
 }
 
+func (c ConfigPlans) redacted(seen map[*ConfigMonitor]*ConfigMonitor) ConfigPlans {
+	if c.Monitor == nil {
+		return c
+	}
+	if redacted, ok := seen[c.Monitor]; ok {
+		c.Monitor = redacted
+		return c
+	}
+
+	redacted := &ConfigMonitor{}
+	seen[c.Monitor] = redacted
+	*redacted = c.Monitor.redacted(seen)
+	c.Monitor = redacted
+	return c
+}
+
 func (c *ConfigPlans) ApplyDefaults(b Config) {
 	if len(c.Files) == 0 && len(b.Plans.Files) > 0 {
 		c.Files = make([]string, len(b.Plans.Files))
@@ -936,6 +1011,26 @@ func (c ConfigPlanChange) Enabled() bool {
 
 type ConfigSinks map[string]map[string]string
 
+func (c ConfigSinks) redacted() ConfigSinks {
+	if c == nil {
+		return nil
+	}
+
+	redacted := make(ConfigSinks, len(c))
+	for sinkName, options := range c {
+		if options == nil {
+			redacted[sinkName] = nil
+			continue
+		}
+
+		redacted[sinkName] = make(map[string]string, len(options))
+		for optionName := range options {
+			redacted[sinkName][optionName] = "..."
+		}
+	}
+	return redacted
+}
+
 func DefaultConfigSinks() ConfigSinks {
 	return ConfigSinks{}
 }

config_test.go

@@ -52,6 +52,208 @@ func TestEnvInterpolationEmpty(t *testing.T) {
 	assert.Equal(t, "bar", cfg.MySQL.Password)
 }
 
+func TestConfigRedacted(t *testing.T) {
+	envKey := "blip_test_TestConfigRedacted"
+	envPassword := "env-monitor-password"
+	t.Setenv(envKey, envPassword)
+
+	cfg := blip.Config{
+		API: blip.ConfigAPI{Bind: "127.0.0.1:7522"},
+		MySQL: blip.ConfigMySQL{
+			Hostname: "defaults.example:3306",
+			Username: "default-user",
+			Password: "default-password",
+		},
+		Monitors: []blip.ConfigMonitor{
+			{
+				MonitorId: "inline-monitor",
+				Hostname:  "inline.example:3306",
+				Username:  "inline-user",
+				Password:  "inline-monitor-password",
+			},
+			{
+				MonitorId: "env-monitor",
+				Hostname:  "env.example:3306",
+				Username:  "env-user",
+				Password:  fmt.Sprintf("${%s}", envKey),
+			},
+		},
+		Plans: blip.ConfigPlans{
+			Table: "blip.plans",
+			Monitor: &blip.ConfigMonitor{
+				MonitorId: "plan-monitor",
+				Hostname:  "plans.example:3306",
+				Username:  "plan-user",
+				Password:  "plan-monitor-password",
+			},
+		},
+	}
+	cfg.Monitors[1].InterpolateEnvVars()
+
+	redacted := cfg.Redacted()
+	dump := fmt.Sprintf("%#v", redacted)
+
+	for _, password := range []string{
+		"default-password",
+		"inline-monitor-password",
+		envPassword,
+		"plan-monitor-password",
+	} {
+		assert.NotContains(t, dump, password)
+	}
+	assert.Equal(t, "...", redacted.MySQL.Password)
+	assert.Equal(t, "...", redacted.Monitors[0].Password)
+	assert.Equal(t, "...", redacted.Monitors[1].Password)
+	assert.Equal(t, "...", redacted.Plans.Monitor.Password)
+
+	// Redaction retains useful non-secret context for debug logging.
+	assert.Contains(t, dump, "127.0.0.1:7522")
+	assert.Contains(t, dump, "inline-monitor")
+	assert.Contains(t, dump, "inline.example:3306")
+	assert.Contains(t, dump, "inline-user")
+	assert.Contains(t, dump, "blip.plans")
+
+	// Redaction must not change the live config.
+	assert.Equal(t, "default-password", cfg.MySQL.Password)
+	assert.Equal(t, "inline-monitor-password", cfg.Monitors[0].Password)
+	assert.Equal(t, envPassword, cfg.Monitors[1].Password)
+	assert.Equal(t, "plan-monitor-password", cfg.Plans.Monitor.Password)
+}
+
+func TestConfigMonitorRedactedPreservesPlanMonitorCycles(t *testing.T) {
+	monitor := blip.ConfigMonitor{
+		MonitorId: "cyclic-monitor",
+		Password:  "cyclic-password",
+	}
+	monitor.Plans.Monitor = &monitor
+
+	redacted := monitor.Redacted()
+
+	assert.Equal(t, "...", redacted.Password)
+	require.NotNil(t, redacted.Plans.Monitor)
+	assert.Equal(t, "...", redacted.Plans.Monitor.Password)
+	assert.Same(t, redacted.Plans.Monitor, redacted.Plans.Monitor.Plans.Monitor)
+	assert.Equal(t, "cyclic-password", monitor.Password)
+}
+
+func TestConfigRedactedRedactsSinkOptionValues(t *testing.T) {
+	envKey := "blip_test_TestConfigRedactedRedactsSinkOptionValues"
+	envToken := "environment-expanded-sink-token"
+	t.Setenv(envKey, envToken)
+
+	cfg := blip.Config{
+		Sinks: blip.ConfigSinks{
+			"datadog": {
+				"api-key-auth": "global-datadog-api-key",
+				"api-compress": "global-non-secret-option-value",
+			},
+		},
+		Monitors: []blip.ConfigMonitor{
+			{
+				MonitorId: "sink-monitor",
+				Sinks: blip.ConfigSinks{
+					"signalfx": {
+						"auth-token": fmt.Sprintf("${%s}", envKey),
+					},
+				},
+			},
+		},
+		Plans: blip.ConfigPlans{
+			Monitor: &blip.ConfigMonitor{
+				MonitorId: "plan-sink-monitor",
+				Sinks: blip.ConfigSinks{
+					"custom-sink": {
+						"unknown-option": "custom-sink-option-value",
+					},
+				},
+			},
+		},
+	}
+	cfg.InterpolateEnvVars()
+	cfg.Monitors[0].InterpolateEnvVars()
+	cfg.Plans.Monitor.InterpolateEnvVars()
+
+	redacted := cfg.Redacted()
+	dump := fmt.Sprintf("%#v", redacted)
+
+	for _, value := range []string{
+		"global-datadog-api-key",
+		"global-non-secret-option-value",
+		envToken,
+		"custom-sink-option-value",
+	} {
+		assert.NotContains(t, dump, value)
+	}
+	for _, optionName := range []string{
+		"datadog",
+		"api-key-auth",
+		"api-compress",
+		"signalfx",
+		"auth-token",
+	} {
+		assert.Contains(t, dump, optionName)
+	}
+	assert.Equal(t, "...", redacted.Sinks["datadog"]["api-key-auth"])
+	assert.Equal(t, "...", redacted.Monitors[0].Sinks["signalfx"]["auth-token"])
+	assert.Equal(t, "...", redacted.Plans.Monitor.Sinks["custom-sink"]["unknown-option"])
+
+	// Redaction must not change the live sink maps.
+	assert.Equal(t, "global-datadog-api-key", cfg.Sinks["datadog"]["api-key-auth"])
+	assert.Equal(t, envToken, cfg.Monitors[0].Sinks["signalfx"]["auth-token"])
+	assert.Equal(t, "custom-sink-option-value", cfg.Plans.Monitor.Sinks["custom-sink"]["unknown-option"])
+}
+
+func TestConfigHTTPRedacted(t *testing.T) {
+	envKey := "blip_test_TestConfigHTTPRedacted"
+	t.Setenv(envKey, "environment-proxy-password")
+
+	tests := []struct {
+		name  string
+		proxy string
+		want  string
+	}{
+		{name: "empty"},
+		{
+			name:  "unauthenticated",
+			proxy: "http://proxy.example:8080",
+			want:  "http://proxy.example:8080",
+		},
+		{
+			name:  "username only",
+			proxy: "http://proxy-user@proxy.example:8080",
+			want:  "http://proxy-user@proxy.example:8080",
+		},
+		{
+			name:  "authenticated",
+			proxy: "http://proxy-user:proxy-password@proxy.example:8080",
+			want:  "http://proxy-user:xxxxx@proxy.example:8080",
+		},
+		{
+			name:  "environment expanded password",
+			proxy: fmt.Sprintf("http://proxy-user:${%s}@proxy.example:8080", envKey),
+			want:  "http://proxy-user:xxxxx@proxy.example:8080",
+		},
+		{
+			name:  "malformed",
+			proxy: "http://proxy-user:%zz@proxy.example:8080",
+			want:  "...",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := blip.ConfigHTTP{Proxy: tt.proxy}
+			cfg.InterpolateEnvVars()
+			original := cfg.Proxy
+
+			redacted := cfg.Redacted()
+
+			assert.Equal(t, tt.want, redacted.Proxy)
+			assert.Equal(t, original, cfg.Proxy, "redaction mutated the live HTTP config")
+		})
+	}
+}
+
 func TestApplyDefaultConfig(t *testing.T) {
 	// Defaults apply when the value isn't set
 	df := blip.DefaultConfig()

monitor/loader.go

@@ -436,7 +436,7 @@ func (ml *Loader) save(newConfigs []blip.ConfigMonitor, validConfigs map[string]
 		// Save valid monitor config. The does NOT create or run the monitor:
 		// the former is done at the end of load (not Load), and the latter is
 		// done in StartMonitors.
-		blip.Debug("loaded monitor: %#v", newcfg)
+		blip.Debug("loaded monitor: %#v", newcfg.Redacted())
 		validConfigs[newcfg.MonitorId] = newcfg
 	}
 	return nil

server/server.go

@@ -67,7 +67,7 @@ func (f httpClientFactory) MakeForSink(sinkName, monitorId string, opts, tags ma
 			return url.Parse(f.cfg.Proxy)
 		}
 		transport.Proxy = proxyFunc
-		blip.Debug("%s sink %s http proxy via %s", monitorId, sinkName, f.cfg.Proxy)
+		blip.Debug("%s sink %s http proxy via %s", monitorId, sinkName, f.cfg.Redacted().Proxy)
 	}
 	return &http.Client{
 		Timeout:   10 * time.Second,
@@ -170,7 +170,7 @@ func (s *Server) Boot(env blip.Env, plugins blip.Plugins, factories blip.Factori
 		return err
 	}
 	s.cfg.InterpolateEnvVars()
-	blip.Debug("config: %#v", s.cfg)
+	blip.Debug("config: %#v", s.cfg.Redacted())
 	if err := s.cfg.Validate(); err != nil {
 		event.Errorf(event.BOOT_CONFIG_INVALID, "%s", err.Error())
 		return err