Skip to content

feat(logs): send JSON via dict-encoded sets of keys #49404

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
TheSafo wants to merge 1 commit into
base: jsaf/dual-send-stacked
Choose a base branch
from
Draft

feat(logs): send JSON via dict-encoded sets of keys #49404

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion comp/otelcol/logsagentpipeline/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@ require (
github.com/DataDog/datadog-agent/pkg/logs/status/statusinterface v0.64.0-rc.3 // indirect
github.com/DataDog/datadog-agent/pkg/logs/status/utils v0.64.0-rc.3 // indirect
github.com/DataDog/datadog-agent/pkg/logs/types v0.71.0-rc.1 // indirect
github.com/DataDog/datadog-agent/pkg/proto v0.74.1 // indirect
github.com/DataDog/datadog-agent/pkg/telemetry v0.64.1 // indirect
github.com/DataDog/datadog-agent/pkg/template v0.65.1 // indirect
github.com/DataDog/datadog-agent/pkg/trace/log v0.77.0-devel // indirect
Expand Down Expand Up @@ -78,6 +77,7 @@ require (
github.com/gofrs/flock v0.13.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/google/flatbuffers v25.2.10+incompatible // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/hectane/go-acl v0.0.0-20230225031251-cdfc9e3acf94 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 // indirect
Expand Down
2 changes: 1 addition & 1 deletion comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,6 @@ require (
github.com/DataDog/datadog-agent/pkg/logs/sender/grpc v0.0.0-00010101000000-000000000000 // indirect
github.com/DataDog/datadog-agent/pkg/logs/status/utils v0.64.0-rc.3 // indirect
github.com/DataDog/datadog-agent/pkg/logs/types v0.71.0-rc.1 // indirect
github.com/DataDog/datadog-agent/pkg/proto v0.74.1 // indirect
github.com/DataDog/datadog-agent/pkg/telemetry v0.64.1 // indirect
github.com/DataDog/datadog-agent/pkg/template v0.65.1 // indirect
github.com/DataDog/datadog-agent/pkg/trace/log v0.77.0-devel // indirect
Expand Down Expand Up @@ -94,6 +93,7 @@ require (
github.com/gofrs/flock v0.13.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/google/flatbuffers v25.2.10+incompatible // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/hectane/go-acl v0.0.0-20230225031251-cdfc9e3acf94 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 // indirect
Expand Down
1 change: 1 addition & 0 deletions pkg/compliance/reporter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ func NewLogReporter(hostname string, sourceName, sourceType string, endpoints *c
compression,
cfg.GetBool("logs_config.disable_distributed_senders"),
false, // serverless
nil, // secretsComp
)
pipelineProvider.Start()

Expand Down
4 changes: 3 additions & 1 deletion pkg/logs/client/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ require (
github.com/DataDog/datadog-agent/pkg/fips v0.0.0 // indirect
github.com/DataDog/datadog-agent/pkg/logs/status/utils v0.61.0 // indirect
github.com/DataDog/datadog-agent/pkg/logs/types v0.71.0-rc.1 // indirect
github.com/DataDog/datadog-agent/pkg/proto v0.0.0-00010101000000-000000000000 // indirect
github.com/DataDog/datadog-agent/pkg/template v0.65.1 // indirect
github.com/DataDog/datadog-agent/pkg/util/executable v0.61.0 // indirect
github.com/DataDog/datadog-agent/pkg/util/filesystem v0.61.0 // indirect
Expand Down Expand Up @@ -88,6 +87,9 @@ require (
github.com/tklauser/go-sysconf v0.3.16 // indirect
github.com/tklauser/numcpus v0.11.0 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.opentelemetry.io/otel/metric v1.42.0 // indirect
go.opentelemetry.io/otel/sdk v1.41.0 // indirect
go.opentelemetry.io/otel/trace v1.42.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/dig v1.19.0 // indirect
go.uber.org/fx v1.24.0 // indirect
Expand Down
4 changes: 3 additions & 1 deletion pkg/logs/diagnostic/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ require (
github.com/DataDog/datadog-agent/pkg/fips v0.0.0 // indirect
github.com/DataDog/datadog-agent/pkg/logs/status/utils v0.61.0 // indirect
github.com/DataDog/datadog-agent/pkg/logs/types v0.71.0-rc.1 // indirect
github.com/DataDog/datadog-agent/pkg/proto v0.0.0-00010101000000-000000000000 // indirect
github.com/DataDog/datadog-agent/pkg/template v0.65.1 // indirect
github.com/DataDog/datadog-agent/pkg/util/executable v0.61.0 // indirect
github.com/DataDog/datadog-agent/pkg/util/filesystem v0.61.0 // indirect
Expand Down Expand Up @@ -70,6 +69,9 @@ require (
github.com/tklauser/go-sysconf v0.3.16 // indirect
github.com/tklauser/numcpus v0.11.0 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.opentelemetry.io/otel/metric v1.42.0 // indirect
go.opentelemetry.io/otel/sdk v1.41.0 // indirect
go.opentelemetry.io/otel/trace v1.42.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/dig v1.19.0 // indirect
go.uber.org/fx v1.24.0 // indirect
Expand Down
7 changes: 3 additions & 4 deletions pkg/logs/message/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ require (
github.com/DataDog/datadog-agent/comp/logs/agent/config v0.61.0
github.com/DataDog/datadog-agent/pkg/logs/sources v0.61.0
github.com/DataDog/datadog-agent/pkg/logs/types v0.71.0-rc.1
github.com/DataDog/datadog-agent/pkg/proto v0.0.0-00010101000000-000000000000
github.com/DataDog/datadog-agent/pkg/util/log v0.73.0-rc.5
github.com/stretchr/testify v1.11.1
)
Expand Down Expand Up @@ -63,9 +62,8 @@ require (
github.com/tklauser/go-sysconf v0.3.16 // indirect
github.com/tklauser/numcpus v0.11.0 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.opentelemetry.io/otel/metric v1.42.0 // indirect
go.opentelemetry.io/otel/sdk v1.41.0 // indirect
go.opentelemetry.io/otel/trace v1.42.0 // indirect
go.opentelemetry.io/otel v1.42.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.41.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.yaml.in/yaml/v2 v2.4.3 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
Expand All @@ -74,6 +72,7 @@ require (
golang.org/x/sys v0.42.0 // indirect
golang.org/x/text v0.35.0 // indirect
golang.org/x/time v0.14.0 // indirect
gonum.org/v1/gonum v0.17.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 // indirect
google.golang.org/grpc v1.79.3 // indirect
google.golang.org/protobuf v1.36.11 // indirect
Expand Down
81 changes: 66 additions & 15 deletions pkg/logs/patterns/processor/json.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ package processor
import (
"bytes"
"encoding/json"
"fmt"
"sort"
"strings"
)

Expand All @@ -19,8 +21,15 @@ type ExtractionResult struct {
IsJSON bool
// Message is the extracted message field (empty if not found or not JSON)
Message string
// JSONContext is the ordered, serialized remaining JSON fields (nil if not JSON or extraction failed)
JSONContext []byte
// MessageKey is the JSON key the message was extracted from (e.g. "msg", "message")
MessageKey string
// JSONContextSchema is a comma-separated sorted list of top-level keys for the JSON context.
// When populated, JSONContextValues contains the corresponding values in the same order.
// Example: for {"level":"info","pid":1234,"service":"api"}, schema is "level,pid,service".
JSONContextSchema string
// JSONContextValues contains the leaf values corresponding to JSONContextSchema keys, in order.
// Nested objects/arrays are serialized as JSON strings.
JSONContextValues []string
}

// Common top-level message field names (Layer 0)
Expand Down Expand Up @@ -64,26 +73,68 @@ func PreprocessJSON(content []byte) ExtractionResult {
// Remove the extracted message field from data for jsoncontext construction
removeFieldByPath(data, extractedPath)

// If no fields remain after removing the message, keep json_context nil (avoid sending "{}").
// If no fields remain after removing the message, no context to send.
if len(data) == 0 {
return ExtractionResult{
IsJSON: true,
Message: message,
JSONContext: nil,
IsJSON: true,
Message: message,
MessageKey: extractedPath,
}
}

// Serialize remaining fields as JSON context.
// encoding/json marshals maps with deterministic key ordering for better compression.
jsonContext, err := json.Marshal(data)
if err != nil {
return fail
}
// Schema-based encoding: extract sorted keys and leaf values.
// Nested objects/arrays are serialized as JSON strings in the values list.
schema, values := extractSchemaAndValues(data)

return ExtractionResult{
IsJSON: true,
Message: message,
JSONContext: jsonContext,
IsJSON: true,
Message: message,
MessageKey: extractedPath,
JSONContextSchema: schema,
JSONContextValues: values,
}
}

// extractSchemaAndValues extracts sorted keys and their corresponding values from a JSON map.
// Primitive values (string, number, bool, null) are converted to strings.
// Nested objects and arrays are serialized as JSON strings.
func extractSchemaAndValues(data map[string]interface{}) (string, []string) {
keys := make([]string, 0, len(data))
for k := range data {
keys = append(keys, k)
}
sort.Strings(keys)

values := make([]string, len(keys))
for i, k := range keys {
values[i] = valueToString(data[k])
}

return strings.Join(keys, ","), values
}

// valueToString converts a JSON value to its string representation.
// Primitives are converted directly; objects and arrays are serialized as JSON.
func valueToString(v interface{}) string {
switch val := v.(type) {
case string:
return val
case float64:
return fmt.Sprintf("%g", val)
case bool:
if val {
return "true"
}
return "false"
case nil:
return ""
default:
// Nested object or array — serialize as JSON
b, err := json.Marshal(val)
if err != nil {
return ""
}
return string(b)
}
}

Expand Down
Loading
Loading