Skip to content

Commit f53f30d

Browse files
api-clients-generation-pipeline[bot]ci.datadog-api-spec
api-clients-generation-pipeline[bot]
and
ci.datadog-api-spec
authored
Adding custom mapper support to Observability Pipelines OCSF Mapper (#3478)
Co-authored-by: ci.datadog-api-spec <packages@datadoghq.com>
1 parent 3d40014 commit f53f30d

16 files changed

+1767
-4
lines changed

‎.generator/schemas/v2/openapi.yaml‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41369,6 +41369,7 @@ components:
4136941369
example: CloudTrail Account Change
4137041370
oneOf:
4137141371
- $ref: '#/components/schemas/ObservabilityPipelineOcsfMappingLibrary'
41372+
- $ref: '#/components/schemas/ObservabilityPipelineOcsfMappingCustom'
4137241373
ObservabilityPipelineOcsfMapperProcessorType:
4137341374
default: ocsf_mapper
4137441375
description: The processor type. The value should always be `ocsf_mapper`.
@@ -41378,6 +41379,116 @@ components:
4137841379
type: string
4137941380
x-enum-varnames:
4138041381
- OCSF_MAPPER
41382+
ObservabilityPipelineOcsfMappingCustom:
41383+
description: Custom OCSF mapping configuration for transforming logs.
41384+
properties:
41385+
mapping:
41386+
description: A list of field mapping rules for transforming log fields to
41387+
OCSF schema fields.
41388+
items:
41389+
$ref: '#/components/schemas/ObservabilityPipelineOcsfMappingCustomFieldMapping'
41390+
type: array
41391+
metadata:
41392+
$ref: '#/components/schemas/ObservabilityPipelineOcsfMappingCustomMetadata'
41393+
version:
41394+
description: The version of the custom mapping configuration.
41395+
example: 1
41396+
format: int64
41397+
type: integer
41398+
required:
41399+
- mapping
41400+
- metadata
41401+
- version
41402+
type: object
41403+
ObservabilityPipelineOcsfMappingCustomFieldMapping:
41404+
description: Defines a single field mapping rule for transforming a source field
41405+
to an OCSF destination field.
41406+
properties:
41407+
default:
41408+
description: The default value to use if the source field is missing or
41409+
empty.
41410+
example: ''
41411+
dest:
41412+
description: The destination OCSF field path.
41413+
example: device.type
41414+
type: string
41415+
lookup:
41416+
$ref: '#/components/schemas/ObservabilityPipelineOcsfMappingCustomLookup'
41417+
source:
41418+
description: The source field path from the log event.
41419+
example: host.type
41420+
sources:
41421+
description: Multiple source field paths for combined mapping.
41422+
example:
41423+
- field1
41424+
- field2
41425+
value:
41426+
description: A static value to use for the destination field.
41427+
example: static_value
41428+
required:
41429+
- dest
41430+
type: object
41431+
ObservabilityPipelineOcsfMappingCustomLookup:
41432+
description: Lookup table configuration for mapping source values to destination
41433+
values.
41434+
properties:
41435+
default:
41436+
description: The default value to use if no lookup match is found.
41437+
example: unknown
41438+
table:
41439+
description: A list of lookup table entries for value transformation.
41440+
items:
41441+
$ref: '#/components/schemas/ObservabilityPipelineOcsfMappingCustomLookupTableEntry'
41442+
type: array
41443+
type: object
41444+
ObservabilityPipelineOcsfMappingCustomLookupTableEntry:
41445+
description: A single entry in a lookup table for value transformation.
41446+
properties:
41447+
contains:
41448+
description: The substring to match in the source value.
41449+
example: Desktop
41450+
type: string
41451+
equals:
41452+
description: The exact value to match in the source.
41453+
example: desktop
41454+
equals_source:
41455+
description: The source field to match against.
41456+
example: device_type
41457+
type: string
41458+
matches:
41459+
description: A regex pattern to match in the source value.
41460+
example: ^Desktop.*
41461+
type: string
41462+
not_matches:
41463+
description: A regex pattern that must not match the source value.
41464+
example: ^Mobile.*
41465+
type: string
41466+
value:
41467+
description: The value to use when a match is found.
41468+
example: desktop
41469+
type: object
41470+
ObservabilityPipelineOcsfMappingCustomMetadata:
41471+
description: Metadata for the custom OCSF mapping.
41472+
properties:
41473+
class:
41474+
description: The OCSF event class name.
41475+
example: Device Inventory Info
41476+
type: string
41477+
profiles:
41478+
description: A list of OCSF profiles to apply.
41479+
example:
41480+
- container
41481+
items:
41482+
type: string
41483+
type: array
41484+
version:
41485+
description: The OCSF schema version.
41486+
example: 1.3.0
41487+
type: string
41488+
required:
41489+
- class
41490+
- version
41491+
type: object
4138141492
ObservabilityPipelineOcsfMappingLibrary:
4138241493
description: Predefined library mappings for common log formats.
4138341494
enum:

‎examples/v2/observability-pipelines/ValidatePipeline_3024756866.java‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
// Validate an observability pipeline with OCSF mapper custom mapping returns "OK" response
2+
3+
import com.datadog.api.client.ApiClient;
4+
import com.datadog.api.client.ApiException;
5+
import com.datadog.api.client.v2.api.ObservabilityPipelinesApi;
6+
import com.datadog.api.client.v2.model.ObservabilityPipelineConfig;
7+
import com.datadog.api.client.v2.model.ObservabilityPipelineConfigDestinationItem;
8+
import com.datadog.api.client.v2.model.ObservabilityPipelineConfigProcessorGroup;
9+
import com.datadog.api.client.v2.model.ObservabilityPipelineConfigProcessorItem;
10+
import com.datadog.api.client.v2.model.ObservabilityPipelineConfigSourceItem;
11+
import com.datadog.api.client.v2.model.ObservabilityPipelineDataAttributes;
12+
import com.datadog.api.client.v2.model.ObservabilityPipelineDatadogAgentSource;
13+
import com.datadog.api.client.v2.model.ObservabilityPipelineDatadogAgentSourceType;
14+
import com.datadog.api.client.v2.model.ObservabilityPipelineDatadogLogsDestination;
15+
import com.datadog.api.client.v2.model.ObservabilityPipelineDatadogLogsDestinationType;
16+
import com.datadog.api.client.v2.model.ObservabilityPipelineOcsfMapperProcessor;
17+
import com.datadog.api.client.v2.model.ObservabilityPipelineOcsfMapperProcessorMapping;
18+
import com.datadog.api.client.v2.model.ObservabilityPipelineOcsfMapperProcessorMappingMapping;
19+
import com.datadog.api.client.v2.model.ObservabilityPipelineOcsfMapperProcessorType;
20+
import com.datadog.api.client.v2.model.ObservabilityPipelineOcsfMappingCustom;
21+
import com.datadog.api.client.v2.model.ObservabilityPipelineOcsfMappingCustomFieldMapping;
22+
import com.datadog.api.client.v2.model.ObservabilityPipelineOcsfMappingCustomLookup;
23+
import com.datadog.api.client.v2.model.ObservabilityPipelineOcsfMappingCustomLookupTableEntry;
24+
import com.datadog.api.client.v2.model.ObservabilityPipelineOcsfMappingCustomMetadata;
25+
import com.datadog.api.client.v2.model.ObservabilityPipelineSpec;
26+
import com.datadog.api.client.v2.model.ObservabilityPipelineSpecData;
27+
import com.datadog.api.client.v2.model.ValidationResponse;
28+
import java.util.Arrays;
29+
import java.util.Collections;
30+
31+
public class Example {
32+
public static void main(String[] args) {
33+
ApiClient defaultClient = ApiClient.getDefaultApiClient();
34+
ObservabilityPipelinesApi apiInstance = new ObservabilityPipelinesApi(defaultClient);
35+
36+
ObservabilityPipelineSpec body =
37+
new ObservabilityPipelineSpec()
38+
.data(
39+
new ObservabilityPipelineSpecData()
40+
.attributes(
41+
new ObservabilityPipelineDataAttributes()
42+
.config(
43+
new ObservabilityPipelineConfig()
44+
.destinations(
45+
Collections.singletonList(
46+
new ObservabilityPipelineConfigDestinationItem(
47+
new ObservabilityPipelineDatadogLogsDestination()
48+
.id("datadog-logs-destination")
49+
.inputs(
50+
Collections.singletonList(
51+
"my-processor-group"))
52+
.type(
53+
ObservabilityPipelineDatadogLogsDestinationType
54+
.DATADOG_LOGS))))
55+
.processorGroups(
56+
Collections.singletonList(
57+
new ObservabilityPipelineConfigProcessorGroup()
58+
.enabled(true)
59+
.id("my-processor-group")
60+
.include("service:my-service")
61+
.inputs(
62+
Collections.singletonList(
63+
"datadog-agent-source"))
64+
.processors(
65+
Collections.singletonList(
66+
new ObservabilityPipelineConfigProcessorItem(
67+
new ObservabilityPipelineOcsfMapperProcessor()
68+
.enabled(true)
69+
.id("ocsf-mapper-processor")
70+
.include("service:my-service")
71+
.mappings(
72+
Collections.singletonList(
73+
new ObservabilityPipelineOcsfMapperProcessorMapping()
74+
.include(
75+
"source:custom")
76+
.mapping(
77+
new ObservabilityPipelineOcsfMapperProcessorMappingMapping(
78+
new ObservabilityPipelineOcsfMappingCustom()
79+
.mapping(
80+
Arrays
81+
.asList(
82+
new ObservabilityPipelineOcsfMappingCustomFieldMapping()
83+
._default(
84+
"")
85+
.dest(
86+
"time")
87+
.source(
88+
"timestamp"),
89+
new ObservabilityPipelineOcsfMappingCustomFieldMapping()
90+
._default(
91+
"")
92+
.dest(
93+
"severity")
94+
.source(
95+
"level"),
96+
new ObservabilityPipelineOcsfMappingCustomFieldMapping()
97+
._default(
98+
"")
99+
.dest(
100+
"device.type")
101+
.lookup(
102+
new ObservabilityPipelineOcsfMappingCustomLookup()
103+
.table(
104+
Collections
105+
.singletonList(
106+
new ObservabilityPipelineOcsfMappingCustomLookupTableEntry()
107+
.contains(
108+
"Desktop")
109+
.value(
110+
"desktop"))))
111+
.source(
112+
"host.type")))
113+
.metadata(
114+
new ObservabilityPipelineOcsfMappingCustomMetadata()
115+
._class(
116+
"Device"
117+
+ " Inventory"
118+
+ " Info")
119+
.profiles(
120+
Collections
121+
.singletonList(
122+
"container"))
123+
.version(
124+
"1.3.0"))
125+
.version(
126+
1L)))))
127+
.type(
128+
ObservabilityPipelineOcsfMapperProcessorType
129+
.OCSF_MAPPER))))))
130+
.sources(
131+
Collections.singletonList(
132+
new ObservabilityPipelineConfigSourceItem(
133+
new ObservabilityPipelineDatadogAgentSource()
134+
.id("datadog-agent-source")
135+
.type(
136+
ObservabilityPipelineDatadogAgentSourceType
137+
.DATADOG_AGENT)))))
138+
.name("OCSF Custom Mapper Pipeline"))
139+
.type("pipelines"));
140+
141+
try {
142+
ValidationResponse result = apiInstance.validatePipeline(body);
143+
System.out.println(result);
144+
} catch (ApiException e) {
145+
System.err.println("Exception when calling ObservabilityPipelinesApi#validatePipeline");
146+
System.err.println("Status code: " + e.getCode());
147+
System.err.println("Reason: " + e.getResponseBody());
148+
System.err.println("Response headers: " + e.getResponseHeaders());
149+
e.printStackTrace();
150+
}
151+
}
152+
}

0 commit comments

Comments
 (0)