Skip to content

Commit 09521b2

Browse files
mjwolfmjwolf
authored
Generate data stream input configuration and documentation (#2826)
This introduces a new interactive prompt to the create data-stream command. When a user creates a data stream of type logs, they can now select from a list of common input types (e.g., aws-s3, gcp-pubsub, udp, tcp). Based on the user's selection, the command now automatically populates the data_stream//manifest.yml with the corresponding input streams and their required variables. This streamlines the creation process for new log-based data streams by providing sensible defaults and reducing manual configuration. This also updates the package README generation step to support a new function "inputDocs". When this is used in the readme template, all inputs used in the package will be found, and all will be listed in the readme. For inputs which have additional documentation added to elastic-package, the input documentation will also be inserted into the rendered readme.
1 parent 8727ed5 commit 09521b2

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+4723
-34
lines changed

‎cmd/create_data_stream.go‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ package cmd
77
import (
88
"errors"
99
"fmt"
10+
"maps"
11+
"slices"
1012

1113
"github.com/AlecAivazis/survey/v2"
1214

@@ -25,6 +27,7 @@ type newDataStreamAnswers struct {
2527
Name string
2628
Title string
2729
Type string
30+
Inputs []string
2831
Subobjects bool
2932
SyntheticAndTimeSeries bool
3033
Synthetic bool
@@ -126,6 +129,50 @@ func createDataStreamCommandAction(cmd *cobra.Command, args []string) error {
126129
}
127130
}
128131

132+
if answers.Type == "logs" {
133+
// Map of possible inputs that can be used in the wizard, and their description.
134+
inputsMap := map[string]string{
135+
"aws-cloudwatch": "AWS Cloudwatch",
136+
"aws-s3": "AWS S3",
137+
"azure-blob-storage": "Azure Blob Storage",
138+
"azure-eventhub": "Azure Eventhub",
139+
"cel": "Common Expression Language (CEL)",
140+
"entity-analytics": "Entity Analytics",
141+
"etw": "Event Tracing for Windows (ETW)",
142+
"filestream": "Filestream",
143+
"gcp-pubsub": "GCP PubSub",
144+
"gcs": "Google Cloud Storage (GCS)",
145+
"http_endpoint": "HTTP Endpoint",
146+
"journald": "Journald",
147+
"netflow": "Netflow",
148+
"redis": "Redis",
149+
"tcp": "TCP",
150+
"udp": "UDP",
151+
"winlog": "WinLogBeat",
152+
}
153+
qs := []*survey.Question{
154+
{
155+
Name: "inputs",
156+
Prompt: &survey.MultiSelect{
157+
Message: "Select input types which will be used in this data stream. See https://www.elastic.co/docs/reference/fleet/elastic-agent-inputs-list for description of the inputs",
158+
Options: slices.Sorted(maps.Keys(inputsMap)),
159+
PageSize: 50,
160+
Description: func(value string, index int) string {
161+
val, ok := inputsMap[value]
162+
if ok {
163+
return val
164+
}
165+
return ""
166+
},
167+
},
168+
},
169+
}
170+
err = survey.Ask(qs, &answers)
171+
if err != nil {
172+
return fmt.Errorf("prompt failed: %w", err)
173+
}
174+
}
175+
129176
descriptor := createDataStreamDescriptorFromAnswers(answers, packageRoot)
130177
err = archetype.CreateDataStream(descriptor)
131178
if err != nil {
@@ -163,6 +210,22 @@ func createDataStreamDescriptorFromAnswers(answers newDataStreamAnswers, package
163210
}
164211
}
165212

213+
// If no inputs were selected, insert one so the datastream shows an example of an input.
214+
if answers.Type == "logs" && len(answers.Inputs) == 0 {
215+
answers.Inputs = []string{"filestream"}
216+
}
217+
218+
if len(answers.Inputs) > 0 {
219+
var streams []packages.Stream
220+
for _, input := range answers.Inputs {
221+
streams = append(streams, packages.Stream{
222+
Input: input,
223+
Vars: []packages.Variable{},
224+
})
225+
}
226+
manifest.Streams = streams
227+
}
228+
166229
return archetype.DataStreamDescriptor{
167230
Manifest: manifest,
168231
PackageRoot: packageRoot,

‎internal/docs/_static/agent/aws_cloudwatch.yml.hbs‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
{{#unless log_group_name}}
2+
{{#unless log_group_name_prefix}}
3+
{{#if log_group_arn }}
4+
log_group_arn: {{ log_group_arn }}
5+
{{/if}}
6+
{{/unless}}
7+
{{/unless}}
8+
{{#unless log_group_arn}}
9+
{{#unless log_group_name}}
10+
{{#if log_group_name_prefix }}
11+
log_group_name_prefix: {{ log_group_name_prefix }}
12+
{{/if}}
13+
{{/unless}}
14+
{{/unless}}
15+
{{#unless log_group_arn}}
16+
{{#unless log_group_name_prefix}}
17+
{{#if log_group_name }}
18+
log_group_name: {{ log_group_name }}
19+
{{/if}}
20+
{{/unless}}
21+
{{/unless}}
22+
{{#unless log_group_arn}}
23+
region_name: {{ region_name }}
24+
{{/unless}}
25+
{{#unless log_stream_prefix}}
26+
{{#if log_streams }}
27+
log_streams: {{ log_streams }}
28+
{{/if}}
29+
{{/unless}}
30+
{{#unless log_streams}}
31+
{{#if log_stream_prefix }}
32+
log_stream_prefix: {{ log_stream_prefix }}
33+
{{/if}}
34+
{{/unless}}
35+
{{#if start_position }}
36+
start_position: {{ start_position }}
37+
{{/if}}
38+
{{#if scan_frequency }}
39+
scan_frequency: {{ scan_frequency }}
40+
{{/if}}
41+
{{#if api_sleep }}
42+
api_sleep: {{ api_sleep }}
43+
{{/if}}
44+
{{#if api_timeout}}
45+
api_timeout: {{api_timeout}}
46+
{{/if}}
47+
{{#if latency }}
48+
latency: {{ latency }}
49+
{{/if}}
50+
{{#if number_of_workers }}
51+
number_of_workers: {{ number_of_workers }}
52+
{{/if}}
53+
{{#if credential_profile_name}}
54+
credential_profile_name: {{credential_profile_name}}
55+
{{/if}}
56+
{{#if shared_credential_file}}
57+
shared_credential_file: {{shared_credential_file}}
58+
{{/if}}
59+
{{#if default_region}}
60+
default_region: {{default_region}}
61+
{{/if}}
62+
{{#if access_key_id}}
63+
access_key_id: {{access_key_id}}
64+
{{/if}}
65+
{{#if secret_access_key}}
66+
secret_access_key: {{secret_access_key}}
67+
{{/if}}
68+
{{#if session_token}}
69+
session_token: {{session_token}}
70+
{{/if}}
71+
{{#if role_arn}}
72+
role_arn: {{role_arn}}
73+
{{/if}}
74+
{{#if proxy_url }}
75+
proxy_url: {{proxy_url}}
76+
{{/if}}
77+
78+
{{#if tags.length}}
79+
tags:
80+
{{#each tags as |tag|}}
81+
- {{tag}}
82+
{{/each}}
83+
{{#if preserve_original_event}}
84+
- preserve_original_event
85+
{{/if}}
86+
{{else}}
87+
{{#if preserve_original_event}}
88+
tags:
89+
- preserve_original_event
90+
{{/if}}
91+
{{/if}}
92+
93+
{{#contains "forwarded" tags}}
94+
publisher_pipeline.disable_host: true
95+
{{/contains}}
96+
97+
{{#if processors}}
98+
processors:
99+
{{processors}}
100+
{{/if}}

‎internal/docs/_static/agent/aws_s3.yml.hbs‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
{{! start SQS queue }}
2+
{{#unless bucket_arn}}
3+
{{#unless non_aws_bucket_name}}
4+
{{#if queue_url }}
5+
queue_url: {{ queue_url }}
6+
{{/if}}
7+
{{/unless}}
8+
{{/unless}}
9+
{{! end SQS queue }}
10+
11+
{{#unless queue_url}}{{! start S3 bucket polling }}
12+
13+
{{!
14+
When using an S3 bucket, you can specify only one of the following options:
15+
- An AWS bucket ARN
16+
- A non-AWS bucket name
17+
}}
18+
19+
{{! shared S3 bucket polling options }}
20+
{{#if number_of_workers }}
21+
number_of_workers: {{ number_of_workers }}
22+
{{/if}}
23+
{{#if bucket_list_prefix }}
24+
bucket_list_prefix: {{ bucket_list_prefix }}
25+
{{/if}}
26+
{{#if bucket_list_interval }}
27+
bucket_list_interval: {{ bucket_list_interval }}
28+
{{/if}}
29+
30+
{{! AWS S3 bucket ARN options }}
31+
{{#unless non_aws_bucket_name}}
32+
{{#if bucket_arn }}
33+
bucket_arn: {{ bucket_arn }}
34+
{{/if}}
35+
{{/unless}}{{! end AWS S3 bucket ARN options }}
36+
37+
{{! non-AWS S3 bucket ARN options }}
38+
{{#unless bucket_arn}}
39+
{{#if non_aws_bucket_name }}
40+
non_aws_bucket_name: {{ non_aws_bucket_name }}
41+
{{/if}}
42+
{{/unless}}{{! end non-AWS S3 bucket ARN options }}
43+
44+
{{/unless}}{{! end S3 bucket polling }}
45+
46+
{{#if buffer_size }}
47+
buffer_size: {{ buffer_size }}
48+
{{/if}}
49+
{{#if content_type }}
50+
content_type: {{ content_type }}
51+
{{/if}}
52+
{{#if encoding }}
53+
encoding: {{ encoding }}
54+
{{/if}}
55+
{{#if expand_event_list_from_field }}
56+
expand_event_list_from_field: {{ expand_event_list_from_field }}
57+
{{/if}}
58+
{{#if buffer_size }}
59+
buffer_size: {{ buffer_size }}
60+
{{/if}}
61+
{{#if fips_enabled }}
62+
fips_enabled: {{ fips_enabled }}
63+
{{/if}}
64+
{{#if include_s3_metadata }}
65+
include_s3_metadata: {{ include_s3_metadata }}
66+
{{/if}}
67+
{{#if max_bytes }}
68+
max_bytes: {{ max_bytes }}
69+
{{/if}}
70+
{{#if max_number_of_messages }}
71+
max_number_of_messages: {{ max_number_of_messages }}
72+
{{/if}}
73+
{{#if path_style }}
74+
path_style: {{ path_style }}
75+
{{/if}}
76+
{{#if provider }}
77+
provider: {{ provider }}
78+
{{/if}}
79+
{{#if sqs.max_receive_count }}
80+
sqs.max_receive_count: {{ sqs.max_receive_count }}
81+
{{/if}}
82+
{{#if sqs.wait_time }}
83+
sqs.wait_time: {{ sqs.wait_time }}
84+
{{/if}}
85+
86+
{{#if file_selectors}}
87+
file_selectors:
88+
{{file_selectors}}
89+
{{/if}}
90+
91+
{{#if credential_profile_name}}
92+
credential_profile_name: {{credential_profile_name}}
93+
{{/if}}
94+
{{#if shared_credential_file}}
95+
shared_credential_file: {{shared_credential_file}}
96+
{{/if}}
97+
{{#if visibility_timeout}}
98+
visibility_timeout: {{visibility_timeout}}
99+
{{/if}}
100+
{{#if api_timeout}}
101+
api_timeout: {{api_timeout}}
102+
{{/if}}
103+
{{#if endpoint}}
104+
endpoint: {{endpoint}}
105+
{{/if}}
106+
{{#if default_region}}
107+
default_region: {{default_region}}
108+
{{/if}}
109+
{{#if access_key_id}}
110+
access_key_id: {{access_key_id}}
111+
{{/if}}
112+
{{#if secret_access_key}}
113+
secret_access_key: {{secret_access_key}}
114+
{{/if}}
115+
{{#if session_token}}
116+
session_token: {{session_token}}
117+
{{/if}}
118+
{{#if role_arn}}
119+
role_arn: {{role_arn}}
120+
{{/if}}
121+
{{#if fips_enabled}}
122+
fips_enabled: {{fips_enabled}}
123+
{{/if}}
124+
{{#if proxy_url }}
125+
proxy_url: {{proxy_url}}
126+
{{/if}}
127+
{{#if parsers}}
128+
parsers:
129+
{{parsers}}
130+
{{/if}}
131+
132+
{{#if tags.length}}
133+
tags:
134+
{{#each tags as |tag|}}
135+
- {{tag}}
136+
{{/each}}
137+
{{#if preserve_original_event}}
138+
- preserve_original_event
139+
{{/if}}
140+
{{else}}
141+
{{#if preserve_original_event}}
142+
tags:
143+
- preserve_original_event
144+
{{/if}}
145+
{{/if}}
146+
147+
{{#contains "forwarded" tags}}
148+
publisher_pipeline.disable_host: true
149+
{{/contains}}
150+
151+
{{#if processors}}
152+
processors:
153+
{{processors}}
154+
{{/if}}

0 commit comments

Comments
 (0)