diff --git a/internal/docs/exported_fields.go b/internal/docs/exported_fields.go index df5830484..5c05475d5 100644 --- a/internal/docs/exported_fields.go +++ b/internal/docs/exported_fields.go @@ -9,20 +9,27 @@ import ( "sort" "strings" + "github.com/Masterminds/semver/v3" + "github.com/elastic/elastic-package/internal/fields" + "github.com/elastic/elastic-package/internal/packages" ) +var semver3_2_3 = semver.MustParse("3.4.0") + type fieldsTableRecord struct { name string description string aType string unit string metricType string + value string + example string } var escaper = strings.NewReplacer("*", "\\*", "{", "\\{", "}", "\\}", "<", "\\<", ">", "\\>") -func renderExportedFields(fieldsParentDir string) (string, error) { +func renderExportedFields(fieldsParentDir string, pkgManifest *packages.PackageManifest) (string, error) { injectOptions := fields.InjectFieldsOptions{ // Keep External parameter when rendering fields, so we can render // documentation for empty groups imported from ECS, for backwards compatibility. @@ -46,13 +53,27 @@ func renderExportedFields(fieldsParentDir string) (string, error) { builder.WriteString("(no fields available)\n") return builder.String(), nil } - renderFieldsTable(&builder, collected) + + sv, err := semver.NewVersion(pkgManifest.SpecVersion) + if err != nil { + return "", fmt.Errorf("failed to obtain spec version from package manifest: %w", err) + } + + renderExamples := true + if sv.LessThan(semver3_2_3) { + renderExamples = false + } + + renderFieldsTable(&builder, collected, renderExamples) + return builder.String(), nil } -func renderFieldsTable(builder *strings.Builder, collected []fieldsTableRecord) { +func renderFieldsTable(builder *strings.Builder, collected []fieldsTableRecord, includeExamples bool) { unitsPresent := areUnitsPresent(collected) metricTypesPresent := areMetricTypesPresent(collected) + examplePresent := areExamplesPresent(collected) + valuesPresent := areValuesPresent(collected) builder.WriteString("| Field | Description | Type |") if unitsPresent { @@ -61,6 +82,9 @@ func renderFieldsTable(builder *strings.Builder, collected []fieldsTableRecord) if metricTypesPresent { builder.WriteString(" Metric Type |") } + if (examplePresent || valuesPresent) && includeExamples { + builder.WriteString(" Example |") + } builder.WriteString("\n") builder.WriteString("|---|---|---|") @@ -70,6 +94,9 @@ func renderFieldsTable(builder *strings.Builder, collected []fieldsTableRecord) if metricTypesPresent { builder.WriteString("---|") } + if (examplePresent || valuesPresent) && includeExamples { // Check whether there are examples to render or whether fields are constants + builder.WriteString("---|") + } builder.WriteString("\n") for _, c := range collected { @@ -84,6 +111,13 @@ func renderFieldsTable(builder *strings.Builder, collected []fieldsTableRecord) if metricTypesPresent { builder.WriteString(fmt.Sprintf(" %s |", c.metricType)) } + if (examplePresent || valuesPresent) && includeExamples { + if c.aType == "constant_keyword" { + builder.WriteString(fmt.Sprintf(" %s |", c.value)) + } else { + builder.WriteString(fmt.Sprintf(" %s |", c.example)) + } + } builder.WriteString("\n") } } @@ -106,6 +140,24 @@ func areMetricTypesPresent(collected []fieldsTableRecord) bool { return false } +func areValuesPresent(collected []fieldsTableRecord) bool { + for _, c := range collected { + if c.aType == "constant_keyword" && c.value != "" { + return true + } + } + return false +} + +func areExamplesPresent(collected []fieldsTableRecord) bool { + for _, c := range collected { + if c.example != "" { + return true + } + } + return false +} + func collectFieldsFromDefinitions(validator *fields.Validator) []fieldsTableRecord { var records []fieldsTableRecord @@ -130,6 +182,8 @@ func visitFields(namePrefix string, f fields.FieldDefinition, records []fieldsTa aType: f.Type, unit: f.Unit, metricType: f.MetricType, + value: f.Value, + example: f.Example, }) for _, multiField := range f.MultiFields { diff --git a/internal/docs/readme.go b/internal/docs/readme.go index 256b7d2d9..a3e6fc115 100644 --- a/internal/docs/readme.go +++ b/internal/docs/readme.go @@ -191,11 +191,15 @@ func renderReadme(fileName, packageRoot, templatePath string, linksMap linkMap) return renderSampleEvent(packageRoot, "") }, "fields": func(args ...string) (string, error) { + manifest, err := packages.ReadPackageManifestFromPackageRoot(packageRoot) + if err != nil { + return "", fmt.Errorf("reading package manifest failed (path: %s): %w", packageRoot, err) + } if len(args) > 0 { dataStreamPath := filepath.Join(packageRoot, "data_stream", args[0]) - return renderExportedFields(dataStreamPath) + return renderExportedFields(dataStreamPath, manifest) } - return renderExportedFields(packageRoot) + return renderExportedFields(packageRoot, manifest) }, "url": func(args ...string) (string, error) { options := linkOptions{} diff --git a/internal/fields/model.go b/internal/fields/model.go index b72463955..d14af0520 100644 --- a/internal/fields/model.go +++ b/internal/fields/model.go @@ -27,6 +27,7 @@ type FieldDefinition struct { Unit string `yaml:"unit"` MetricType string `yaml:"metric_type"` External string `yaml:"external"` + Example string `yaml:"example"` Index *bool `yaml:"index"` Enabled *bool `yaml:"enabled"` DocValues *bool `yaml:"doc_values"` diff --git a/internal/fields/testdata/disabled.json b/internal/fields/testdata/disabled.json index 96ed92797..1e5918711 100644 --- a/internal/fields/testdata/disabled.json +++ b/internal/fields/testdata/disabled.json @@ -1,6 +1,6 @@ -{ - "disabled": { - "id": "42", - "status": "ok" - } -} +{ + "disabled": { + "id": "42", + "status": "ok" + } +} diff --git a/internal/fields/testdata/enabled_not_mapped.json b/internal/fields/testdata/enabled_not_mapped.json index 0acaa2ac0..7641ccdc5 100644 --- a/internal/fields/testdata/enabled_not_mapped.json +++ b/internal/fields/testdata/enabled_not_mapped.json @@ -1,6 +1,6 @@ -{ - "enabled": { - "id": "42", - "status": "ok" - } -} +{ + "enabled": { + "id": "42", + "status": "ok" + } +} diff --git a/test/packages/parallel/ti_anomali/changelog.yml b/test/packages/parallel/ti_anomali/changelog.yml index bbdef7327..60ff9d648 100644 --- a/test/packages/parallel/ti_anomali/changelog.yml +++ b/test/packages/parallel/ti_anomali/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.23.0-rc1" + changes: + - description: Update format spec to 3.4.0 + type: enhancement + link: https://github.com/elastic/elastic-package/pull/2167 - version: "1.22.1" changes: - description: Fix ECS date mapping on threat fields. diff --git a/test/packages/parallel/ti_anomali/docs/README.md b/test/packages/parallel/ti_anomali/docs/README.md index 02f9ef713..64f0e686e 100644 --- a/test/packages/parallel/ti_anomali/docs/README.md +++ b/test/packages/parallel/ti_anomali/docs/README.md @@ -141,44 +141,44 @@ An example event for `threatstream` looks as following: **Exported fields** -| Field | Description | Type | -|---|---|---| -| @timestamp | Event timestamp. | date | -| anomali.threatstream.added_at | Date when IOC was added. | date | -| anomali.threatstream.classification | Indicates whether an indicator is private or from a public feed and available publicly. Possible values: private, public. | keyword | -| anomali.threatstream.confidence | The measure of the accuracy (from 0 to 100) assigned by ThreatStream's predictive analytics technology to indicators. | short | -| anomali.threatstream.deleted_at | Date when IOC was deleted/expired. | date | -| anomali.threatstream.detail2 | Detail text for indicator. | text | -| anomali.threatstream.id | The ID of the indicator. | keyword | -| anomali.threatstream.import_session_id | ID of the import session that created the indicator on ThreatStream. | keyword | -| anomali.threatstream.itype | Indicator type. Possible values: "apt_domain", "apt_email", "apt_ip", "apt_url", "bot_ip", "c2_domain", "c2_ip", "c2_url", "i2p_ip", "mal_domain", "mal_email", "mal_ip", "mal_md5", "mal_url", "parked_ip", "phish_email", "phish_ip", "phish_url", "scan_ip", "spam_domain", "ssh_ip", "suspicious_domain", "tor_ip" and "torrent_tracker_url". | keyword | -| anomali.threatstream.maltype | Information regarding a malware family, a CVE ID, or another attack or threat, associated with the indicator. | wildcard | -| anomali.threatstream.md5 | Hash for the indicator. | keyword | -| anomali.threatstream.resource_uri | Relative URI for the indicator details. | keyword | -| anomali.threatstream.severity | Criticality associated with the threat feed that supplied the indicator. Possible values: low, medium, high, very-high. | keyword | -| anomali.threatstream.source | Source for the indicator. | keyword | -| anomali.threatstream.source_feed_id | ID for the integrator source. | keyword | -| anomali.threatstream.state | State for this indicator. | keyword | -| anomali.threatstream.trusted_circle_ids | ID of the trusted circle that imported the indicator. | keyword | -| anomali.threatstream.update_id | Update ID. | keyword | -| anomali.threatstream.url | URL for the indicator. | keyword | -| anomali.threatstream.value_type | Data type of the indicator. Possible values: ip, domain, url, email, md5. | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | -| data_stream.dataset | Data stream dataset name. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| event.dataset | Event dataset | constant_keyword | -| event.module | Event module | constant_keyword | -| host.containerized | If the host is a container. | boolean | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | -| input.type | Type of Filebeat input. | keyword | -| labels.is_ioc_transform_source | Field indicating if its the transform source for supporting IOC expiration. This field is dropped from destination indices to facilitate easier filtering of indicators. | constant_keyword | -| log.flags | Flags for the log file. | keyword | -| log.offset | Offset of the entry in the log file. | long | -| threat.feed.dashboard_id | Dashboard ID used for Kibana CTI UI | constant_keyword | -| threat.feed.name | Display friendly feed name | constant_keyword | -| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date | -| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date | -| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date | +| Field | Description | Type | Example | +|---|---|---|---| +| @timestamp | Event timestamp. | date | | +| anomali.threatstream.added_at | Date when IOC was added. | date | | +| anomali.threatstream.classification | Indicates whether an indicator is private or from a public feed and available publicly. Possible values: private, public. | keyword | private | +| anomali.threatstream.confidence | The measure of the accuracy (from 0 to 100) assigned by ThreatStream's predictive analytics technology to indicators. | short | | +| anomali.threatstream.deleted_at | Date when IOC was deleted/expired. | date | | +| anomali.threatstream.detail2 | Detail text for indicator. | text | Imported by user 42. | +| anomali.threatstream.id | The ID of the indicator. | keyword | | +| anomali.threatstream.import_session_id | ID of the import session that created the indicator on ThreatStream. | keyword | | +| anomali.threatstream.itype | Indicator type. Possible values: "apt_domain", "apt_email", "apt_ip", "apt_url", "bot_ip", "c2_domain", "c2_ip", "c2_url", "i2p_ip", "mal_domain", "mal_email", "mal_ip", "mal_md5", "mal_url", "parked_ip", "phish_email", "phish_ip", "phish_url", "scan_ip", "spam_domain", "ssh_ip", "suspicious_domain", "tor_ip" and "torrent_tracker_url". | keyword | | +| anomali.threatstream.maltype | Information regarding a malware family, a CVE ID, or another attack or threat, associated with the indicator. | wildcard | | +| anomali.threatstream.md5 | Hash for the indicator. | keyword | | +| anomali.threatstream.resource_uri | Relative URI for the indicator details. | keyword | | +| anomali.threatstream.severity | Criticality associated with the threat feed that supplied the indicator. Possible values: low, medium, high, very-high. | keyword | | +| anomali.threatstream.source | Source for the indicator. | keyword | Analyst | +| anomali.threatstream.source_feed_id | ID for the integrator source. | keyword | | +| anomali.threatstream.state | State for this indicator. | keyword | active | +| anomali.threatstream.trusted_circle_ids | ID of the trusted circle that imported the indicator. | keyword | | +| anomali.threatstream.update_id | Update ID. | keyword | | +| anomali.threatstream.url | URL for the indicator. | keyword | | +| anomali.threatstream.value_type | Data type of the indicator. Possible values: ip, domain, url, email, md5. | keyword | | +| cloud.image.id | Image ID for the cloud instance. | keyword | | +| data_stream.dataset | Data stream dataset name. | constant_keyword | | +| data_stream.namespace | Data stream namespace. | constant_keyword | | +| data_stream.type | Data stream type. | constant_keyword | | +| event.dataset | Event dataset | constant_keyword | ti_anomali.threatstream | +| event.module | Event module | constant_keyword | ti_anomali | +| host.containerized | If the host is a container. | boolean | | +| host.os.build | OS build information. | keyword | 18D109 | +| host.os.codename | OS codename, if any. | keyword | stretch | +| input.type | Type of Filebeat input. | keyword | | +| labels.is_ioc_transform_source | Field indicating if its the transform source for supporting IOC expiration. This field is dropped from destination indices to facilitate easier filtering of indicators. | constant_keyword | true | +| log.flags | Flags for the log file. | keyword | | +| log.offset | Offset of the entry in the log file. | long | | +| threat.feed.dashboard_id | Dashboard ID used for Kibana CTI UI | constant_keyword | ti_anomali-96fe1e60-4261-11ec-b7be-d3026acdf1cf | +| threat.feed.name | Display friendly feed name | constant_keyword | Anomali ThreatStream | +| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date | | +| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date | | +| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date | | diff --git a/test/packages/parallel/ti_anomali/manifest.yml b/test/packages/parallel/ti_anomali/manifest.yml index df591d3e9..3e4026ebe 100644 --- a/test/packages/parallel/ti_anomali/manifest.yml +++ b/test/packages/parallel/ti_anomali/manifest.yml @@ -1,9 +1,9 @@ name: ti_anomali title: Anomali -version: "1.22.1" +version: "1.23.0-rc1" description: Ingest threat intelligence indicators from Anomali with Elastic Agent. type: integration -format_version: 3.0.2 +format_version: 3.4.0 categories: ["security", "threat_intel"] conditions: kibana: