diff --git a/README.md b/README.md index d125db17b2..3707a833fa 100644 --- a/README.md +++ b/README.md @@ -354,6 +354,14 @@ Use this command to export dashboards with referenced objects from the Kibana in Use this command to download selected dashboards and other associated saved objects from Kibana. This command adjusts the downloaded saved objects according to package naming conventions (prefixes, unique IDs) and writes them locally into folders corresponding to saved object types (dashboard, visualization, map, etc.). +### `elastic-package export ingest-pipelines` + +_Context: package_ + +Use this command to export ingest pipelines with referenced pipelines from the Elasticsearch instance. + +Use this command to download selected ingest pipelines and its referenced processor pipelines from Elasticsearch. Select data stream or the package root directories to download the pipelines. Pipelines are downloaded as is and will need adjustment to meet your package needs. + ### `elastic-package format` _Context: package_ diff --git a/cmd/export.go b/cmd/export.go index da2adb56db..da96190000 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -5,27 +5,16 @@ package cmd import ( - "context" "fmt" - "github.com/AlecAivazis/survey/v2" - "github.com/spf13/cobra" "github.com/elastic/elastic-package/internal/cobraext" - "github.com/elastic/elastic-package/internal/common" - "github.com/elastic/elastic-package/internal/export" "github.com/elastic/elastic-package/internal/install" - "github.com/elastic/elastic-package/internal/kibana" - "github.com/elastic/elastic-package/internal/stack" ) const exportLongDescription = `Use this command to export assets relevant for the package, e.g. Kibana dashboards.` -const exportDashboardsLongDescription = `Use this command to export dashboards with referenced objects from the Kibana instance. - -Use this command to download selected dashboards and other associated saved objects from Kibana. This command adjusts the downloaded saved objects according to package naming conventions (prefixes, unique IDs) and writes them locally into folders corresponding to saved object types (dashboard, visualization, map, etc.).` - func setupExportCommand() *cobraext.Command { exportDashboardCmd := &cobra.Command{ Use: "dashboards", @@ -38,111 +27,26 @@ func setupExportCommand() *cobraext.Command { exportDashboardCmd.Flags().Bool(cobraext.TLSSkipVerifyFlagName, false, cobraext.TLSSkipVerifyFlagDescription) exportDashboardCmd.Flags().Bool(cobraext.AllowSnapshotFlagName, false, cobraext.AllowSnapshotDescription) + exportIngestPipelinesCmd := &cobra.Command{ + Use: "ingest-pipelines", + Short: "Export ingest pipelines from Elasticsearch", + Long: exportIngestPipelinesLongDescription, + Args: cobra.NoArgs, + RunE: exportIngestPipelinesCmd, + } + + exportIngestPipelinesCmd.Flags().StringSliceP(cobraext.IngestPipelineIDsFlagName, "d", nil, cobraext.IngestPipelineIDsFlagDescription) + exportIngestPipelinesCmd.Flags().Bool(cobraext.TLSSkipVerifyFlagName, false, cobraext.TLSSkipVerifyFlagDescription) + exportIngestPipelinesCmd.Flags().Bool(cobraext.AllowSnapshotFlagName, false, cobraext.AllowSnapshotDescription) + cmd := &cobra.Command{ Use: "export", Short: "Export package assets", Long: exportLongDescription, } cmd.AddCommand(exportDashboardCmd) + cmd.AddCommand(exportIngestPipelinesCmd) cmd.PersistentFlags().StringP(cobraext.ProfileFlagName, "p", "", fmt.Sprintf(cobraext.ProfileFlagDescription, install.ProfileNameEnvVar)) return cobraext.NewCommand(cmd, cobraext.ContextPackage) } - -func exportDashboardsCmd(cmd *cobra.Command, args []string) error { - cmd.Println("Export Kibana dashboards") - - dashboardIDs, err := cmd.Flags().GetStringSlice(cobraext.DashboardIDsFlagName) - if err != nil { - return cobraext.FlagParsingError(err, cobraext.DashboardIDsFlagName) - } - - common.TrimStringSlice(dashboardIDs) - - var opts []kibana.ClientOption - tlsSkipVerify, _ := cmd.Flags().GetBool(cobraext.TLSSkipVerifyFlagName) - if tlsSkipVerify { - opts = append(opts, kibana.TLSSkipVerify()) - } - - allowSnapshot, _ := cmd.Flags().GetBool(cobraext.AllowSnapshotFlagName) - if err != nil { - return cobraext.FlagParsingError(err, cobraext.AllowSnapshotFlagName) - } - - profile, err := cobraext.GetProfileFlag(cmd) - if err != nil { - return err - } - - kibanaClient, err := stack.NewKibanaClientFromProfile(profile, opts...) - if err != nil { - return fmt.Errorf("can't create Kibana client: %w", err) - } - - kibanaVersion, err := kibanaClient.Version() - if err != nil { - return fmt.Errorf("can't get Kibana status information: %w", err) - } - - if kibanaVersion.IsSnapshot() { - message := fmt.Sprintf("exporting dashboards from a SNAPSHOT version of Kibana (%s) is discouraged. It could lead to invalid dashboards (for example if they use features that are reverted or modified before the final release)", kibanaVersion.Version()) - if !allowSnapshot { - return fmt.Errorf("%s. --%s flag can be used to ignore this error", message, cobraext.AllowSnapshotFlagName) - } - fmt.Printf("Warning: %s\n", message) - } - - if len(dashboardIDs) == 0 { - dashboardIDs, err = promptDashboardIDs(cmd.Context(), kibanaClient) - if err != nil { - return fmt.Errorf("prompt for dashboard selection failed: %w", err) - } - - if len(dashboardIDs) == 0 { - fmt.Println("No dashboards were found in Kibana.") - return nil - } - } - - err = export.Dashboards(cmd.Context(), kibanaClient, dashboardIDs) - if err != nil { - return fmt.Errorf("dashboards export failed: %w", err) - } - - cmd.Println("Done") - return nil -} - -func promptDashboardIDs(ctx context.Context, kibanaClient *kibana.Client) ([]string, error) { - savedDashboards, err := kibanaClient.FindDashboards(ctx) - if err != nil { - return nil, fmt.Errorf("finding dashboards failed: %w", err) - } - - if len(savedDashboards) == 0 { - return []string{}, nil - } - - dashboardsPrompt := &survey.MultiSelect{ - Message: "Which dashboards would you like to export?", - Options: savedDashboards.Strings(), - PageSize: 100, - } - - var selectedOptions []string - err = survey.AskOne(dashboardsPrompt, &selectedOptions, survey.WithValidator(survey.Required)) - if err != nil { - return nil, err - } - - var selected []string - for _, option := range selectedOptions { - for _, sd := range savedDashboards { - if sd.String() == option { - selected = append(selected, sd.ID) - } - } - } - return selected, nil -} diff --git a/cmd/export_dashboards.go b/cmd/export_dashboards.go new file mode 100644 index 0000000000..bd6fa1e41c --- /dev/null +++ b/cmd/export_dashboards.go @@ -0,0 +1,122 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cmd + +import ( + "context" + "fmt" + + "github.com/AlecAivazis/survey/v2" + + "github.com/spf13/cobra" + + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/common" + "github.com/elastic/elastic-package/internal/export" + "github.com/elastic/elastic-package/internal/kibana" + "github.com/elastic/elastic-package/internal/stack" +) + +const exportDashboardsLongDescription = `Use this command to export dashboards with referenced objects from the Kibana instance. + +Use this command to download selected dashboards and other associated saved objects from Kibana. This command adjusts the downloaded saved objects according to package naming conventions (prefixes, unique IDs) and writes them locally into folders corresponding to saved object types (dashboard, visualization, map, etc.).` + +func exportDashboardsCmd(cmd *cobra.Command, args []string) error { + cmd.Println("Export Kibana dashboards") + + dashboardIDs, err := cmd.Flags().GetStringSlice(cobraext.DashboardIDsFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.DashboardIDsFlagName) + } + + common.TrimStringSlice(dashboardIDs) + + var opts []kibana.ClientOption + tlsSkipVerify, _ := cmd.Flags().GetBool(cobraext.TLSSkipVerifyFlagName) + if tlsSkipVerify { + opts = append(opts, kibana.TLSSkipVerify()) + } + + allowSnapshot, _ := cmd.Flags().GetBool(cobraext.AllowSnapshotFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.AllowSnapshotFlagName) + } + + profile, err := cobraext.GetProfileFlag(cmd) + if err != nil { + return err + } + + kibanaClient, err := stack.NewKibanaClientFromProfile(profile, opts...) + if err != nil { + return fmt.Errorf("can't create Kibana client: %w", err) + } + + kibanaVersion, err := kibanaClient.Version() + if err != nil { + return fmt.Errorf("can't get Kibana status information: %w", err) + } + + if kibanaVersion.IsSnapshot() { + message := fmt.Sprintf("exporting dashboards from a SNAPSHOT version of Kibana (%s) is discouraged. It could lead to invalid dashboards (for example if they use features that are reverted or modified before the final release)", kibanaVersion.Version()) + if !allowSnapshot { + return fmt.Errorf("%s. --%s flag can be used to ignore this error", message, cobraext.AllowSnapshotFlagName) + } + fmt.Printf("Warning: %s\n", message) + } + + if len(dashboardIDs) == 0 { + dashboardIDs, err = promptDashboardIDs(cmd.Context(), kibanaClient) + if err != nil { + return fmt.Errorf("prompt for dashboard selection failed: %w", err) + } + + if len(dashboardIDs) == 0 { + fmt.Println("No dashboards were found in Kibana.") + return nil + } + } + + err = export.Dashboards(cmd.Context(), kibanaClient, dashboardIDs) + if err != nil { + return fmt.Errorf("dashboards export failed: %w", err) + } + + cmd.Println("Done") + return nil +} + +func promptDashboardIDs(ctx context.Context, kibanaClient *kibana.Client) ([]string, error) { + savedDashboards, err := kibanaClient.FindDashboards(ctx) + if err != nil { + return nil, fmt.Errorf("finding dashboards failed: %w", err) + } + + if len(savedDashboards) == 0 { + return []string{}, nil + } + + dashboardsPrompt := &survey.MultiSelect{ + Message: "Which dashboards would you like to export?", + Options: savedDashboards.Strings(), + PageSize: 100, + } + + var selectedOptions []string + err = survey.AskOne(dashboardsPrompt, &selectedOptions, survey.WithValidator(survey.Required)) + if err != nil { + return nil, err + } + + var selected []string + for _, option := range selectedOptions { + for _, sd := range savedDashboards { + if sd.String() == option { + selected = append(selected, sd.ID) + } + } + } + return selected, nil +} diff --git a/cmd/export_ingest_pipelines.go b/cmd/export_ingest_pipelines.go new file mode 100644 index 0000000000..c12c1708f1 --- /dev/null +++ b/cmd/export_ingest_pipelines.go @@ -0,0 +1,210 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cmd + +import ( + "context" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/AlecAivazis/survey/v2" + + "github.com/spf13/cobra" + + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/common" + "github.com/elastic/elastic-package/internal/elasticsearch" + "github.com/elastic/elastic-package/internal/elasticsearch/ingest" + "github.com/elastic/elastic-package/internal/export" + "github.com/elastic/elastic-package/internal/packages" + "github.com/elastic/elastic-package/internal/stack" +) + +const exportIngestPipelinesLongDescription = `Use this command to export ingest pipelines with referenced pipelines from the Elasticsearch instance. + +Use this command to download selected ingest pipelines and its referenced processor pipelines from Elasticsearch. Select data stream or the package root directories to download the pipelines. Pipelines are downloaded as is and will need adjustment to meet your package needs.` + +func exportIngestPipelinesCmd(cmd *cobra.Command, args []string) error { + cmd.Println("Export Elasticsearch ingest pipelines") + + pipelineIDs, err := cmd.Flags().GetStringSlice(cobraext.IngestPipelineIDsFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.IngestPipelineIDsFlagName) + } + + common.TrimStringSlice(pipelineIDs) + + var opts []elasticsearch.ClientOption + tlsSkipVerify, _ := cmd.Flags().GetBool(cobraext.TLSSkipVerifyFlagName) + if tlsSkipVerify { + opts = append(opts, elasticsearch.OptionWithSkipTLSVerify()) + } + + profile, err := cobraext.GetProfileFlag(cmd) + if err != nil { + return err + } + + esClient, err := stack.NewElasticsearchClientFromProfile(profile, opts...) + if err != nil { + return fmt.Errorf("can't create Elasticsearch client: %w", err) + } + + if len(pipelineIDs) == 0 { + pipelineIDs, err = promptIngestPipelineIDs(cmd.Context(), esClient.API) + + if err != nil { + return fmt.Errorf("prompt for ingest pipeline selection failed: %w", err) + } + + if len(pipelineIDs) == 0 { + cmd.Println("No ingest pipelines were found in Elasticsearch.") + return nil + } + } + + packageRoot, err := packages.MustFindPackageRoot() + + if err != nil { + return fmt.Errorf("locating package root failed: %w", err) + } + + dataStreamDirs, err := getDataStreamDirs(packageRoot) + + if err != nil { + return fmt.Errorf("getting data stream directories failed: %w", err) + } + + rootWriteLocation := export.PipelineWriteLocation{ + Type: export.PipelineWriteLocationTypeRoot, + Name: "Package root", + ParentPath: packageRoot, + } + + pipelineWriteLocations := append(dataStreamDirs, rootWriteLocation) + + pipelineWriteAssignments, err := promptWriteLocations(pipelineIDs, pipelineWriteLocations) + + if err != nil { + return fmt.Errorf("prompt for ingest pipeline export locations failed: %w", err) + } + + err = export.IngestPipelines(cmd.Context(), esClient.API, pipelineWriteAssignments) + + if err != nil { + return err + } + + cmd.Println("Done") + return nil +} + +func getDataStreamDirs(packageRoot string) ([]export.PipelineWriteLocation, error) { + dataStreamDir := filepath.Join(packageRoot, "data_stream") + + _, err := os.Stat(dataStreamDir) + + if err != nil { + return nil, fmt.Errorf("data_stream directory does not exist: %w", err) + } + + dataStreamEntries, err := os.ReadDir(dataStreamDir) + + if err != nil { + return nil, fmt.Errorf("could not read data_stream directory: %w", err) + } + + var dataStreamDirs []export.PipelineWriteLocation + + for _, dirEntry := range dataStreamEntries { + if dirEntry.IsDir() { + pipelineWriteLocation := export.PipelineWriteLocation{ + Type: export.PipelineWriteLocationTypeDataStream, + Name: dirEntry.Name(), + ParentPath: filepath.Join(dataStreamDir, dirEntry.Name()), + } + dataStreamDirs = append(dataStreamDirs, pipelineWriteLocation) + } + } + + return dataStreamDirs, nil +} + +func promptIngestPipelineIDs(ctx context.Context, api *elasticsearch.API) ([]string, error) { + ingestPipelineNames, err := ingest.GetRemotePipelineNames(ctx, api) + if err != nil { + return nil, fmt.Errorf("finding ingest pipelines failed: %w", err) + } + + ingestPipelineNames = slices.DeleteFunc(ingestPipelineNames, func(name string) bool { + // Filter out system pipelines that start with dot "." or global@ + return strings.HasPrefix(name, ".") || strings.HasPrefix(name, "global@") + }) + + ingestPipelinesPrompt := &survey.MultiSelect{ + Message: "Which ingest pipelines would you like to export?", + Options: ingestPipelineNames, + PageSize: 20, + } + + var selectedOptions []string + err = survey.AskOne(ingestPipelinesPrompt, &selectedOptions, survey.WithValidator(survey.Required)) + if err != nil { + return nil, err + } + + return selectedOptions, nil +} + +func promptWriteLocations(pipelineNames []string, writeLocations []export.PipelineWriteLocation) (export.PipelineWriteAssignments, error) { + var options []string + + for _, writeLocation := range writeLocations { + options = append(options, writeLocation.Name) + } + + var questions []*survey.Question + + for _, pipelineName := range pipelineNames { + question := &survey.Question{ + Name: pipelineName, + Prompt: &survey.Select{ + Message: fmt.Sprintf("Select a location to export ingest pipeline '%s'", pipelineName), + Options: options, + Description: func(value string, index int) string { + if writeLocations[index].Type == export.PipelineWriteLocationTypeDataStream { + return "data stream" + } + + return "" + }, + }, + Validate: survey.Required, + } + + questions = append(questions, question) + } + + answers := make(map[string]string) + + err := survey.Ask(questions, &answers) + + if err != nil { + return nil, err + } + + pipelinesToWriteLocations := make(export.PipelineWriteAssignments) + + for pipeline, writeLocationName := range answers { + writeLocationIdx := slices.IndexFunc(writeLocations, func(p export.PipelineWriteLocation) bool { return p.Name == writeLocationName }) + + pipelinesToWriteLocations[pipeline] = writeLocations[writeLocationIdx] + } + + return pipelinesToWriteLocations, nil +} diff --git a/internal/cobraext/flags.go b/internal/cobraext/flags.go index c8f19eff22..6307c553e0 100644 --- a/internal/cobraext/flags.go +++ b/internal/cobraext/flags.go @@ -138,6 +138,9 @@ const ( GenerateTestResultFlagName = "generate" GenerateTestResultFlagDescription = "generate test result file" + IngestPipelineIDsFlagName = "id" + IngestPipelineIDsFlagDescription = "Elasticsearch ingest pipeline IDs (comma-separated values)" + ProfileFlagName = "profile" ProfileFlagDescription = "select a profile to use for the stack configuration. Can also be set with %s" diff --git a/internal/dump/indextemplates.go b/internal/dump/indextemplates.go index e1cba11b4e..73aba95269 100644 --- a/internal/dump/indextemplates.go +++ b/internal/dump/indextemplates.go @@ -13,6 +13,7 @@ import ( type IndexTemplate = ingest.IndexTemplate type TemplateSettings = ingest.TemplateSettings +type RemoteIngestPipeline = ingest.RemotePipeline func getIndexTemplatesForPackage(ctx context.Context, api *elasticsearch.API, packageName string) ([]ingest.IndexTemplate, error) { return ingest.GetIndexTemplatesForPackage(ctx, api, packageName) diff --git a/internal/dump/ingestpipelines.go b/internal/dump/ingestpipelines.go index 77a534b443..22c55fc003 100644 --- a/internal/dump/ingestpipelines.go +++ b/internal/dump/ingestpipelines.go @@ -6,122 +6,15 @@ package dump import ( "context" - "encoding/json" - "fmt" - "io" - "net/http" - "slices" "github.com/elastic/elastic-package/internal/elasticsearch" + "github.com/elastic/elastic-package/internal/elasticsearch/ingest" ) -// IngestPipeline contains the information needed to export an ingest pipeline. -type IngestPipeline struct { - Processors []struct { - Pipeline *struct { - Name string `json:"name"` - } `json:"pipeline,omitempty"` - } `json:"processors"` - - id string - raw []byte -} - -// Name returns the name of the ingest pipeline. -func (p IngestPipeline) Name() string { - return p.id -} - -// JSON returns the JSON representation of the ingest pipeline. -func (p IngestPipeline) JSON() []byte { - return p.raw -} - -func getIngestPipelines(ctx context.Context, api *elasticsearch.API, ids ...string) ([]IngestPipeline, error) { +func getIngestPipelines(ctx context.Context, api *elasticsearch.API, ids ...string) ([]RemoteIngestPipeline, error) { if len(ids) == 0 { return nil, nil } - var pipelines []IngestPipeline - var collected []string - pending := ids - for len(pending) > 0 { - for _, id := range pending { - resultPipelines, err := getIngestPipelineByID(ctx, api, id) - if err != nil { - return nil, err - } - pipelines = append(pipelines, resultPipelines...) - } - collected = append(collected, pending...) - pending = pendingNestedPipelines(pipelines, collected) - } - - return pipelines, nil -} - -type getIngestPipelineResponse map[string]json.RawMessage - -func getIngestPipelineByID(ctx context.Context, api *elasticsearch.API, id string) ([]IngestPipeline, error) { - resp, err := api.Ingest.GetPipeline( - api.Ingest.GetPipeline.WithContext(ctx), - api.Ingest.GetPipeline.WithPipelineID(id), - ) - if err != nil { - return nil, fmt.Errorf("failed to get ingest pipeline %s: %w", id, err) - } - defer resp.Body.Close() - - // Ingest templates referenced by other templates may not exist. - if resp.StatusCode == http.StatusNotFound { - return nil, nil - } - if resp.IsError() { - return nil, fmt.Errorf("failed to get ingest pipeline %s: %s", id, resp.String()) - } - - d, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read response body: %w", err) - } - - var pipelinesResponse getIngestPipelineResponse - err = json.Unmarshal(d, &pipelinesResponse) - if err != nil { - return nil, fmt.Errorf("failed to decode response: %w", err) - } - - var pipelines []IngestPipeline - for id, raw := range pipelinesResponse { - var pipeline IngestPipeline - err := json.Unmarshal(raw, &pipeline) - if err != nil { - return nil, fmt.Errorf("failed to decode pipeline %s: %w", id, err) - } - pipeline.id = id - pipeline.raw = raw - pipelines = append(pipelines, pipeline) - } - - return pipelines, nil -} - -func pendingNestedPipelines(pipelines []IngestPipeline, collected []string) []string { - var names []string - for _, p := range pipelines { - for _, processor := range p.Processors { - if processor.Pipeline == nil { - continue - } - name := processor.Pipeline.Name - if slices.Contains(collected, name) { - continue - } - if slices.Contains(names, name) { - continue - } - names = append(names, name) - } - } - return names + return ingest.GetRemotePipelinesWithNested(ctx, api, ids...) } diff --git a/internal/dump/installedobjects.go b/internal/dump/installedobjects.go index cdc4d746a1..6e1ed82ccc 100644 --- a/internal/dump/installedobjects.go +++ b/internal/dump/installedobjects.go @@ -29,7 +29,7 @@ type InstalledObjectsDumper struct { componentTemplates []ComponentTemplate ilmPolicies []ILMPolicy indexTemplates []IndexTemplate - ingestPipelines []IngestPipeline + ingestPipelines []RemoteIngestPipeline mlModels []MLModel } @@ -213,7 +213,7 @@ func (e *InstalledObjectsDumper) dumpIngestPipelines(ctx context.Context, dir st return len(ingestPipelines), nil } -func (e *InstalledObjectsDumper) getIngestPipelines(ctx context.Context) ([]IngestPipeline, error) { +func (e *InstalledObjectsDumper) getIngestPipelines(ctx context.Context) ([]RemoteIngestPipeline, error) { if len(e.ingestPipelines) == 0 { templates, err := e.getTemplatesWithSettings(ctx) if err != nil { diff --git a/internal/dump/installedobjects_test.go b/internal/dump/installedobjects_test.go index 073c3a78a1..fd15177f1b 100644 --- a/internal/dump/installedobjects_test.go +++ b/internal/dump/installedobjects_test.go @@ -9,13 +9,17 @@ import ( "encoding/json" "errors" "io/fs" + "net/http" "os" "path/filepath" + "slices" + "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "gopkg.in/dnaeon/go-vcr.v3/cassette" estest "github.com/elastic/elastic-package/internal/elasticsearch/test" "github.com/elastic/elastic-package/internal/files" @@ -39,18 +43,20 @@ func TestDumpInstalledObjects(t *testing.T) { &installedObjectsDumpSuite{ // To reproduce the scenario: // - Start the stack with version 7.16.2. - // - Install apache package (1.3.4). + // - Install apache package (1.3.5). PackageName: "apache", Record: "./testdata/elasticsearch-7-mock-dump-apache", DumpDir: "./testdata/elasticsearch-7-apache-dump-all", + Matcher: ingestPipelineRequestMatcher, }, &installedObjectsDumpSuite{ // To reproduce the scenario: // - Start the stack with version 8.1.0. - // - Install apache package (1.3.6). + // - Install apache package (1.8.2). PackageName: "apache", Record: "./testdata/elasticsearch-8-mock-dump-apache", DumpDir: "./testdata/elasticsearch-8-apache-dump-all", + Matcher: ingestPipelineRequestMatcher, }, &installedObjectsDumpSuite{ // To reproduce the scenario: @@ -60,6 +66,7 @@ func TestDumpInstalledObjects(t *testing.T) { PackageName: "dga", Record: "./testdata/elasticsearch-8-mock-dump-dga", DumpDir: "./testdata/elasticsearch-8-dga-dump-all", + Matcher: ingestPipelineRequestMatcher, }, } @@ -79,6 +86,9 @@ type installedObjectsDumpSuite struct { // DumpDir is where the expected dumped files are stored. DumpDir string + + // Function that helps match an outbound request to a recorded one + Matcher cassette.MatcherFunc } func (s *installedObjectsDumpSuite) SetupTest() { @@ -97,7 +107,7 @@ func (s *installedObjectsDumpSuite) SetupTest() { } func (s *installedObjectsDumpSuite) TestDumpAll() { - client := estest.NewClient(s.T(), s.Record) + client := estest.NewClient(s.T(), s.Record, s.Matcher) outputDir := s.T().TempDir() dumper := NewInstalledObjectsDumper(client.API, s.PackageName) @@ -114,7 +124,7 @@ func (s *installedObjectsDumpSuite) TestDumpAll() { } func (s *installedObjectsDumpSuite) TestDumpSome() { - client := estest.NewClient(s.T(), s.Record) + client := estest.NewClient(s.T(), s.Record, s.Matcher) dumper := NewInstalledObjectsDumper(client.API, s.PackageName) // In a map so order of execution is randomized. @@ -217,3 +227,26 @@ func subDir(t *testing.T, dir, name string) string { return tmpDir } + +// Ingest Pipelines are requested in bulk and the param values are non-deterministic, +// which makes matching to recorded requests flaky. +// This custom cassette matcher helps match pipeline requests, and sends all others to the default matcher. +func ingestPipelineRequestMatcher(r *http.Request, cr cassette.Request) bool { + urlStartPattern := "https://127.0.0.1:9200/_ingest/pipeline/" + rSplitUrl := strings.Split(r.URL.String(), urlStartPattern) + crSplitUrl := strings.Split(cr.URL, urlStartPattern) + + isURLsPattern := len(rSplitUrl) == 2 && len(crSplitUrl) == 2 + + if !isURLsPattern { + return cassette.DefaultMatcher(r, cr) + } + + rPipelineValues := strings.Split(rSplitUrl[1], ",") + crPipelineValues := strings.Split(crSplitUrl[1], ",") + + slices.Sort(rPipelineValues) + slices.Sort(crPipelineValues) + + return slices.Equal(rPipelineValues, crPipelineValues) +} diff --git a/internal/dump/testdata/elasticsearch-7-apache-dump-all/ilm_policies/logs.json b/internal/dump/testdata/elasticsearch-7-apache-dump-all/ilm_policies/logs.json index 66ae87ef6a..e0d8ec6062 100644 --- a/internal/dump/testdata/elasticsearch-7-apache-dump-all/ilm_policies/logs.json +++ b/internal/dump/testdata/elasticsearch-7-apache-dump-all/ilm_policies/logs.json @@ -1,6 +1,6 @@ { "version": 1, - "modified_date": "2023-11-27T13:55:33.162Z", + "modified_date": "2025-05-08T16:05:52.238Z", "policy": { "phases": { "hot": { @@ -20,10 +20,10 @@ }, "in_use_by": { "indices": [ - ".ds-logs-elastic_agent.metricbeat-default-2023.11.27-000001", - ".ds-logs-elastic_agent.fleet_server-default-2023.11.27-000001", - ".ds-logs-elastic_agent.filebeat-default-2023.11.27-000001", - ".ds-logs-elastic_agent-default-2023.11.27-000001" + ".ds-logs-elastic_agent.filebeat-default-2025.05.08-000001", + ".ds-logs-elastic_agent.metricbeat-default-2025.05.08-000001", + ".ds-logs-elastic_agent-default-2025.05.08-000001", + ".ds-logs-elastic_agent.fleet_server-default-2025.05.08-000001" ], "data_streams": [ "logs-elastic_agent-default", diff --git a/internal/dump/testdata/elasticsearch-7-apache-dump-all/ilm_policies/metrics.json b/internal/dump/testdata/elasticsearch-7-apache-dump-all/ilm_policies/metrics.json index c12fdf1f50..e86fa51e65 100644 --- a/internal/dump/testdata/elasticsearch-7-apache-dump-all/ilm_policies/metrics.json +++ b/internal/dump/testdata/elasticsearch-7-apache-dump-all/ilm_policies/metrics.json @@ -1,6 +1,6 @@ { "version": 1, - "modified_date": "2023-11-27T13:55:33.210Z", + "modified_date": "2025-05-08T16:05:52.220Z", "policy": { "phases": { "hot": { @@ -20,26 +20,38 @@ }, "in_use_by": { "indices": [ - ".ds-metrics-system.process.summary-default-2023.11.27-000001", - ".ds-metrics-system.fsstat-default-2023.11.27-000001", - ".ds-metrics-system.uptime-default-2023.11.27-000001", - ".ds-metrics-system.network-default-2023.11.27-000001", - ".ds-metrics-system.filesystem-default-2023.11.27-000001", - ".ds-metrics-elastic_agent.elastic_agent-default-2023.11.27-000001", - ".ds-metrics-system.socket_summary-default-2023.11.27-000001", - ".ds-metrics-system.diskio-default-2023.11.27-000001", - ".ds-metrics-elastic_agent.filebeat-default-2023.11.27-000001", - ".ds-metrics-system.process-default-2023.11.27-000001", - ".ds-metrics-system.cpu-default-2023.11.27-000001", - ".ds-metrics-elastic_agent.fleet_server-default-2023.11.27-000001", - ".ds-metrics-elastic_agent.metricbeat-default-2023.11.27-000001", - ".ds-metrics-system.memory-default-2023.11.27-000001", - ".ds-metrics-system.load-default-2023.11.27-000001" + ".ds-metrics-system.process-default-2025.05.08-000001", + ".ds-metrics-docker.healthcheck-default-2025.05.08-000001", + ".ds-metrics-elastic_agent.elastic_agent-default-2025.05.08-000001", + ".ds-metrics-docker.network-default-2025.05.08-000001", + ".ds-metrics-system.load-default-2025.05.08-000001", + ".ds-metrics-docker.cpu-default-2025.05.08-000001", + ".ds-metrics-system.memory-default-2025.05.08-000001", + ".ds-metrics-system.socket_summary-default-2025.05.08-000001", + ".ds-metrics-system.cpu-default-2025.05.08-000001", + ".ds-metrics-system.diskio-default-2025.05.08-000001", + ".ds-metrics-elastic_agent.fleet_server-default-2025.05.08-000001", + ".ds-metrics-elastic_agent.filebeat-default-2025.05.08-000001", + ".ds-metrics-system.uptime-default-2025.05.08-000001", + ".ds-metrics-apache.status-default-2025.05.08-000001", + ".ds-metrics-elastic_agent.metricbeat-default-2025.05.08-000001", + ".ds-metrics-docker.container-default-2025.05.08-000001", + ".ds-metrics-docker.memory-default-2025.05.08-000001", + ".ds-metrics-system.process.summary-default-2025.05.08-000001", + ".ds-metrics-system.fsstat-default-2025.05.08-000001", + ".ds-metrics-system.network-default-2025.05.08-000001", + ".ds-metrics-docker.info-default-2025.05.08-000001", + ".ds-metrics-docker.diskio-default-2025.05.08-000001" ], "data_streams": [ - "metrics-system.filesystem-default", "metrics-system.cpu-default", + "metrics-docker.diskio-default", + "metrics-docker.container-default", + "metrics-docker.network-default", + "metrics-docker.cpu-default", + "metrics-docker.healthcheck-default", "metrics-system.process.summary-default", + "metrics-apache.status-default", "metrics-system.memory-default", "metrics-elastic_agent.fleet_server-default", "metrics-system.uptime-default", @@ -47,39 +59,50 @@ "metrics-elastic_agent.metricbeat-default", "metrics-system.fsstat-default", "metrics-system.process-default", + "metrics-docker.info-default", "metrics-elastic_agent.filebeat-default", "metrics-system.network-default", "metrics-system.diskio-default", + "metrics-docker.memory-default", "metrics-system.load-default", "metrics-system.socket_summary-default" ], "composable_templates": [ "metrics-system.process", "metrics-elastic_agent.packetbeat", + "metrics-docker.memory", "metrics-system.fsstat", "metrics-elastic_agent.osquerybeat", "metrics-elastic_agent.endpoint_security", "metrics-elastic_agent.apm_server", + "metrics-docker.info", "metrics-system.memory", + "metrics-docker.network", "metrics-system.socket_summary", + "metrics-docker.image", "metrics-apache.status", "metrics-elastic_agent.elastic_agent", "metrics-elastic_agent.fleet_server", "metrics-system.load", + "metrics-docker.cpu", "metrics-system.core", "metrics-elastic_agent.filebeat", "metrics-elastic_agent.filebeat_input", "metrics-system.uptime", + "metrics-docker.container", "metrics-system.process.summary", "metrics-system.cpu", + "metrics-docker.diskio", "metrics-elastic_agent.heartbeat", + "metrics-docker.healthcheck", "metrics-system.diskio", "metrics-elastic_agent.cloudbeat", "metrics-elastic_agent.metricbeat", "metrics-elastic_agent.auditbeat", "metrics-system.network", "metrics-system.filesystem", - "metrics" + "metrics", + "metrics-docker.event" ] } } \ No newline at end of file diff --git a/internal/dump/testdata/elasticsearch-7-apache-dump-all/index_templates/logs-apache.access.json b/internal/dump/testdata/elasticsearch-7-apache-dump-all/index_templates/logs-apache.access.json index 9e5936000b..3745d89b2c 100644 --- a/internal/dump/testdata/elasticsearch-7-apache-dump-all/index_templates/logs-apache.access.json +++ b/internal/dump/testdata/elasticsearch-7-apache-dump-all/index_templates/logs-apache.access.json @@ -7,7 +7,7 @@ "template": { "settings": { "index": { - "default_pipeline": "logs-apache.access-1.3.4" + "default_pipeline": "logs-apache.access-1.3.5" } }, "mappings": { @@ -137,8 +137,7 @@ "properties": { "name": { "ignore_above": 1024, - "type": "keyword", - "fields": {} + "type": "keyword" } } } @@ -268,8 +267,7 @@ "properties": { "path": { "ignore_above": 1024, - "type": "keyword", - "fields": {} + "type": "keyword" } } }, @@ -470,8 +468,7 @@ "properties": { "name": { "ignore_above": 1024, - "type": "keyword", - "fields": {} + "type": "keyword" } } }, @@ -479,15 +476,13 @@ "properties": { "original": { "ignore_above": 1024, - "type": "keyword", - "fields": {} + "type": "keyword" }, "os": { "properties": { "name": { "ignore_above": 1024, - "type": "keyword", - "fields": {} + "type": "keyword" }, "version": { "ignore_above": 1024, @@ -495,8 +490,7 @@ }, "full": { "ignore_above": 1024, - "type": "keyword", - "fields": {} + "type": "keyword" } } }, diff --git a/internal/dump/testdata/elasticsearch-7-apache-dump-all/index_templates/logs-apache.error.json b/internal/dump/testdata/elasticsearch-7-apache-dump-all/index_templates/logs-apache.error.json index 33748c52d7..f1eb9ba668 100644 --- a/internal/dump/testdata/elasticsearch-7-apache-dump-all/index_templates/logs-apache.error.json +++ b/internal/dump/testdata/elasticsearch-7-apache-dump-all/index_templates/logs-apache.error.json @@ -7,7 +7,7 @@ "template": { "settings": { "index": { - "default_pipeline": "logs-apache.error-1.3.4" + "default_pipeline": "logs-apache.error-1.3.5" } }, "mappings": { @@ -129,8 +129,7 @@ "properties": { "name": { "ignore_above": 1024, - "type": "keyword", - "fields": {} + "type": "keyword" } } } @@ -259,8 +258,7 @@ "properties": { "path": { "ignore_above": 1024, - "type": "keyword", - "fields": {} + "type": "keyword" } } }, @@ -438,8 +436,7 @@ "properties": { "name": { "ignore_above": 1024, - "type": "keyword", - "fields": {} + "type": "keyword" } } }, @@ -447,15 +444,13 @@ "properties": { "original": { "ignore_above": 1024, - "type": "keyword", - "fields": {} + "type": "keyword" }, "os": { "properties": { "name": { "ignore_above": 1024, - "type": "keyword", - "fields": {} + "type": "keyword" } } }, diff --git a/internal/dump/testdata/elasticsearch-7-apache-dump-all/ingest_pipelines/logs-apache.access-1.3.4-third-party.json b/internal/dump/testdata/elasticsearch-7-apache-dump-all/ingest_pipelines/logs-apache.access-1.3.5-third-party.json similarity index 100% rename from internal/dump/testdata/elasticsearch-7-apache-dump-all/ingest_pipelines/logs-apache.access-1.3.4-third-party.json rename to internal/dump/testdata/elasticsearch-7-apache-dump-all/ingest_pipelines/logs-apache.access-1.3.5-third-party.json diff --git a/internal/dump/testdata/elasticsearch-7-apache-dump-all/ingest_pipelines/logs-apache.access-1.3.4.json b/internal/dump/testdata/elasticsearch-7-apache-dump-all/ingest_pipelines/logs-apache.access-1.3.5.json similarity index 99% rename from internal/dump/testdata/elasticsearch-7-apache-dump-all/ingest_pipelines/logs-apache.access-1.3.4.json rename to internal/dump/testdata/elasticsearch-7-apache-dump-all/ingest_pipelines/logs-apache.access-1.3.5.json index e2e3a38164..fb4ca8190d 100644 --- a/internal/dump/testdata/elasticsearch-7-apache-dump-all/ingest_pipelines/logs-apache.access-1.3.4.json +++ b/internal/dump/testdata/elasticsearch-7-apache-dump-all/ingest_pipelines/logs-apache.access-1.3.5.json @@ -4,7 +4,7 @@ { "pipeline": { "if": "ctx.message.startsWith('{')", - "name": "logs-apache.access-1.3.4-third-party" + "name": "logs-apache.access-1.3.5-third-party" } }, { diff --git a/internal/dump/testdata/elasticsearch-7-apache-dump-all/ingest_pipelines/logs-apache.error-1.3.4-third-party.json b/internal/dump/testdata/elasticsearch-7-apache-dump-all/ingest_pipelines/logs-apache.error-1.3.5-third-party.json similarity index 100% rename from internal/dump/testdata/elasticsearch-7-apache-dump-all/ingest_pipelines/logs-apache.error-1.3.4-third-party.json rename to internal/dump/testdata/elasticsearch-7-apache-dump-all/ingest_pipelines/logs-apache.error-1.3.5-third-party.json diff --git a/internal/dump/testdata/elasticsearch-7-apache-dump-all/ingest_pipelines/logs-apache.error-1.3.4.json b/internal/dump/testdata/elasticsearch-7-apache-dump-all/ingest_pipelines/logs-apache.error-1.3.5.json similarity index 98% rename from internal/dump/testdata/elasticsearch-7-apache-dump-all/ingest_pipelines/logs-apache.error-1.3.4.json rename to internal/dump/testdata/elasticsearch-7-apache-dump-all/ingest_pipelines/logs-apache.error-1.3.5.json index abda1839b1..8e95cb9fdf 100644 --- a/internal/dump/testdata/elasticsearch-7-apache-dump-all/ingest_pipelines/logs-apache.error-1.3.4.json +++ b/internal/dump/testdata/elasticsearch-7-apache-dump-all/ingest_pipelines/logs-apache.error-1.3.5.json @@ -4,7 +4,7 @@ { "pipeline": { "if": "ctx.message.startsWith('{')", - "name": "logs-apache.error-1.3.4-third-party" + "name": "logs-apache.error-1.3.5-third-party" } }, { diff --git a/internal/dump/testdata/elasticsearch-7-mock-dump-apache.yaml b/internal/dump/testdata/elasticsearch-7-mock-dump-apache.yaml index acafd7bac3..9b455b000c 100644 --- a/internal/dump/testdata/elasticsearch-7-mock-dump-apache.yaml +++ b/internal/dump/testdata/elasticsearch-7-mock-dump-apache.yaml @@ -1,717 +1,594 @@ --- version: 2 interactions: - - id: 0 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: "" - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Authorization: - - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== - User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) - X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 - url: https://127.0.0.1:9200/ - method: GET - response: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - transfer_encoding: [] - trailer: {} - content_length: 545 - uncompressed: false - body: | - { - "name" : "8342dac6ee38", - "cluster_name" : "elasticsearch", - "cluster_uuid" : "SU93KHrmS8OHbarblnhy-g", - "version" : { - "number" : "7.16.2", - "build_flavor" : "default", - "build_type" : "docker", - "build_hash" : "2b937c44140b6559905130a8650c64dbd0879cfb", - "build_date" : "2021-12-18T19:42:46.604893745Z", - "build_snapshot" : false, - "lucene_version" : "8.10.1", - "minimum_wire_compatibility_version" : "6.8.0", - "minimum_index_compatibility_version" : "6.0.0-beta1" - }, - "tagline" : "You Know, for Search" - } - headers: - Content-Length: - - "545" - Content-Type: - - application/json; charset=UTF-8 - X-Elastic-Product: - - Elasticsearch - status: 200 OK - code: 200 - duration: 3.640457ms - - id: 1 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: "" - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Authorization: - - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== - User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) - X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 - url: https://127.0.0.1:9200/_index_template/*-apache.* - method: GET - response: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - transfer_encoding: [] - trailer: {} - content_length: 15969 - uncompressed: false - body: '{"index_templates":[{"name":"logs-apache.error","index_template":{"index_patterns":["logs-apache.error-*"],"template":{"settings":{"index":{"default_pipeline":"logs-apache.error-1.3.4"}},"mappings":{"_meta":{"package":{"name":"apache"},"managed_by":"ingest-manager","managed":true},"dynamic_templates":[{"strings_as_keyword":{"mapping":{"ignore_above":1024,"type":"keyword"},"match_mapping_type":"string"}}],"date_detection":false,"properties":{"container":{"properties":{"image":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}},"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"},"labels":{"type":"object"}}},"process":{"properties":{"pid":{"type":"long"},"thread":{"properties":{"id":{"type":"long"}}}}},"log":{"properties":{"file":{"properties":{"path":{"ignore_above":1024,"type":"keyword"}}},"offset":{"type":"long"},"level":{"ignore_above":1024,"type":"keyword"}}},"source":{"properties":{"geo":{"properties":{"continent_name":{"ignore_above":1024,"type":"keyword"},"region_iso_code":{"ignore_above":1024,"type":"keyword"},"city_name":{"ignore_above":1024,"type":"keyword"},"country_iso_code":{"ignore_above":1024,"type":"keyword"},"country_name":{"ignore_above":1024,"type":"keyword"},"location":{"type":"geo_point"},"region_name":{"ignore_above":1024,"type":"keyword"}}},"as":{"properties":{"number":{"type":"long"},"organization":{"properties":{"name":{"ignore_above":1024,"type":"keyword","fields":{}}}}}},"address":{"ignore_above":1024,"type":"keyword"},"port":{"type":"long"},"ip":{"type":"ip"}}},"error":{"properties":{"message":{"type":"match_only_text"}}},"message":{"type":"match_only_text"},"url":{"properties":{"path":{"type":"wildcard"},"extension":{"ignore_above":1024,"type":"keyword"},"original":{"type":"wildcard"},"domain":{"ignore_above":1024,"type":"keyword"},"query":{"ignore_above":1024,"type":"keyword"}}},"tags":{"ignore_above":1024,"type":"keyword"},"cloud":{"properties":{"availability_zone":{"ignore_above":1024,"type":"keyword"},"image":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}},"instance":{"properties":{"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"}}},"provider":{"ignore_above":1024,"type":"keyword"},"machine":{"properties":{"type":{"ignore_above":1024,"type":"keyword"}}},"project":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}},"region":{"ignore_above":1024,"type":"keyword"},"account":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}}}},"input":{"properties":{"type":{"ignore_above":1024,"type":"keyword"}}},"@timestamp":{"type":"date"},"file":{"properties":{"path":{"ignore_above":1024,"type":"keyword","fields":{}}}},"apache":{"properties":{"error":{"properties":{"module":{"ignore_above":1024,"type":"keyword"}}}}},"ecs":{"properties":{"version":{"ignore_above":1024,"type":"keyword"}}},"data_stream":{"properties":{"namespace":{"type":"constant_keyword"},"type":{"type":"constant_keyword"},"dataset":{"type":"constant_keyword"}}},"host":{"properties":{"hostname":{"ignore_above":1024,"type":"keyword"},"os":{"properties":{"build":{"ignore_above":1024,"type":"keyword"},"kernel":{"ignore_above":1024,"type":"keyword"},"codename":{"ignore_above":1024,"type":"keyword"},"name":{"ignore_above":1024,"type":"keyword","fields":{"text":{"type":"text"}}},"family":{"ignore_above":1024,"type":"keyword"},"version":{"ignore_above":1024,"type":"keyword"},"platform":{"ignore_above":1024,"type":"keyword"}}},"domain":{"ignore_above":1024,"type":"keyword"},"ip":{"type":"ip"},"containerized":{"type":"boolean"},"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"},"type":{"ignore_above":1024,"type":"keyword"},"mac":{"ignore_above":1024,"type":"keyword"},"architecture":{"ignore_above":1024,"type":"keyword"}}},"http":{"properties":{"request":{"properties":{"referrer":{"ignore_above":1024,"type":"keyword"},"method":{"ignore_above":1024,"type":"keyword"}}},"response":{"properties":{"status_code":{"type":"long"},"body":{"properties":{"bytes":{"type":"long"}}}}},"version":{"ignore_above":1024,"type":"keyword"}}},"event":{"properties":{"kind":{"ignore_above":1024,"type":"keyword"},"timezone":{"ignore_above":1024,"type":"keyword"},"module":{"type":"constant_keyword","value":"apache"},"category":{"ignore_above":1024,"type":"keyword"},"type":{"ignore_above":1024,"type":"keyword"},"dataset":{"type":"constant_keyword","value":"apache.error"}}},"user":{"properties":{"name":{"ignore_above":1024,"type":"keyword","fields":{}}}},"user_agent":{"properties":{"original":{"ignore_above":1024,"type":"keyword","fields":{}},"os":{"properties":{"name":{"ignore_above":1024,"type":"keyword","fields":{}}}},"name":{"ignore_above":1024,"type":"keyword"},"device":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}}}}}}},"composed_of":["logs-apache.error@settings","logs-apache.error@custom",".fleet_component_template-1"],"priority":200,"_meta":{"package":{"name":"apache"},"managed_by":"ingest-manager","managed":true},"data_stream":{"hidden":false}}},{"name":"logs-apache.access","index_template":{"index_patterns":["logs-apache.access-*"],"template":{"settings":{"index":{"default_pipeline":"logs-apache.access-1.3.4"}},"mappings":{"_meta":{"package":{"name":"apache"},"managed_by":"ingest-manager","managed":true},"dynamic_templates":[{"strings_as_keyword":{"mapping":{"ignore_above":1024,"type":"keyword"},"match_mapping_type":"string"}}],"date_detection":false,"properties":{"container":{"properties":{"image":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}},"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"},"labels":{"type":"object"}}},"process":{"properties":{"pid":{"type":"long"},"thread":{"properties":{"id":{"type":"long"}}}}},"log":{"properties":{"file":{"properties":{"path":{"ignore_above":1024,"type":"keyword"}}},"offset":{"type":"long"},"level":{"ignore_above":1024,"type":"keyword"}}},"destination":{"properties":{"domain":{"ignore_above":1024,"type":"keyword"}}},"source":{"properties":{"geo":{"properties":{"continent_name":{"ignore_above":1024,"type":"keyword"},"region_iso_code":{"ignore_above":1024,"type":"keyword"},"city_name":{"ignore_above":1024,"type":"keyword"},"country_iso_code":{"ignore_above":1024,"type":"keyword"},"country_name":{"ignore_above":1024,"type":"keyword"},"location":{"type":"geo_point"},"region_name":{"ignore_above":1024,"type":"keyword"}}},"as":{"properties":{"number":{"type":"long"},"organization":{"properties":{"name":{"ignore_above":1024,"type":"keyword","fields":{}}}}}},"address":{"ignore_above":1024,"type":"keyword"},"domain":{"ignore_above":1024,"type":"keyword"},"ip":{"type":"ip"}}},"error":{"properties":{"message":{"type":"match_only_text"}}},"message":{"type":"match_only_text"},"url":{"properties":{"path":{"type":"wildcard"},"extension":{"ignore_above":1024,"type":"keyword"},"original":{"type":"wildcard"},"domain":{"ignore_above":1024,"type":"keyword"},"query":{"ignore_above":1024,"type":"keyword"}}},"tags":{"ignore_above":1024,"type":"keyword"},"cloud":{"properties":{"availability_zone":{"ignore_above":1024,"type":"keyword"},"image":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}},"instance":{"properties":{"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"}}},"provider":{"ignore_above":1024,"type":"keyword"},"machine":{"properties":{"type":{"ignore_above":1024,"type":"keyword"}}},"project":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}},"region":{"ignore_above":1024,"type":"keyword"},"account":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}}}},"input":{"properties":{"type":{"ignore_above":1024,"type":"keyword"}}},"@timestamp":{"type":"date"},"file":{"properties":{"path":{"ignore_above":1024,"type":"keyword","fields":{}}}},"apache":{"properties":{"access":{"properties":{"ssl":{"properties":{"cipher":{"ignore_above":1024,"type":"keyword"},"protocol":{"ignore_above":1024,"type":"keyword"}}}}}}},"ecs":{"properties":{"version":{"ignore_above":1024,"type":"keyword"}}},"data_stream":{"properties":{"namespace":{"type":"constant_keyword"},"type":{"type":"constant_keyword"},"dataset":{"type":"constant_keyword"}}},"host":{"properties":{"hostname":{"ignore_above":1024,"type":"keyword"},"os":{"properties":{"build":{"ignore_above":1024,"type":"keyword"},"kernel":{"ignore_above":1024,"type":"keyword"},"codename":{"ignore_above":1024,"type":"keyword"},"name":{"ignore_above":1024,"type":"keyword","fields":{"text":{"type":"text"}}},"family":{"ignore_above":1024,"type":"keyword"},"version":{"ignore_above":1024,"type":"keyword"},"platform":{"ignore_above":1024,"type":"keyword"}}},"domain":{"ignore_above":1024,"type":"keyword"},"ip":{"type":"ip"},"containerized":{"type":"boolean"},"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"},"type":{"ignore_above":1024,"type":"keyword"},"mac":{"ignore_above":1024,"type":"keyword"},"architecture":{"ignore_above":1024,"type":"keyword"}}},"http":{"properties":{"request":{"properties":{"referrer":{"ignore_above":1024,"type":"keyword"},"method":{"ignore_above":1024,"type":"keyword"}}},"response":{"properties":{"status_code":{"type":"long"},"body":{"properties":{"bytes":{"type":"long"}}}}},"version":{"ignore_above":1024,"type":"keyword"}}},"tls":{"properties":{"cipher":{"ignore_above":1024,"type":"keyword"},"version":{"ignore_above":1024,"type":"keyword"},"version_protocol":{"ignore_above":1024,"type":"keyword"}}},"event":{"properties":{"created":{"type":"date"},"kind":{"ignore_above":1024,"type":"keyword"},"module":{"type":"constant_keyword","value":"apache"},"category":{"ignore_above":1024,"type":"keyword"},"dataset":{"type":"constant_keyword","value":"apache.access"},"outcome":{"ignore_above":1024,"type":"keyword"}}},"user":{"properties":{"name":{"ignore_above":1024,"type":"keyword","fields":{}}}},"user_agent":{"properties":{"original":{"ignore_above":1024,"type":"keyword","fields":{}},"os":{"properties":{"name":{"ignore_above":1024,"type":"keyword","fields":{}},"version":{"ignore_above":1024,"type":"keyword"},"full":{"ignore_above":1024,"type":"keyword","fields":{}}}},"name":{"ignore_above":1024,"type":"keyword"},"device":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}},"version":{"ignore_above":1024,"type":"keyword"}}}}}},"composed_of":["logs-apache.access@settings","logs-apache.access@custom",".fleet_component_template-1"],"priority":200,"_meta":{"package":{"name":"apache"},"managed_by":"ingest-manager","managed":true},"data_stream":{"hidden":false}}},{"name":"metrics-apache.status","index_template":{"index_patterns":["metrics-apache.status-*"],"template":{"settings":{},"mappings":{"_meta":{"package":{"name":"apache"},"managed_by":"ingest-manager","managed":true},"dynamic_templates":[{"strings_as_keyword":{"mapping":{"ignore_above":1024,"type":"keyword"},"match_mapping_type":"string"}}],"date_detection":false,"properties":{"cloud":{"properties":{"availability_zone":{"ignore_above":1024,"type":"keyword"},"image":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}},"instance":{"properties":{"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"}}},"provider":{"ignore_above":1024,"type":"keyword"},"machine":{"properties":{"type":{"ignore_above":1024,"type":"keyword"}}},"project":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}},"region":{"ignore_above":1024,"type":"keyword"},"account":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}}}},"container":{"properties":{"image":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}},"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"},"labels":{"type":"object"}}},"@timestamp":{"type":"date"},"apache":{"properties":{"status":{"properties":{"bytes_per_request":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"load":{"properties":{"1":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"15":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"5":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"}}},"bytes_per_sec":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"total_bytes":{"meta":{"unit":"byte","metric_type":"counter"},"type":"long"},"cpu":{"properties":{"system":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"load":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"children_system":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"children_user":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"user":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"}}},"total_accesses":{"meta":{"metric_type":"counter"},"type":"long"},"scoreboard":{"properties":{"total":{"meta":{"metric_type":"gauge"},"type":"long"},"keepalive":{"meta":{"metric_type":"gauge"},"type":"long"},"idle_cleanup":{"meta":{"metric_type":"gauge"},"type":"long"},"waiting_for_connection":{"meta":{"metric_type":"gauge"},"type":"long"},"logging":{"meta":{"metric_type":"gauge"},"type":"long"},"gracefully_finishing":{"meta":{"metric_type":"gauge"},"type":"long"},"open_slot":{"meta":{"metric_type":"gauge"},"type":"long"},"dns_lookup":{"meta":{"metric_type":"gauge"},"type":"long"},"sending_reply":{"meta":{"metric_type":"gauge"},"type":"long"},"closing_connection":{"meta":{"metric_type":"gauge"},"type":"long"},"starting_up":{"meta":{"metric_type":"gauge"},"type":"long"},"reading_request":{"meta":{"metric_type":"gauge"},"type":"long"}}},"workers":{"properties":{"idle":{"meta":{"metric_type":"gauge"},"type":"long"},"busy":{"meta":{"metric_type":"gauge"},"type":"long"}}},"connections":{"properties":{"async":{"properties":{"closing":{"meta":{"metric_type":"gauge"},"type":"long"},"writing":{"meta":{"metric_type":"gauge"},"type":"long"},"keep_alive":{"meta":{"metric_type":"gauge"},"type":"long"}}},"total":{"meta":{"metric_type":"counter"},"type":"long"}}},"requests_per_sec":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"uptime":{"properties":{"server_uptime":{"meta":{"metric_type":"counter"},"type":"long"},"uptime":{"meta":{"metric_type":"counter"},"type":"long"}}}}}}},"ecs":{"properties":{"version":{"ignore_above":1024,"type":"keyword"}}},"data_stream":{"properties":{"namespace":{"type":"constant_keyword"},"type":{"type":"constant_keyword"},"dataset":{"type":"constant_keyword"}}},"service":{"properties":{"address":{"ignore_above":1024,"type":"keyword"},"type":{"ignore_above":1024,"type":"keyword"}}},"host":{"properties":{"hostname":{"ignore_above":1024,"type":"keyword"},"os":{"properties":{"build":{"ignore_above":1024,"type":"keyword"},"kernel":{"ignore_above":1024,"type":"keyword"},"codename":{"ignore_above":1024,"type":"keyword"},"name":{"ignore_above":1024,"type":"keyword","fields":{"text":{"type":"text"}}},"family":{"ignore_above":1024,"type":"keyword"},"version":{"ignore_above":1024,"type":"keyword"},"platform":{"ignore_above":1024,"type":"keyword"}}},"domain":{"ignore_above":1024,"type":"keyword"},"ip":{"type":"ip"},"containerized":{"type":"boolean"},"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"},"type":{"ignore_above":1024,"type":"keyword"},"mac":{"ignore_above":1024,"type":"keyword"},"architecture":{"ignore_above":1024,"type":"keyword"}}},"event":{"properties":{"module":{"type":"constant_keyword","value":"apache"},"dataset":{"type":"constant_keyword","value":"apache.status"}}},"error":{"properties":{"message":{"type":"match_only_text"}}}}}},"composed_of":["metrics-apache.status@settings","metrics-apache.status@custom",".fleet_component_template-1"],"priority":200,"_meta":{"package":{"name":"apache"},"managed_by":"ingest-manager","managed":true},"data_stream":{"hidden":false}}}]}' - headers: - Content-Length: - - "15969" - Content-Type: - - application/json; charset=UTF-8 - X-Elastic-Product: - - Elasticsearch - status: 200 OK - code: 200 - duration: 803.374µs - - id: 2 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: "" - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Authorization: - - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== - User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) - X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 - url: https://127.0.0.1:9200/_component_template/logs-apache.error@settings - method: GET - response: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - transfer_encoding: [] - trailer: {} - content_length: 1306 - uncompressed: false - body: '{"component_templates":[{"name":"logs-apache.error@settings","component_template":{"template":{"settings":{"index":{"lifecycle":{"name":"logs"},"codec":"best_compression","mapping":{"total_fields":{"limit":"10000"}},"query":{"default_field":["cloud.account.id","cloud.availability_zone","cloud.instance.id","cloud.instance.name","cloud.machine.type","cloud.provider","cloud.region","cloud.project.id","cloud.image.id","container.id","container.image.name","container.name","host.architecture","host.domain","host.hostname","host.id","host.mac","host.name","host.os.family","host.os.kernel","host.os.name","host.os.platform","host.os.version","host.type","host.os.build","host.os.codename","input.type","tags","ecs.version","event.category","event.kind","event.timezone","event.type","file.path","http.request.method","http.request.referrer","http.version","log.file.path","log.level","source.address","source.as.organization.name","source.geo.city_name","source.geo.continent_name","source.geo.country_iso_code","source.geo.country_name","source.geo.region_iso_code","source.geo.region_name","tags","url.domain","url.extension","url.query","user.name","user_agent.device.name","user_agent.name","user_agent.original","user_agent.os.name","apache.error.module"]}}}},"_meta":{"package":{"name":"apache"}}}}]}' - headers: - Content-Length: - - "1306" - Content-Type: - - application/json; charset=UTF-8 - X-Elastic-Product: - - Elasticsearch - status: 200 OK - code: 200 - duration: 325.514µs - - id: 3 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: "" - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Authorization: - - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== - User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) - X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 - url: https://127.0.0.1:9200/_component_template/logs-apache.error@custom - method: GET - response: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - transfer_encoding: [] - trailer: {} - content_length: 149 - uncompressed: false - body: '{"component_templates":[{"name":"logs-apache.error@custom","component_template":{"template":{"settings":{}},"_meta":{"package":{"name":"apache"}}}}]}' - headers: - Content-Length: - - "149" - Content-Type: - - application/json; charset=UTF-8 - X-Elastic-Product: - - Elasticsearch - status: 200 OK - code: 200 - duration: 259.991µs - - id: 4 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: "" - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Authorization: - - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== - User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) - X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 - url: https://127.0.0.1:9200/_component_template/.fleet_component_template-1 - method: GET - response: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - transfer_encoding: [] - trailer: {} - content_length: 393 - uncompressed: false - body: '{"component_templates":[{"name":".fleet_component_template-1","component_template":{"template":{"settings":{"index":{"final_pipeline":".fleet_final_pipeline-1"}},"mappings":{"properties":{"event":{"properties":{"agent_id_status":{"ignore_above":1024,"type":"keyword"},"ingested":{"format":"strict_date_time_no_millis||strict_date_optional_time||epoch_millis","type":"date"}}}}}},"_meta":{}}}]}' - headers: - Content-Length: - - "393" - Content-Type: - - application/json; charset=UTF-8 - X-Elastic-Product: - - Elasticsearch - status: 200 OK - code: 200 - duration: 347.417µs - - id: 5 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: "" - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Authorization: - - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== - User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) - X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 - url: https://127.0.0.1:9200/_component_template/logs-apache.access@settings - method: GET - response: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - transfer_encoding: [] - trailer: {} - content_length: 1559 - uncompressed: false - body: '{"component_templates":[{"name":"logs-apache.access@settings","component_template":{"template":{"settings":{"index":{"lifecycle":{"name":"logs"},"codec":"best_compression","mapping":{"total_fields":{"limit":"10000"}},"query":{"default_field":["cloud.account.id","cloud.availability_zone","cloud.instance.id","cloud.instance.name","cloud.machine.type","cloud.provider","cloud.region","cloud.project.id","cloud.image.id","container.id","container.image.name","container.name","host.architecture","host.domain","host.hostname","host.id","host.mac","host.name","host.os.family","host.os.kernel","host.os.name","host.os.platform","host.os.version","host.type","host.os.build","host.os.codename","input.type","destination.domain","ecs.version","event.category","event.kind","event.outcome","file.path","http.request.method","http.request.referrer","http.version","log.file.path","log.level","source.address","source.as.organization.name","source.domain","source.geo.city_name","source.geo.continent_name","source.geo.country_iso_code","source.geo.country_name","source.geo.region_iso_code","source.geo.region_name","tags","tls.cipher","tls.version","tls.version_protocol","url.domain","url.extension","url.query","user.name","user_agent.device.name","user_agent.device.name","user_agent.name","user_agent.name","user_agent.original","user_agent.original","user_agent.os.full","user_agent.os.name","user_agent.os.name","user_agent.os.version","user_agent.version","apache.access.ssl.protocol","apache.access.ssl.cipher"]}}}},"_meta":{"package":{"name":"apache"}}}}]}' - headers: - Content-Length: - - "1559" - Content-Type: - - application/json; charset=UTF-8 - X-Elastic-Product: - - Elasticsearch - status: 200 OK - code: 200 - duration: 257.04µs - - id: 6 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: "" - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Authorization: - - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== - User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) - X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 - url: https://127.0.0.1:9200/_component_template/logs-apache.access@custom - method: GET - response: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - transfer_encoding: [] - trailer: {} - content_length: 150 - uncompressed: false - body: '{"component_templates":[{"name":"logs-apache.access@custom","component_template":{"template":{"settings":{}},"_meta":{"package":{"name":"apache"}}}}]}' - headers: - Content-Length: - - "150" - Content-Type: - - application/json; charset=UTF-8 - X-Elastic-Product: - - Elasticsearch - status: 200 OK - code: 200 - duration: 241.005µs - - id: 7 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: "" - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Authorization: - - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== - User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) - X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 - url: https://127.0.0.1:9200/_component_template/metrics-apache.status@settings - method: GET - response: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - transfer_encoding: [] - trailer: {} - content_length: 789 - uncompressed: false - body: '{"component_templates":[{"name":"metrics-apache.status@settings","component_template":{"template":{"settings":{"index":{"lifecycle":{"name":"metrics"},"codec":"best_compression","mapping":{"total_fields":{"limit":"10000"}},"query":{"default_field":["cloud.account.id","cloud.availability_zone","cloud.instance.id","cloud.instance.name","cloud.machine.type","cloud.provider","cloud.region","cloud.project.id","cloud.image.id","container.id","container.image.name","container.name","host.architecture","host.domain","host.hostname","host.id","host.mac","host.name","host.os.family","host.os.kernel","host.os.name","host.os.platform","host.os.version","host.type","host.os.build","host.os.codename","ecs.version","service.address","service.type"]}}}},"_meta":{"package":{"name":"apache"}}}}]}' - headers: - Content-Length: - - "789" - Content-Type: - - application/json; charset=UTF-8 - X-Elastic-Product: - - Elasticsearch - status: 200 OK - code: 200 - duration: 264.377µs - - id: 8 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: "" - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Authorization: - - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== - User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) - X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 - url: https://127.0.0.1:9200/_component_template/metrics-apache.status@custom - method: GET - response: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - transfer_encoding: [] - trailer: {} - content_length: 153 - uncompressed: false - body: '{"component_templates":[{"name":"metrics-apache.status@custom","component_template":{"template":{"settings":{}},"_meta":{"package":{"name":"apache"}}}}]}' - headers: - Content-Length: - - "153" - Content-Type: - - application/json; charset=UTF-8 - X-Elastic-Product: - - Elasticsearch - status: 200 OK - code: 200 - duration: 315.098µs - - id: 9 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: "" - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Authorization: - - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== - User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) - X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 - url: https://127.0.0.1:9200/_ilm/policy/logs - method: GET - response: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - transfer_encoding: [] - trailer: {} - content_length: 1316 - uncompressed: false - body: '{"logs":{"version":1,"modified_date":"2023-11-27T13:55:33.162Z","policy":{"phases":{"hot":{"min_age":"0ms","actions":{"rollover":{"max_primary_shard_size":"50gb","max_age":"30d"}}}},"_meta":{"managed":true,"description":"default policy for the logs index template installed by x-pack"}},"in_use_by":{"indices":[".ds-logs-elastic_agent.metricbeat-default-2023.11.27-000001",".ds-logs-elastic_agent.fleet_server-default-2023.11.27-000001",".ds-logs-elastic_agent.filebeat-default-2023.11.27-000001",".ds-logs-elastic_agent-default-2023.11.27-000001"],"data_streams":["logs-elastic_agent-default","logs-elastic_agent.metricbeat-default","logs-elastic_agent.filebeat-default","logs-elastic_agent.fleet_server-default"],"composable_templates":["logs-apache.access","logs-elastic_agent.cloudbeat","logs-elastic_agent.apm_server","logs-elastic_agent.cloud_defend","logs-system.security","logs-system.auth","logs-elastic_agent.metricbeat","logs-elastic_agent.filebeat","logs-elastic_agent.packetbeat","logs-elastic_agent.filebeat_input","logs-elastic_agent.endpoint_security","logs-elastic_agent.fleet_server","logs-apache.error","logs-system.system","logs-system.application","logs-elastic_agent.osquerybeat","logs-elastic_agent.heartbeat","logs-system.syslog","logs-elastic_agent.auditbeat","logs","logs-elastic_agent"]}}}' - headers: - Content-Length: - - "1316" - Content-Type: - - application/json; charset=UTF-8 - X-Elastic-Product: - - Elasticsearch - status: 200 OK - code: 200 - duration: 1.292625ms - - id: 10 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: "" - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Authorization: - - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== - User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) - X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 - url: https://127.0.0.1:9200/_ilm/policy/metrics - method: GET - response: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - transfer_encoding: [] - trailer: {} - content_length: 2552 - uncompressed: false - body: '{"metrics":{"version":1,"modified_date":"2023-11-27T13:55:33.210Z","policy":{"phases":{"hot":{"min_age":"0ms","actions":{"rollover":{"max_primary_shard_size":"50gb","max_age":"30d"}}}},"_meta":{"managed":true,"description":"default policy for the metrics index template installed by x-pack"}},"in_use_by":{"indices":[".ds-metrics-system.process.summary-default-2023.11.27-000001",".ds-metrics-system.fsstat-default-2023.11.27-000001",".ds-metrics-system.uptime-default-2023.11.27-000001",".ds-metrics-system.network-default-2023.11.27-000001",".ds-metrics-system.filesystem-default-2023.11.27-000001",".ds-metrics-elastic_agent.elastic_agent-default-2023.11.27-000001",".ds-metrics-system.socket_summary-default-2023.11.27-000001",".ds-metrics-system.diskio-default-2023.11.27-000001",".ds-metrics-elastic_agent.filebeat-default-2023.11.27-000001",".ds-metrics-system.process-default-2023.11.27-000001",".ds-metrics-system.cpu-default-2023.11.27-000001",".ds-metrics-elastic_agent.fleet_server-default-2023.11.27-000001",".ds-metrics-elastic_agent.metricbeat-default-2023.11.27-000001",".ds-metrics-system.memory-default-2023.11.27-000001",".ds-metrics-system.load-default-2023.11.27-000001"],"data_streams":["metrics-system.filesystem-default","metrics-system.cpu-default","metrics-system.process.summary-default","metrics-system.memory-default","metrics-elastic_agent.fleet_server-default","metrics-system.uptime-default","metrics-elastic_agent.elastic_agent-default","metrics-elastic_agent.metricbeat-default","metrics-system.fsstat-default","metrics-system.process-default","metrics-elastic_agent.filebeat-default","metrics-system.network-default","metrics-system.diskio-default","metrics-system.load-default","metrics-system.socket_summary-default"],"composable_templates":["metrics-system.process","metrics-elastic_agent.packetbeat","metrics-system.fsstat","metrics-elastic_agent.osquerybeat","metrics-elastic_agent.endpoint_security","metrics-elastic_agent.apm_server","metrics-system.memory","metrics-system.socket_summary","metrics-apache.status","metrics-elastic_agent.elastic_agent","metrics-elastic_agent.fleet_server","metrics-system.load","metrics-system.core","metrics-elastic_agent.filebeat","metrics-elastic_agent.filebeat_input","metrics-system.uptime","metrics-system.process.summary","metrics-system.cpu","metrics-elastic_agent.heartbeat","metrics-system.diskio","metrics-elastic_agent.cloudbeat","metrics-elastic_agent.metricbeat","metrics-elastic_agent.auditbeat","metrics-system.network","metrics-system.filesystem","metrics"]}}}' - headers: - Content-Length: - - "2552" - Content-Type: - - application/json; charset=UTF-8 - X-Elastic-Product: - - Elasticsearch - status: 200 OK - code: 200 - duration: 1.250469ms - - id: 11 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: "" - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Authorization: - - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== - User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) - X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 - url: https://127.0.0.1:9200/_ingest/pipeline/logs-apache.error-1.3.4 - method: GET - response: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - transfer_encoding: [] - trailer: {} - content_length: 3693 - uncompressed: false - body: '{"logs-apache.error-1.3.4":{"description":"Pipeline for parsing apache error logs","processors":[{"pipeline":{"if":"ctx.message.startsWith(''{'')","name":"logs-apache.error-1.3.4-third-party"}},{"set":{"field":"event.ingested","value":"{{_ingest.timestamp}}"}},{"set":{"field":"ecs.version","value":"1.12.0"}},{"rename":{"field":"message","target_field":"event.original"}},{"grok":{"field":"event.original","patterns":["\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{LOGLEVEL:log.level}\\]( \\[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\\])? %{GREEDYDATA:message}","\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{DATA:apache.error.module}:%{APACHE_LOGLEVEL:log.level}\\] \\[pid %{NUMBER:process.pid:long}(:tid %{NUMBER:process.thread.id:long})?\\]( \\[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\\])? %{GREEDYDATA:message}"],"pattern_definitions":{"APACHE_LOGLEVEL":"%{LOGLEVEL}[0-9]*","APACHE_TIME":"%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"},"ignore_missing":true}},{"grok":{"field":"message","patterns":["File does not exist: %{URIPATH:file.path}, referer: %{URI:http.request.referrer}","File does not exist: %{URIPATH:file.path}"],"ignore_missing":true,"ignore_failure":true}},{"date":{"if":"ctx.event.timezone == null","field":"apache.error.timestamp","target_field":"@timestamp","formats":["EEE MMM dd H:m:s yyyy","EEE MMM dd H:m:s.SSSSSS yyyy"],"on_failure":[{"append":{"field":"error.message","value":"{{ _ingest.on_failure_message }}"}}]}},{"date":{"if":"ctx.event.timezone != null","field":"apache.error.timestamp","target_field":"@timestamp","formats":["EEE MMM dd H:m:s yyyy","EEE MMM dd H:m:s.SSSSSS yyyy"],"timezone":"{{ event.timezone }}","on_failure":[{"append":{"field":"error.message","value":"{{ _ingest.on_failure_message }}"}}]}},{"remove":{"field":"apache.error.timestamp","ignore_failure":true}},{"set":{"field":"event.kind","value":"event"}},{"set":{"field":"event.category","value":"web"}},{"script":{"if":"ctx?.log?.level != null","lang":"painless","source":"def err_levels = [\"emerg\", \"alert\", \"crit\", \"error\", \"warn\"]; if (err_levels.contains(ctx.log.level)) {\n ctx.event.type = \"error\";\n} else {\n ctx.event.type = \"info\";\n}"}},{"grok":{"field":"source.address","ignore_missing":true,"patterns":["^(%{IP:source.ip}|%{HOSTNAME:source.domain})$"]}},{"geoip":{"field":"source.ip","target_field":"source.geo","ignore_missing":true}},{"geoip":{"database_file":"GeoLite2-ASN.mmdb","field":"source.ip","target_field":"source.as","properties":["asn","organization_name"],"ignore_missing":true}},{"rename":{"field":"source.as.asn","target_field":"source.as.number","ignore_missing":true}},{"rename":{"field":"source.as.organization_name","target_field":"source.as.organization.name","ignore_missing":true}},{"convert":{"field":"source.port","type":"long","ignore_missing":true}},{"script":{"lang":"painless","description":"This script processor iterates over the whole document to remove fields with null values.","source":"void handleMap(Map map) {\n for (def x : map.values()) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n map.values().removeIf(v -> v == null);\n}\nvoid handleList(List list) {\n for (def x : list) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n}\nhandleMap(ctx);\n"}},{"remove":{"field":"event.original","if":"ctx?.tags == null || !(ctx.tags.contains(''preserve_original_event''))","ignore_failure":true,"ignore_missing":true}}],"on_failure":[{"set":{"field":"error.message","value":"{{ _ingest.on_failure_message }}"}}]}}' - headers: - Content-Length: - - "3693" - Content-Type: - - application/json; charset=UTF-8 - X-Elastic-Product: - - Elasticsearch - status: 200 OK - code: 200 - duration: 754.413µs - - id: 12 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: "" - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Authorization: - - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== - User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) - X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 - url: https://127.0.0.1:9200/_ingest/pipeline/logs-apache.access-1.3.4 - method: GET - response: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - transfer_encoding: [] - trailer: {} - content_length: 4500 - uncompressed: false - body: '{"logs-apache.access-1.3.4":{"description":"Pipeline for parsing Apache HTTP Server access logs. Requires the geoip and user_agent plugins.","processors":[{"pipeline":{"if":"ctx.message.startsWith(''{'')","name":"logs-apache.access-1.3.4-third-party"}},{"set":{"field":"event.ingested","value":"{{_ingest.timestamp}}"}},{"set":{"field":"ecs.version","value":"1.12.0"}},{"rename":{"field":"message","target_field":"event.original"}},{"grok":{"field":"event.original","patterns":["%{IPORHOST:destination.domain} %{IPORHOST:source.ip} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?\" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( \"%{DATA:http.request.referrer}\")?( \"%{DATA:user_agent.original}\")?","%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?\" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( \"%{DATA:http.request.referrer}\")?( \"%{DATA:user_agent.original}\")?","%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"-\" %{NUMBER:http.response.status_code:long} -","\\[%{HTTPDATE:apache.access.time}\\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol} %{DATA:apache.access.ssl.cipher} \"%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}\" (-|%{NUMBER:http.response.body.bytes:long})"],"ignore_missing":true}},{"uri_parts":{"field":"_tmp.url_orig","ignore_failure":true}},{"remove":{"field":["_tmp"],"ignore_missing":true}},{"set":{"field":"url.domain","value":"{{destination.domain}}","if":"ctx.url?.domain == null && ctx.destination?.domain != null"}},{"set":{"field":"event.kind","value":"event"}},{"set":{"field":"event.category","value":"web"}},{"set":{"field":"event.outcome","value":"success","if":"ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400"}},{"set":{"field":"event.outcome","value":"failure","if":"ctx?.http?.response?.status_code != null && ctx.http.response.status_code > 399"}},{"grok":{"field":"source.address","ignore_missing":true,"patterns":["^(%{IP:source.ip}|%{HOSTNAME:source.domain})$"]}},{"remove":{"field":"event.created","ignore_missing":true,"ignore_failure":true}},{"rename":{"field":"@timestamp","target_field":"event.created"}},{"date":{"field":"apache.access.time","target_field":"@timestamp","formats":["dd/MMM/yyyy:H:m:s Z"],"ignore_failure":true}},{"remove":{"field":"apache.access.time","ignore_failure":true}},{"user_agent":{"field":"user_agent.original","ignore_failure":true}},{"geoip":{"field":"source.ip","target_field":"source.geo","ignore_missing":true}},{"geoip":{"database_file":"GeoLite2-ASN.mmdb","field":"source.ip","target_field":"source.as","properties":["asn","organization_name"],"ignore_missing":true}},{"rename":{"field":"source.as.asn","target_field":"source.as.number","ignore_missing":true}},{"rename":{"field":"source.as.organization_name","target_field":"source.as.organization.name","ignore_missing":true}},{"set":{"field":"tls.cipher","value":"{{apache.access.ssl.cipher}}","if":"ctx?.apache?.access?.ssl?.cipher != null"}},{"script":{"lang":"painless","if":"ctx?.apache?.access?.ssl?.protocol != null","source":"def parts = ctx.apache.access.ssl.protocol.toLowerCase().splitOnToken(\"v\"); if (parts.length != 2) {\n return;\n} if (parts[1].contains(\".\")) {\n ctx.tls.version = parts[1];\n} else {\n ctx.tls.version = parts[1] + \".0\";\n} ctx.tls.version_protocol = parts[0];"}},{"script":{"lang":"painless","description":"This script processor iterates over the whole document to remove fields with null values.","source":"void handleMap(Map map) {\n for (def x : map.values()) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n map.values().removeIf(v -> v == null);\n}\nvoid handleList(List list) {\n for (def x : list) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n}\nhandleMap(ctx);\n"}},{"remove":{"field":"event.original","if":"ctx?.tags == null || !(ctx.tags.contains(''preserve_original_event''))","ignore_failure":true,"ignore_missing":true}}],"on_failure":[{"set":{"field":"error.message","value":"{{ _ingest.on_failure_message }}"}}]}}' - headers: - Content-Length: - - "4500" - Content-Type: - - application/json; charset=UTF-8 - X-Elastic-Product: - - Elasticsearch - status: 200 OK - code: 200 - duration: 521.734µs - - id: 13 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: "" - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Authorization: - - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== - User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) - X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 - url: https://127.0.0.1:9200/_ingest/pipeline/.fleet_final_pipeline-1 - method: GET - response: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - transfer_encoding: [] - trailer: {} - content_length: 2865 - uncompressed: false - body: '{".fleet_final_pipeline-1":{"version":1,"description":"Final pipeline for processing all incoming Fleet Agent documents.\n","processors":[{"set":{"description":"Add time when event was ingested.","field":"event.ingested","copy_from":"_ingest.timestamp"}},{"script":{"description":"Remove sub-seconds from event.ingested to improve storage efficiency.","tag":"truncate-subseconds-event-ingested","source":"ctx.event.ingested = ctx.event.ingested.withNano(0).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);","ignore_failure":true}},{"remove":{"description":"Remove any pre-existing untrusted values.","field":["event.agent_id_status","_security"],"ignore_missing":true}},{"set_security_user":{"field":"_security","properties":["authentication_type","username","realm","api_key"]}},{"script":{"description":"Add event.agent_id_status based on the API key metadata and the agent.id contained in the event.\n","tag":"agent-id-status","source":"boolean is_user_trusted(def ctx, def users) {\n if (ctx?._security?.username == null) {\n return false;\n }\n\n def user = null;\n for (def item : users) {\n if (item?.username == ctx._security.username) {\n user = item;\n break;\n }\n }\n\n if (user == null || user?.realm == null || ctx?._security?.realm?.name == null) {\n return false;\n }\n\n if (ctx._security.realm.name != user.realm) {\n return false;\n }\n\n return true;\n}\n\nString verified(def ctx, def params) {\n // No agent.id field to validate.\n if (ctx?.agent?.id == null) {\n return \"missing\";\n }\n\n // Check auth metadata from API key.\n if (ctx?._security?.authentication_type == null\n // Agents only use API keys.\n || ctx._security.authentication_type != ''API_KEY''\n // Verify the API key owner before trusting any metadata it contains.\n || !is_user_trusted(ctx, params.trusted_users)\n // Verify the API key has metadata indicating the assigned agent ID.\n || ctx?._security?.api_key?.metadata?.agent_id == null) {\n return \"auth_metadata_missing\";\n }\n\n // The API key can only be used represent the agent.id it was issued to.\n if (ctx._security.api_key.metadata.agent_id != ctx.agent.id) {\n // Potential masquerade attempt.\n return \"mismatch\";\n }\n\n return \"verified\";\n}\n\nif (ctx?.event == null) {\n ctx.event = [:];\n}\n\nctx.event.agent_id_status = verified(ctx, params);","params":{"trusted_users":[{"username":"elastic/fleet-server","realm":"_service_account"},{"username":"cloud-internal-agent-server","realm":"found"},{"username":"elastic","realm":"reserved"}]}}},{"remove":{"field":"_security","ignore_missing":true}}],"on_failure":[{"remove":{"field":"_security","ignore_missing":true,"ignore_failure":true}},{"append":{"field":"error.message","value":["failed in Fleet agent final_pipeline: {{ _ingest.on_failure_message }}"]}}]}}' - headers: - Content-Length: - - "2865" - Content-Type: - - application/json; charset=UTF-8 - X-Elastic-Product: - - Elasticsearch - status: 200 OK - code: 200 - duration: 394.152µs - - id: 14 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: "" - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Authorization: - - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== - User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) - X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 - url: https://127.0.0.1:9200/_ingest/pipeline/logs-apache.error-1.3.4-third-party - method: GET - response: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - transfer_encoding: [] - trailer: {} - content_length: 1047 - uncompressed: false - body: '{"logs-apache.error-1.3.4-third-party":{"description":"Pipeline for parsing Apache HTTP Server logs from third party api","processors":[{"json":{"field":"message","target_field":"json"}},{"drop":{"if":"ctx.json?.result == null"}},{"fingerprint":{"fields":["json.result._cd","json.result._indextime","json.result._raw","json.result._time","json.result.host","json.result.source"],"target_field":"_id","ignore_missing":true}},{"set":{"copy_from":"json.result._raw","field":"message","ignore_empty_value":true}},{"set":{"copy_from":"json.result.host","field":"host.name","ignore_empty_value":true}},{"set":{"copy_from":"json.result.source","field":"file.path","ignore_empty_value":true}},{"remove":{"field":["json"],"ignore_missing":true}}],"on_failure":[{"append":{"field":"error.message","value":"error in third-party pipeline: error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}} with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}} {{ _ingest.on_failure_message }}"}}]}}' - headers: - Content-Length: - - "1047" - Content-Type: - - application/json; charset=UTF-8 - X-Elastic-Product: - - Elasticsearch - status: 200 OK - code: 200 - duration: 314.524µs - - id: 15 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: "" - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Authorization: - - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== - User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) - X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 - url: https://127.0.0.1:9200/_ingest/pipeline/logs-apache.access-1.3.4-third-party - method: GET - response: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - transfer_encoding: [] - trailer: {} - content_length: 1048 - uncompressed: false - body: '{"logs-apache.access-1.3.4-third-party":{"description":"Pipeline for parsing Apache HTTP Server logs from third party api","processors":[{"json":{"field":"message","target_field":"json"}},{"drop":{"if":"ctx.json?.result == null"}},{"fingerprint":{"fields":["json.result._cd","json.result._indextime","json.result._raw","json.result._time","json.result.host","json.result.source"],"target_field":"_id","ignore_missing":true}},{"set":{"copy_from":"json.result._raw","field":"message","ignore_empty_value":true}},{"set":{"copy_from":"json.result.host","field":"host.name","ignore_empty_value":true}},{"set":{"copy_from":"json.result.source","field":"file.path","ignore_empty_value":true}},{"remove":{"field":["json"],"ignore_missing":true}}],"on_failure":[{"append":{"field":"error.message","value":"error in third-party pipeline: error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}} with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}} {{ _ingest.on_failure_message }}"}}]}}' - headers: - Content-Length: - - "1048" - Content-Type: - - application/json; charset=UTF-8 - X-Elastic-Product: - - Elasticsearch - status: 200 OK - code: 200 - duration: 364.107µs - - id: 16 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: "" - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Authorization: - - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== - User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) - X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 - url: https://127.0.0.1:9200/_ml/trained_models/apache_*?decompress_definition=false&include=definition%2Cfeature_importance_baseline%2Chyperparameters%2Ctotal_feature_importance - method: GET - response: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - transfer_encoding: [] - trailer: {} - content_length: 38 - uncompressed: false - body: '{"count":0,"trained_model_configs":[]}' - headers: - Content-Length: - - "38" - Content-Type: - - application/json; charset=UTF-8 - X-Elastic-Product: - - Elasticsearch - status: 200 OK - code: 200 - duration: 1.04779ms + - id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: "" + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Authorization: + - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== + User-Agent: + - go-elasticsearch/7.17.10 (darwin arm64; Go 1.24.2) + X-Elastic-Client-Meta: + - es=7.17.10,go=1.24.2,t=7.17.10,hc=1.24.2 + url: https://127.0.0.1:9200/ + method: GET + response: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + transfer_encoding: [] + trailer: {} + content_length: 545 + uncompressed: false + body: | + { + "name" : "2bb69ec8be53", + "cluster_name" : "elasticsearch", + "cluster_uuid" : "8aSsUHKoSw23B8HgcUiiXg", + "version" : { + "number" : "7.16.2", + "build_flavor" : "default", + "build_type" : "docker", + "build_hash" : "2b937c44140b6559905130a8650c64dbd0879cfb", + "build_date" : "2021-12-18T19:42:46.604893745Z", + "build_snapshot" : false, + "lucene_version" : "8.10.1", + "minimum_wire_compatibility_version" : "6.8.0", + "minimum_index_compatibility_version" : "6.0.0-beta1" + }, + "tagline" : "You Know, for Search" + } + headers: + Content-Length: + - "545" + Content-Type: + - application/json; charset=UTF-8 + X-Elastic-Product: + - Elasticsearch + status: 200 OK + code: 200 + duration: 3.737541ms + - id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: "" + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Authorization: + - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== + User-Agent: + - go-elasticsearch/7.17.10 (darwin arm64; Go 1.24.2) + X-Elastic-Client-Meta: + - es=7.17.10,go=1.24.2,t=7.17.10,hc=1.24.2 + url: https://127.0.0.1:9200/_index_template/*-apache.* + method: GET + response: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + transfer_encoding: [] + trailer: {} + content_length: 15837 + uncompressed: false + body: '{"index_templates":[{"name":"logs-apache.error","index_template":{"index_patterns":["logs-apache.error-*"],"template":{"settings":{"index":{"default_pipeline":"logs-apache.error-1.3.5"}},"mappings":{"_meta":{"package":{"name":"apache"},"managed_by":"ingest-manager","managed":true},"dynamic_templates":[{"strings_as_keyword":{"mapping":{"ignore_above":1024,"type":"keyword"},"match_mapping_type":"string"}}],"date_detection":false,"properties":{"container":{"properties":{"image":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}},"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"},"labels":{"type":"object"}}},"process":{"properties":{"pid":{"type":"long"},"thread":{"properties":{"id":{"type":"long"}}}}},"log":{"properties":{"file":{"properties":{"path":{"ignore_above":1024,"type":"keyword"}}},"offset":{"type":"long"},"level":{"ignore_above":1024,"type":"keyword"}}},"source":{"properties":{"geo":{"properties":{"continent_name":{"ignore_above":1024,"type":"keyword"},"region_iso_code":{"ignore_above":1024,"type":"keyword"},"city_name":{"ignore_above":1024,"type":"keyword"},"country_iso_code":{"ignore_above":1024,"type":"keyword"},"country_name":{"ignore_above":1024,"type":"keyword"},"location":{"type":"geo_point"},"region_name":{"ignore_above":1024,"type":"keyword"}}},"as":{"properties":{"number":{"type":"long"},"organization":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}}}},"address":{"ignore_above":1024,"type":"keyword"},"port":{"type":"long"},"ip":{"type":"ip"}}},"error":{"properties":{"message":{"type":"match_only_text"}}},"message":{"type":"match_only_text"},"url":{"properties":{"path":{"type":"wildcard"},"extension":{"ignore_above":1024,"type":"keyword"},"original":{"type":"wildcard"},"domain":{"ignore_above":1024,"type":"keyword"},"query":{"ignore_above":1024,"type":"keyword"}}},"tags":{"ignore_above":1024,"type":"keyword"},"cloud":{"properties":{"availability_zone":{"ignore_above":1024,"type":"keyword"},"image":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}},"instance":{"properties":{"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"}}},"provider":{"ignore_above":1024,"type":"keyword"},"machine":{"properties":{"type":{"ignore_above":1024,"type":"keyword"}}},"project":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}},"region":{"ignore_above":1024,"type":"keyword"},"account":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}}}},"input":{"properties":{"type":{"ignore_above":1024,"type":"keyword"}}},"@timestamp":{"type":"date"},"file":{"properties":{"path":{"ignore_above":1024,"type":"keyword"}}},"apache":{"properties":{"error":{"properties":{"module":{"ignore_above":1024,"type":"keyword"}}}}},"ecs":{"properties":{"version":{"ignore_above":1024,"type":"keyword"}}},"data_stream":{"properties":{"namespace":{"type":"constant_keyword"},"type":{"type":"constant_keyword"},"dataset":{"type":"constant_keyword"}}},"host":{"properties":{"hostname":{"ignore_above":1024,"type":"keyword"},"os":{"properties":{"build":{"ignore_above":1024,"type":"keyword"},"kernel":{"ignore_above":1024,"type":"keyword"},"codename":{"ignore_above":1024,"type":"keyword"},"name":{"ignore_above":1024,"type":"keyword","fields":{"text":{"type":"text"}}},"family":{"ignore_above":1024,"type":"keyword"},"version":{"ignore_above":1024,"type":"keyword"},"platform":{"ignore_above":1024,"type":"keyword"}}},"domain":{"ignore_above":1024,"type":"keyword"},"ip":{"type":"ip"},"containerized":{"type":"boolean"},"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"},"type":{"ignore_above":1024,"type":"keyword"},"mac":{"ignore_above":1024,"type":"keyword"},"architecture":{"ignore_above":1024,"type":"keyword"}}},"http":{"properties":{"request":{"properties":{"referrer":{"ignore_above":1024,"type":"keyword"},"method":{"ignore_above":1024,"type":"keyword"}}},"response":{"properties":{"status_code":{"type":"long"},"body":{"properties":{"bytes":{"type":"long"}}}}},"version":{"ignore_above":1024,"type":"keyword"}}},"event":{"properties":{"kind":{"ignore_above":1024,"type":"keyword"},"timezone":{"ignore_above":1024,"type":"keyword"},"module":{"type":"constant_keyword","value":"apache"},"category":{"ignore_above":1024,"type":"keyword"},"type":{"ignore_above":1024,"type":"keyword"},"dataset":{"type":"constant_keyword","value":"apache.error"}}},"user":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}},"user_agent":{"properties":{"original":{"ignore_above":1024,"type":"keyword"},"os":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}},"name":{"ignore_above":1024,"type":"keyword"},"device":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}}}}}}},"composed_of":["logs-apache.error@settings","logs-apache.error@custom",".fleet_component_template-1"],"priority":200,"_meta":{"package":{"name":"apache"},"managed_by":"ingest-manager","managed":true},"data_stream":{"hidden":false}}},{"name":"logs-apache.access","index_template":{"index_patterns":["logs-apache.access-*"],"template":{"settings":{"index":{"default_pipeline":"logs-apache.access-1.3.5"}},"mappings":{"_meta":{"package":{"name":"apache"},"managed_by":"ingest-manager","managed":true},"dynamic_templates":[{"strings_as_keyword":{"mapping":{"ignore_above":1024,"type":"keyword"},"match_mapping_type":"string"}}],"date_detection":false,"properties":{"container":{"properties":{"image":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}},"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"},"labels":{"type":"object"}}},"process":{"properties":{"pid":{"type":"long"},"thread":{"properties":{"id":{"type":"long"}}}}},"log":{"properties":{"file":{"properties":{"path":{"ignore_above":1024,"type":"keyword"}}},"offset":{"type":"long"},"level":{"ignore_above":1024,"type":"keyword"}}},"destination":{"properties":{"domain":{"ignore_above":1024,"type":"keyword"}}},"source":{"properties":{"geo":{"properties":{"continent_name":{"ignore_above":1024,"type":"keyword"},"region_iso_code":{"ignore_above":1024,"type":"keyword"},"city_name":{"ignore_above":1024,"type":"keyword"},"country_iso_code":{"ignore_above":1024,"type":"keyword"},"country_name":{"ignore_above":1024,"type":"keyword"},"location":{"type":"geo_point"},"region_name":{"ignore_above":1024,"type":"keyword"}}},"as":{"properties":{"number":{"type":"long"},"organization":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}}}},"address":{"ignore_above":1024,"type":"keyword"},"domain":{"ignore_above":1024,"type":"keyword"},"ip":{"type":"ip"}}},"error":{"properties":{"message":{"type":"match_only_text"}}},"message":{"type":"match_only_text"},"url":{"properties":{"path":{"type":"wildcard"},"extension":{"ignore_above":1024,"type":"keyword"},"original":{"type":"wildcard"},"domain":{"ignore_above":1024,"type":"keyword"},"query":{"ignore_above":1024,"type":"keyword"}}},"tags":{"ignore_above":1024,"type":"keyword"},"cloud":{"properties":{"availability_zone":{"ignore_above":1024,"type":"keyword"},"image":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}},"instance":{"properties":{"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"}}},"provider":{"ignore_above":1024,"type":"keyword"},"machine":{"properties":{"type":{"ignore_above":1024,"type":"keyword"}}},"project":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}},"region":{"ignore_above":1024,"type":"keyword"},"account":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}}}},"input":{"properties":{"type":{"ignore_above":1024,"type":"keyword"}}},"@timestamp":{"type":"date"},"file":{"properties":{"path":{"ignore_above":1024,"type":"keyword"}}},"apache":{"properties":{"access":{"properties":{"ssl":{"properties":{"cipher":{"ignore_above":1024,"type":"keyword"},"protocol":{"ignore_above":1024,"type":"keyword"}}}}}}},"ecs":{"properties":{"version":{"ignore_above":1024,"type":"keyword"}}},"data_stream":{"properties":{"namespace":{"type":"constant_keyword"},"type":{"type":"constant_keyword"},"dataset":{"type":"constant_keyword"}}},"host":{"properties":{"hostname":{"ignore_above":1024,"type":"keyword"},"os":{"properties":{"build":{"ignore_above":1024,"type":"keyword"},"kernel":{"ignore_above":1024,"type":"keyword"},"codename":{"ignore_above":1024,"type":"keyword"},"name":{"ignore_above":1024,"type":"keyword","fields":{"text":{"type":"text"}}},"family":{"ignore_above":1024,"type":"keyword"},"version":{"ignore_above":1024,"type":"keyword"},"platform":{"ignore_above":1024,"type":"keyword"}}},"domain":{"ignore_above":1024,"type":"keyword"},"ip":{"type":"ip"},"containerized":{"type":"boolean"},"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"},"type":{"ignore_above":1024,"type":"keyword"},"mac":{"ignore_above":1024,"type":"keyword"},"architecture":{"ignore_above":1024,"type":"keyword"}}},"http":{"properties":{"request":{"properties":{"referrer":{"ignore_above":1024,"type":"keyword"},"method":{"ignore_above":1024,"type":"keyword"}}},"response":{"properties":{"status_code":{"type":"long"},"body":{"properties":{"bytes":{"type":"long"}}}}},"version":{"ignore_above":1024,"type":"keyword"}}},"tls":{"properties":{"cipher":{"ignore_above":1024,"type":"keyword"},"version":{"ignore_above":1024,"type":"keyword"},"version_protocol":{"ignore_above":1024,"type":"keyword"}}},"event":{"properties":{"created":{"type":"date"},"kind":{"ignore_above":1024,"type":"keyword"},"module":{"type":"constant_keyword","value":"apache"},"category":{"ignore_above":1024,"type":"keyword"},"dataset":{"type":"constant_keyword","value":"apache.access"},"outcome":{"ignore_above":1024,"type":"keyword"}}},"user":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}},"user_agent":{"properties":{"original":{"ignore_above":1024,"type":"keyword"},"os":{"properties":{"name":{"ignore_above":1024,"type":"keyword"},"version":{"ignore_above":1024,"type":"keyword"},"full":{"ignore_above":1024,"type":"keyword"}}},"name":{"ignore_above":1024,"type":"keyword"},"device":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}},"version":{"ignore_above":1024,"type":"keyword"}}}}}},"composed_of":["logs-apache.access@settings","logs-apache.access@custom",".fleet_component_template-1"],"priority":200,"_meta":{"package":{"name":"apache"},"managed_by":"ingest-manager","managed":true},"data_stream":{"hidden":false}}},{"name":"metrics-apache.status","index_template":{"index_patterns":["metrics-apache.status-*"],"template":{"settings":{},"mappings":{"_meta":{"package":{"name":"apache"},"managed_by":"ingest-manager","managed":true},"dynamic_templates":[{"strings_as_keyword":{"mapping":{"ignore_above":1024,"type":"keyword"},"match_mapping_type":"string"}}],"date_detection":false,"properties":{"cloud":{"properties":{"availability_zone":{"ignore_above":1024,"type":"keyword"},"image":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}},"instance":{"properties":{"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"}}},"provider":{"ignore_above":1024,"type":"keyword"},"machine":{"properties":{"type":{"ignore_above":1024,"type":"keyword"}}},"project":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}},"region":{"ignore_above":1024,"type":"keyword"},"account":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}}}},"container":{"properties":{"image":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}},"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"},"labels":{"type":"object"}}},"@timestamp":{"type":"date"},"apache":{"properties":{"status":{"properties":{"bytes_per_request":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"load":{"properties":{"1":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"15":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"5":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"}}},"bytes_per_sec":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"total_bytes":{"meta":{"unit":"byte","metric_type":"counter"},"type":"long"},"cpu":{"properties":{"system":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"load":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"children_system":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"children_user":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"user":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"}}},"total_accesses":{"meta":{"metric_type":"counter"},"type":"long"},"scoreboard":{"properties":{"total":{"meta":{"metric_type":"gauge"},"type":"long"},"keepalive":{"meta":{"metric_type":"gauge"},"type":"long"},"idle_cleanup":{"meta":{"metric_type":"gauge"},"type":"long"},"waiting_for_connection":{"meta":{"metric_type":"gauge"},"type":"long"},"logging":{"meta":{"metric_type":"gauge"},"type":"long"},"gracefully_finishing":{"meta":{"metric_type":"gauge"},"type":"long"},"open_slot":{"meta":{"metric_type":"gauge"},"type":"long"},"dns_lookup":{"meta":{"metric_type":"gauge"},"type":"long"},"sending_reply":{"meta":{"metric_type":"gauge"},"type":"long"},"closing_connection":{"meta":{"metric_type":"gauge"},"type":"long"},"starting_up":{"meta":{"metric_type":"gauge"},"type":"long"},"reading_request":{"meta":{"metric_type":"gauge"},"type":"long"}}},"workers":{"properties":{"idle":{"meta":{"metric_type":"gauge"},"type":"long"},"busy":{"meta":{"metric_type":"gauge"},"type":"long"}}},"connections":{"properties":{"async":{"properties":{"closing":{"meta":{"metric_type":"gauge"},"type":"long"},"writing":{"meta":{"metric_type":"gauge"},"type":"long"},"keep_alive":{"meta":{"metric_type":"gauge"},"type":"long"}}},"total":{"meta":{"metric_type":"counter"},"type":"long"}}},"requests_per_sec":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"uptime":{"properties":{"server_uptime":{"meta":{"metric_type":"counter"},"type":"long"},"uptime":{"meta":{"metric_type":"counter"},"type":"long"}}}}}}},"ecs":{"properties":{"version":{"ignore_above":1024,"type":"keyword"}}},"data_stream":{"properties":{"namespace":{"type":"constant_keyword"},"type":{"type":"constant_keyword"},"dataset":{"type":"constant_keyword"}}},"service":{"properties":{"address":{"ignore_above":1024,"type":"keyword"},"type":{"ignore_above":1024,"type":"keyword"}}},"host":{"properties":{"hostname":{"ignore_above":1024,"type":"keyword"},"os":{"properties":{"build":{"ignore_above":1024,"type":"keyword"},"kernel":{"ignore_above":1024,"type":"keyword"},"codename":{"ignore_above":1024,"type":"keyword"},"name":{"ignore_above":1024,"type":"keyword","fields":{"text":{"type":"text"}}},"family":{"ignore_above":1024,"type":"keyword"},"version":{"ignore_above":1024,"type":"keyword"},"platform":{"ignore_above":1024,"type":"keyword"}}},"domain":{"ignore_above":1024,"type":"keyword"},"ip":{"type":"ip"},"containerized":{"type":"boolean"},"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"},"type":{"ignore_above":1024,"type":"keyword"},"mac":{"ignore_above":1024,"type":"keyword"},"architecture":{"ignore_above":1024,"type":"keyword"}}},"event":{"properties":{"module":{"type":"constant_keyword","value":"apache"},"dataset":{"type":"constant_keyword","value":"apache.status"}}},"error":{"properties":{"message":{"type":"match_only_text"}}}}}},"composed_of":["metrics-apache.status@settings","metrics-apache.status@custom",".fleet_component_template-1"],"priority":200,"_meta":{"package":{"name":"apache"},"managed_by":"ingest-manager","managed":true},"data_stream":{"hidden":false}}}]}' + headers: + Content-Length: + - "15837" + Content-Type: + - application/json; charset=UTF-8 + X-Elastic-Product: + - Elasticsearch + status: 200 OK + code: 200 + duration: 859.625µs + - id: 2 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: "" + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Authorization: + - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== + User-Agent: + - go-elasticsearch/7.17.10 (darwin arm64; Go 1.24.2) + X-Elastic-Client-Meta: + - es=7.17.10,go=1.24.2,t=7.17.10,hc=1.24.2 + url: https://127.0.0.1:9200/_component_template/logs-apache.error@settings + method: GET + response: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + transfer_encoding: [] + trailer: {} + content_length: 1306 + uncompressed: false + body: '{"component_templates":[{"name":"logs-apache.error@settings","component_template":{"template":{"settings":{"index":{"lifecycle":{"name":"logs"},"codec":"best_compression","mapping":{"total_fields":{"limit":"10000"}},"query":{"default_field":["cloud.account.id","cloud.availability_zone","cloud.instance.id","cloud.instance.name","cloud.machine.type","cloud.provider","cloud.region","cloud.project.id","cloud.image.id","container.id","container.image.name","container.name","host.architecture","host.domain","host.hostname","host.id","host.mac","host.name","host.os.family","host.os.kernel","host.os.name","host.os.platform","host.os.version","host.type","host.os.build","host.os.codename","input.type","tags","ecs.version","event.category","event.kind","event.timezone","event.type","file.path","http.request.method","http.request.referrer","http.version","log.file.path","log.level","source.address","source.as.organization.name","source.geo.city_name","source.geo.continent_name","source.geo.country_iso_code","source.geo.country_name","source.geo.region_iso_code","source.geo.region_name","tags","url.domain","url.extension","url.query","user.name","user_agent.device.name","user_agent.name","user_agent.original","user_agent.os.name","apache.error.module"]}}}},"_meta":{"package":{"name":"apache"}}}}]}' + headers: + Content-Length: + - "1306" + Content-Type: + - application/json; charset=UTF-8 + X-Elastic-Product: + - Elasticsearch + status: 200 OK + code: 200 + duration: 668.958µs + - id: 3 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: "" + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Authorization: + - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== + User-Agent: + - go-elasticsearch/7.17.10 (darwin arm64; Go 1.24.2) + X-Elastic-Client-Meta: + - es=7.17.10,go=1.24.2,t=7.17.10,hc=1.24.2 + url: https://127.0.0.1:9200/_component_template/logs-apache.error@custom + method: GET + response: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + transfer_encoding: [] + trailer: {} + content_length: 149 + uncompressed: false + body: '{"component_templates":[{"name":"logs-apache.error@custom","component_template":{"template":{"settings":{}},"_meta":{"package":{"name":"apache"}}}}]}' + headers: + Content-Length: + - "149" + Content-Type: + - application/json; charset=UTF-8 + X-Elastic-Product: + - Elasticsearch + status: 200 OK + code: 200 + duration: 420.667µs + - id: 4 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: "" + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Authorization: + - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== + User-Agent: + - go-elasticsearch/7.17.10 (darwin arm64; Go 1.24.2) + X-Elastic-Client-Meta: + - es=7.17.10,go=1.24.2,t=7.17.10,hc=1.24.2 + url: https://127.0.0.1:9200/_component_template/.fleet_component_template-1 + method: GET + response: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + transfer_encoding: [] + trailer: {} + content_length: 393 + uncompressed: false + body: '{"component_templates":[{"name":".fleet_component_template-1","component_template":{"template":{"settings":{"index":{"final_pipeline":".fleet_final_pipeline-1"}},"mappings":{"properties":{"event":{"properties":{"agent_id_status":{"ignore_above":1024,"type":"keyword"},"ingested":{"format":"strict_date_time_no_millis||strict_date_optional_time||epoch_millis","type":"date"}}}}}},"_meta":{}}}]}' + headers: + Content-Length: + - "393" + Content-Type: + - application/json; charset=UTF-8 + X-Elastic-Product: + - Elasticsearch + status: 200 OK + code: 200 + duration: 507.167µs + - id: 5 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: "" + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Authorization: + - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== + User-Agent: + - go-elasticsearch/7.17.10 (darwin arm64; Go 1.24.2) + X-Elastic-Client-Meta: + - es=7.17.10,go=1.24.2,t=7.17.10,hc=1.24.2 + url: https://127.0.0.1:9200/_component_template/logs-apache.access@settings + method: GET + response: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + transfer_encoding: [] + trailer: {} + content_length: 1559 + uncompressed: false + body: '{"component_templates":[{"name":"logs-apache.access@settings","component_template":{"template":{"settings":{"index":{"lifecycle":{"name":"logs"},"codec":"best_compression","mapping":{"total_fields":{"limit":"10000"}},"query":{"default_field":["cloud.account.id","cloud.availability_zone","cloud.instance.id","cloud.instance.name","cloud.machine.type","cloud.provider","cloud.region","cloud.project.id","cloud.image.id","container.id","container.image.name","container.name","host.architecture","host.domain","host.hostname","host.id","host.mac","host.name","host.os.family","host.os.kernel","host.os.name","host.os.platform","host.os.version","host.type","host.os.build","host.os.codename","input.type","destination.domain","ecs.version","event.category","event.kind","event.outcome","file.path","http.request.method","http.request.referrer","http.version","log.file.path","log.level","source.address","source.as.organization.name","source.domain","source.geo.city_name","source.geo.continent_name","source.geo.country_iso_code","source.geo.country_name","source.geo.region_iso_code","source.geo.region_name","tags","tls.cipher","tls.version","tls.version_protocol","url.domain","url.extension","url.query","user.name","user_agent.device.name","user_agent.device.name","user_agent.name","user_agent.name","user_agent.original","user_agent.original","user_agent.os.full","user_agent.os.name","user_agent.os.name","user_agent.os.version","user_agent.version","apache.access.ssl.protocol","apache.access.ssl.cipher"]}}}},"_meta":{"package":{"name":"apache"}}}}]}' + headers: + Content-Length: + - "1559" + Content-Type: + - application/json; charset=UTF-8 + X-Elastic-Product: + - Elasticsearch + status: 200 OK + code: 200 + duration: 399.833µs + - id: 6 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: "" + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Authorization: + - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== + User-Agent: + - go-elasticsearch/7.17.10 (darwin arm64; Go 1.24.2) + X-Elastic-Client-Meta: + - es=7.17.10,go=1.24.2,t=7.17.10,hc=1.24.2 + url: https://127.0.0.1:9200/_component_template/logs-apache.access@custom + method: GET + response: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + transfer_encoding: [] + trailer: {} + content_length: 150 + uncompressed: false + body: '{"component_templates":[{"name":"logs-apache.access@custom","component_template":{"template":{"settings":{}},"_meta":{"package":{"name":"apache"}}}}]}' + headers: + Content-Length: + - "150" + Content-Type: + - application/json; charset=UTF-8 + X-Elastic-Product: + - Elasticsearch + status: 200 OK + code: 200 + duration: 679.042µs + - id: 7 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: "" + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Authorization: + - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== + User-Agent: + - go-elasticsearch/7.17.10 (darwin arm64; Go 1.24.2) + X-Elastic-Client-Meta: + - es=7.17.10,go=1.24.2,t=7.17.10,hc=1.24.2 + url: https://127.0.0.1:9200/_component_template/metrics-apache.status@settings + method: GET + response: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + transfer_encoding: [] + trailer: {} + content_length: 789 + uncompressed: false + body: '{"component_templates":[{"name":"metrics-apache.status@settings","component_template":{"template":{"settings":{"index":{"lifecycle":{"name":"metrics"},"codec":"best_compression","mapping":{"total_fields":{"limit":"10000"}},"query":{"default_field":["cloud.account.id","cloud.availability_zone","cloud.instance.id","cloud.instance.name","cloud.machine.type","cloud.provider","cloud.region","cloud.project.id","cloud.image.id","container.id","container.image.name","container.name","host.architecture","host.domain","host.hostname","host.id","host.mac","host.name","host.os.family","host.os.kernel","host.os.name","host.os.platform","host.os.version","host.type","host.os.build","host.os.codename","ecs.version","service.address","service.type"]}}}},"_meta":{"package":{"name":"apache"}}}}]}' + headers: + Content-Length: + - "789" + Content-Type: + - application/json; charset=UTF-8 + X-Elastic-Product: + - Elasticsearch + status: 200 OK + code: 200 + duration: 442µs + - id: 8 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: "" + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Authorization: + - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== + User-Agent: + - go-elasticsearch/7.17.10 (darwin arm64; Go 1.24.2) + X-Elastic-Client-Meta: + - es=7.17.10,go=1.24.2,t=7.17.10,hc=1.24.2 + url: https://127.0.0.1:9200/_component_template/metrics-apache.status@custom + method: GET + response: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + transfer_encoding: [] + trailer: {} + content_length: 153 + uncompressed: false + body: '{"component_templates":[{"name":"metrics-apache.status@custom","component_template":{"template":{"settings":{}},"_meta":{"package":{"name":"apache"}}}}]}' + headers: + Content-Length: + - "153" + Content-Type: + - application/json; charset=UTF-8 + X-Elastic-Product: + - Elasticsearch + status: 200 OK + code: 200 + duration: 416.708µs + - id: 9 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: "" + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Authorization: + - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== + User-Agent: + - go-elasticsearch/7.17.10 (darwin arm64; Go 1.24.2) + X-Elastic-Client-Meta: + - es=7.17.10,go=1.24.2,t=7.17.10,hc=1.24.2 + url: https://127.0.0.1:9200/_ilm/policy/logs + method: GET + response: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + transfer_encoding: [] + trailer: {} + content_length: 1316 + uncompressed: false + body: '{"logs":{"version":1,"modified_date":"2025-05-08T16:05:52.238Z","policy":{"phases":{"hot":{"min_age":"0ms","actions":{"rollover":{"max_primary_shard_size":"50gb","max_age":"30d"}}}},"_meta":{"managed":true,"description":"default policy for the logs index template installed by x-pack"}},"in_use_by":{"indices":[".ds-logs-elastic_agent.filebeat-default-2025.05.08-000001",".ds-logs-elastic_agent.metricbeat-default-2025.05.08-000001",".ds-logs-elastic_agent-default-2025.05.08-000001",".ds-logs-elastic_agent.fleet_server-default-2025.05.08-000001"],"data_streams":["logs-elastic_agent-default","logs-elastic_agent.metricbeat-default","logs-elastic_agent.filebeat-default","logs-elastic_agent.fleet_server-default"],"composable_templates":["logs-apache.access","logs-elastic_agent.cloudbeat","logs-elastic_agent.apm_server","logs-elastic_agent.cloud_defend","logs-system.security","logs-system.auth","logs-elastic_agent.metricbeat","logs-elastic_agent.filebeat","logs-elastic_agent.packetbeat","logs-elastic_agent.filebeat_input","logs-elastic_agent.endpoint_security","logs-elastic_agent.fleet_server","logs-apache.error","logs-system.system","logs-system.application","logs-elastic_agent.osquerybeat","logs-elastic_agent.heartbeat","logs-system.syslog","logs-elastic_agent.auditbeat","logs","logs-elastic_agent"]}}}' + headers: + Content-Length: + - "1316" + Content-Type: + - application/json; charset=UTF-8 + X-Elastic-Product: + - Elasticsearch + status: 200 OK + code: 200 + duration: 965.208µs + - id: 10 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: "" + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Authorization: + - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== + User-Agent: + - go-elasticsearch/7.17.10 (darwin arm64; Go 1.24.2) + X-Elastic-Client-Meta: + - es=7.17.10,go=1.24.2,t=7.17.10,hc=1.24.2 + url: https://127.0.0.1:9200/_ilm/policy/metrics + method: GET + response: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + transfer_encoding: [] + trailer: {} + content_length: 3372 + uncompressed: false + body: '{"metrics":{"version":1,"modified_date":"2025-05-08T16:05:52.220Z","policy":{"phases":{"hot":{"min_age":"0ms","actions":{"rollover":{"max_primary_shard_size":"50gb","max_age":"30d"}}}},"_meta":{"managed":true,"description":"default policy for the metrics index template installed by x-pack"}},"in_use_by":{"indices":[".ds-metrics-system.process-default-2025.05.08-000001",".ds-metrics-docker.healthcheck-default-2025.05.08-000001",".ds-metrics-elastic_agent.elastic_agent-default-2025.05.08-000001",".ds-metrics-docker.network-default-2025.05.08-000001",".ds-metrics-system.load-default-2025.05.08-000001",".ds-metrics-docker.cpu-default-2025.05.08-000001",".ds-metrics-system.memory-default-2025.05.08-000001",".ds-metrics-system.socket_summary-default-2025.05.08-000001",".ds-metrics-system.cpu-default-2025.05.08-000001",".ds-metrics-system.diskio-default-2025.05.08-000001",".ds-metrics-elastic_agent.fleet_server-default-2025.05.08-000001",".ds-metrics-elastic_agent.filebeat-default-2025.05.08-000001",".ds-metrics-system.uptime-default-2025.05.08-000001",".ds-metrics-apache.status-default-2025.05.08-000001",".ds-metrics-elastic_agent.metricbeat-default-2025.05.08-000001",".ds-metrics-docker.container-default-2025.05.08-000001",".ds-metrics-docker.memory-default-2025.05.08-000001",".ds-metrics-system.process.summary-default-2025.05.08-000001",".ds-metrics-system.fsstat-default-2025.05.08-000001",".ds-metrics-system.network-default-2025.05.08-000001",".ds-metrics-docker.info-default-2025.05.08-000001",".ds-metrics-docker.diskio-default-2025.05.08-000001"],"data_streams":["metrics-system.cpu-default","metrics-docker.diskio-default","metrics-docker.container-default","metrics-docker.network-default","metrics-docker.cpu-default","metrics-docker.healthcheck-default","metrics-system.process.summary-default","metrics-apache.status-default","metrics-system.memory-default","metrics-elastic_agent.fleet_server-default","metrics-system.uptime-default","metrics-elastic_agent.elastic_agent-default","metrics-elastic_agent.metricbeat-default","metrics-system.fsstat-default","metrics-system.process-default","metrics-docker.info-default","metrics-elastic_agent.filebeat-default","metrics-system.network-default","metrics-system.diskio-default","metrics-docker.memory-default","metrics-system.load-default","metrics-system.socket_summary-default"],"composable_templates":["metrics-system.process","metrics-elastic_agent.packetbeat","metrics-docker.memory","metrics-system.fsstat","metrics-elastic_agent.osquerybeat","metrics-elastic_agent.endpoint_security","metrics-elastic_agent.apm_server","metrics-docker.info","metrics-system.memory","metrics-docker.network","metrics-system.socket_summary","metrics-docker.image","metrics-apache.status","metrics-elastic_agent.elastic_agent","metrics-elastic_agent.fleet_server","metrics-system.load","metrics-docker.cpu","metrics-system.core","metrics-elastic_agent.filebeat","metrics-elastic_agent.filebeat_input","metrics-system.uptime","metrics-docker.container","metrics-system.process.summary","metrics-system.cpu","metrics-docker.diskio","metrics-elastic_agent.heartbeat","metrics-docker.healthcheck","metrics-system.diskio","metrics-elastic_agent.cloudbeat","metrics-elastic_agent.metricbeat","metrics-elastic_agent.auditbeat","metrics-system.network","metrics-system.filesystem","metrics","metrics-docker.event"]}}}' + headers: + Content-Length: + - "3372" + Content-Type: + - application/json; charset=UTF-8 + X-Elastic-Product: + - Elasticsearch + status: 200 OK + code: 200 + duration: 779.459µs + - id: 11 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: "" + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Authorization: + - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== + User-Agent: + - go-elasticsearch/7.17.10 (darwin arm64; Go 1.24.2) + X-Elastic-Client-Meta: + - es=7.17.10,go=1.24.2,t=7.17.10,hc=1.24.2 + url: https://127.0.0.1:9200/_ingest/pipeline/logs-apache.error-1.3.5,logs-apache.access-1.3.5,.fleet_final_pipeline-1 + method: GET + response: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + transfer_encoding: [] + trailer: {} + content_length: 11056 + uncompressed: false + body: '{"logs-apache.error-1.3.5":{"description":"Pipeline for parsing apache error logs","processors":[{"pipeline":{"if":"ctx.message.startsWith(''{'')","name":"logs-apache.error-1.3.5-third-party"}},{"set":{"field":"event.ingested","value":"{{_ingest.timestamp}}"}},{"set":{"field":"ecs.version","value":"1.12.0"}},{"rename":{"field":"message","target_field":"event.original"}},{"grok":{"field":"event.original","patterns":["\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{LOGLEVEL:log.level}\\]( \\[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\\])? %{GREEDYDATA:message}","\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{DATA:apache.error.module}:%{APACHE_LOGLEVEL:log.level}\\] \\[pid %{NUMBER:process.pid:long}(:tid %{NUMBER:process.thread.id:long})?\\]( \\[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\\])? %{GREEDYDATA:message}"],"pattern_definitions":{"APACHE_LOGLEVEL":"%{LOGLEVEL}[0-9]*","APACHE_TIME":"%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"},"ignore_missing":true}},{"grok":{"field":"message","patterns":["File does not exist: %{URIPATH:file.path}, referer: %{URI:http.request.referrer}","File does not exist: %{URIPATH:file.path}"],"ignore_missing":true,"ignore_failure":true}},{"date":{"if":"ctx.event.timezone == null","field":"apache.error.timestamp","target_field":"@timestamp","formats":["EEE MMM dd H:m:s yyyy","EEE MMM dd H:m:s.SSSSSS yyyy"],"on_failure":[{"append":{"field":"error.message","value":"{{ _ingest.on_failure_message }}"}}]}},{"date":{"if":"ctx.event.timezone != null","field":"apache.error.timestamp","target_field":"@timestamp","formats":["EEE MMM dd H:m:s yyyy","EEE MMM dd H:m:s.SSSSSS yyyy"],"timezone":"{{ event.timezone }}","on_failure":[{"append":{"field":"error.message","value":"{{ _ingest.on_failure_message }}"}}]}},{"remove":{"field":"apache.error.timestamp","ignore_failure":true}},{"set":{"field":"event.kind","value":"event"}},{"set":{"field":"event.category","value":"web"}},{"script":{"if":"ctx?.log?.level != null","lang":"painless","source":"def err_levels = [\"emerg\", \"alert\", \"crit\", \"error\", \"warn\"]; if (err_levels.contains(ctx.log.level)) {\n ctx.event.type = \"error\";\n} else {\n ctx.event.type = \"info\";\n}"}},{"grok":{"field":"source.address","ignore_missing":true,"patterns":["^(%{IP:source.ip}|%{HOSTNAME:source.domain})$"]}},{"geoip":{"field":"source.ip","target_field":"source.geo","ignore_missing":true}},{"geoip":{"database_file":"GeoLite2-ASN.mmdb","field":"source.ip","target_field":"source.as","properties":["asn","organization_name"],"ignore_missing":true}},{"rename":{"field":"source.as.asn","target_field":"source.as.number","ignore_missing":true}},{"rename":{"field":"source.as.organization_name","target_field":"source.as.organization.name","ignore_missing":true}},{"convert":{"field":"source.port","type":"long","ignore_missing":true}},{"script":{"lang":"painless","description":"This script processor iterates over the whole document to remove fields with null values.","source":"void handleMap(Map map) {\n for (def x : map.values()) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n map.values().removeIf(v -> v == null);\n}\nvoid handleList(List list) {\n for (def x : list) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n}\nhandleMap(ctx);\n"}},{"remove":{"field":"event.original","if":"ctx?.tags == null || !(ctx.tags.contains(''preserve_original_event''))","ignore_failure":true,"ignore_missing":true}}],"on_failure":[{"set":{"field":"error.message","value":"{{ _ingest.on_failure_message }}"}}]},"logs-apache.access-1.3.5":{"description":"Pipeline for parsing Apache HTTP Server access logs. Requires the geoip and user_agent plugins.","processors":[{"pipeline":{"if":"ctx.message.startsWith(''{'')","name":"logs-apache.access-1.3.5-third-party"}},{"set":{"field":"event.ingested","value":"{{_ingest.timestamp}}"}},{"set":{"field":"ecs.version","value":"1.12.0"}},{"rename":{"field":"message","target_field":"event.original"}},{"grok":{"field":"event.original","patterns":["%{IPORHOST:destination.domain} %{IPORHOST:source.ip} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?\" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( \"%{DATA:http.request.referrer}\")?( \"%{DATA:user_agent.original}\")?","%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?\" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( \"%{DATA:http.request.referrer}\")?( \"%{DATA:user_agent.original}\")?","%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"-\" %{NUMBER:http.response.status_code:long} -","\\[%{HTTPDATE:apache.access.time}\\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol} %{DATA:apache.access.ssl.cipher} \"%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}\" (-|%{NUMBER:http.response.body.bytes:long})"],"ignore_missing":true}},{"uri_parts":{"field":"_tmp.url_orig","ignore_failure":true}},{"remove":{"field":["_tmp"],"ignore_missing":true}},{"set":{"field":"url.domain","value":"{{destination.domain}}","if":"ctx.url?.domain == null && ctx.destination?.domain != null"}},{"set":{"field":"event.kind","value":"event"}},{"set":{"field":"event.category","value":"web"}},{"set":{"field":"event.outcome","value":"success","if":"ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400"}},{"set":{"field":"event.outcome","value":"failure","if":"ctx?.http?.response?.status_code != null && ctx.http.response.status_code > 399"}},{"grok":{"field":"source.address","ignore_missing":true,"patterns":["^(%{IP:source.ip}|%{HOSTNAME:source.domain})$"]}},{"remove":{"field":"event.created","ignore_missing":true,"ignore_failure":true}},{"rename":{"field":"@timestamp","target_field":"event.created"}},{"date":{"field":"apache.access.time","target_field":"@timestamp","formats":["dd/MMM/yyyy:H:m:s Z"],"ignore_failure":true}},{"remove":{"field":"apache.access.time","ignore_failure":true}},{"user_agent":{"field":"user_agent.original","ignore_failure":true}},{"geoip":{"field":"source.ip","target_field":"source.geo","ignore_missing":true}},{"geoip":{"database_file":"GeoLite2-ASN.mmdb","field":"source.ip","target_field":"source.as","properties":["asn","organization_name"],"ignore_missing":true}},{"rename":{"field":"source.as.asn","target_field":"source.as.number","ignore_missing":true}},{"rename":{"field":"source.as.organization_name","target_field":"source.as.organization.name","ignore_missing":true}},{"set":{"field":"tls.cipher","value":"{{apache.access.ssl.cipher}}","if":"ctx?.apache?.access?.ssl?.cipher != null"}},{"script":{"lang":"painless","if":"ctx?.apache?.access?.ssl?.protocol != null","source":"def parts = ctx.apache.access.ssl.protocol.toLowerCase().splitOnToken(\"v\"); if (parts.length != 2) {\n return;\n} if (parts[1].contains(\".\")) {\n ctx.tls.version = parts[1];\n} else {\n ctx.tls.version = parts[1] + \".0\";\n} ctx.tls.version_protocol = parts[0];"}},{"script":{"lang":"painless","description":"This script processor iterates over the whole document to remove fields with null values.","source":"void handleMap(Map map) {\n for (def x : map.values()) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n map.values().removeIf(v -> v == null);\n}\nvoid handleList(List list) {\n for (def x : list) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n}\nhandleMap(ctx);\n"}},{"remove":{"field":"event.original","if":"ctx?.tags == null || !(ctx.tags.contains(''preserve_original_event''))","ignore_failure":true,"ignore_missing":true}}],"on_failure":[{"set":{"field":"error.message","value":"{{ _ingest.on_failure_message }}"}}]},".fleet_final_pipeline-1":{"version":1,"description":"Final pipeline for processing all incoming Fleet Agent documents.\n","processors":[{"set":{"description":"Add time when event was ingested.","field":"event.ingested","copy_from":"_ingest.timestamp"}},{"script":{"description":"Remove sub-seconds from event.ingested to improve storage efficiency.","tag":"truncate-subseconds-event-ingested","source":"ctx.event.ingested = ctx.event.ingested.withNano(0).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);","ignore_failure":true}},{"remove":{"description":"Remove any pre-existing untrusted values.","field":["event.agent_id_status","_security"],"ignore_missing":true}},{"set_security_user":{"field":"_security","properties":["authentication_type","username","realm","api_key"]}},{"script":{"description":"Add event.agent_id_status based on the API key metadata and the agent.id contained in the event.\n","tag":"agent-id-status","source":"boolean is_user_trusted(def ctx, def users) {\n if (ctx?._security?.username == null) {\n return false;\n }\n\n def user = null;\n for (def item : users) {\n if (item?.username == ctx._security.username) {\n user = item;\n break;\n }\n }\n\n if (user == null || user?.realm == null || ctx?._security?.realm?.name == null) {\n return false;\n }\n\n if (ctx._security.realm.name != user.realm) {\n return false;\n }\n\n return true;\n}\n\nString verified(def ctx, def params) {\n // No agent.id field to validate.\n if (ctx?.agent?.id == null) {\n return \"missing\";\n }\n\n // Check auth metadata from API key.\n if (ctx?._security?.authentication_type == null\n // Agents only use API keys.\n || ctx._security.authentication_type != ''API_KEY''\n // Verify the API key owner before trusting any metadata it contains.\n || !is_user_trusted(ctx, params.trusted_users)\n // Verify the API key has metadata indicating the assigned agent ID.\n || ctx?._security?.api_key?.metadata?.agent_id == null) {\n return \"auth_metadata_missing\";\n }\n\n // The API key can only be used represent the agent.id it was issued to.\n if (ctx._security.api_key.metadata.agent_id != ctx.agent.id) {\n // Potential masquerade attempt.\n return \"mismatch\";\n }\n\n return \"verified\";\n}\n\nif (ctx?.event == null) {\n ctx.event = [:];\n}\n\nctx.event.agent_id_status = verified(ctx, params);","params":{"trusted_users":[{"username":"elastic/fleet-server","realm":"_service_account"},{"username":"cloud-internal-agent-server","realm":"found"},{"username":"elastic","realm":"reserved"}]}}},{"remove":{"field":"_security","ignore_missing":true}}],"on_failure":[{"remove":{"field":"_security","ignore_missing":true,"ignore_failure":true}},{"append":{"field":"error.message","value":["failed in Fleet agent final_pipeline: {{ _ingest.on_failure_message }}"]}}]}}' + headers: + Content-Length: + - "11056" + Content-Type: + - application/json; charset=UTF-8 + X-Elastic-Product: + - Elasticsearch + status: 200 OK + code: 200 + duration: 1.020458ms + - id: 12 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: "" + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Authorization: + - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== + User-Agent: + - go-elasticsearch/7.17.10 (darwin arm64; Go 1.24.2) + X-Elastic-Client-Meta: + - es=7.17.10,go=1.24.2,t=7.17.10,hc=1.24.2 + url: https://127.0.0.1:9200/_ingest/pipeline/logs-apache.access-1.3.5-third-party,logs-apache.error-1.3.5-third-party + method: GET + response: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + transfer_encoding: [] + trailer: {} + content_length: 2094 + uncompressed: false + body: '{"logs-apache.access-1.3.5-third-party":{"description":"Pipeline for parsing Apache HTTP Server logs from third party api","processors":[{"json":{"field":"message","target_field":"json"}},{"drop":{"if":"ctx.json?.result == null"}},{"fingerprint":{"fields":["json.result._cd","json.result._indextime","json.result._raw","json.result._time","json.result.host","json.result.source"],"target_field":"_id","ignore_missing":true}},{"set":{"copy_from":"json.result._raw","field":"message","ignore_empty_value":true}},{"set":{"copy_from":"json.result.host","field":"host.name","ignore_empty_value":true}},{"set":{"copy_from":"json.result.source","field":"file.path","ignore_empty_value":true}},{"remove":{"field":["json"],"ignore_missing":true}}],"on_failure":[{"append":{"field":"error.message","value":"error in third-party pipeline: error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}} with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}} {{ _ingest.on_failure_message }}"}}]},"logs-apache.error-1.3.5-third-party":{"description":"Pipeline for parsing Apache HTTP Server logs from third party api","processors":[{"json":{"field":"message","target_field":"json"}},{"drop":{"if":"ctx.json?.result == null"}},{"fingerprint":{"fields":["json.result._cd","json.result._indextime","json.result._raw","json.result._time","json.result.host","json.result.source"],"target_field":"_id","ignore_missing":true}},{"set":{"copy_from":"json.result._raw","field":"message","ignore_empty_value":true}},{"set":{"copy_from":"json.result.host","field":"host.name","ignore_empty_value":true}},{"set":{"copy_from":"json.result.source","field":"file.path","ignore_empty_value":true}},{"remove":{"field":["json"],"ignore_missing":true}}],"on_failure":[{"append":{"field":"error.message","value":"error in third-party pipeline: error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}} with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}} {{ _ingest.on_failure_message }}"}}]}}' + headers: + Content-Length: + - "2094" + Content-Type: + - application/json; charset=UTF-8 + X-Elastic-Product: + - Elasticsearch + status: 200 OK + code: 200 + duration: 687.959µs + - id: 13 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: "" + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Authorization: + - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== + User-Agent: + - go-elasticsearch/7.17.10 (darwin arm64; Go 1.24.2) + X-Elastic-Client-Meta: + - es=7.17.10,go=1.24.2,t=7.17.10,hc=1.24.2 + url: https://127.0.0.1:9200/_ml/trained_models/apache_*?decompress_definition=false&include=definition%2Cfeature_importance_baseline%2Chyperparameters%2Ctotal_feature_importance + method: GET + response: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + transfer_encoding: [] + trailer: {} + content_length: 38 + uncompressed: false + body: '{"count":0,"trained_model_configs":[]}' + headers: + Content-Length: + - "38" + Content-Type: + - application/json; charset=UTF-8 + X-Elastic-Product: + - Elasticsearch + status: 200 OK + code: 200 + duration: 790.583µs diff --git a/internal/dump/testdata/elasticsearch-8-apache-dump-all/component_templates/logs-apache.access@settings.json b/internal/dump/testdata/elasticsearch-8-apache-dump-all/component_templates/logs-apache.access@settings.json index 547e96d54d..3ab7bb4e91 100644 --- a/internal/dump/testdata/elasticsearch-8-apache-dump-all/component_templates/logs-apache.access@settings.json +++ b/internal/dump/testdata/elasticsearch-8-apache-dump-all/component_templates/logs-apache.access@settings.json @@ -82,7 +82,8 @@ "user_agent.os.version", "user_agent.version", "apache.access.ssl.protocol", - "apache.access.ssl.cipher" + "apache.access.ssl.cipher", + "apache.access.remote_addresses" ] } } diff --git a/internal/dump/testdata/elasticsearch-8-apache-dump-all/ilm_policies/logs.json b/internal/dump/testdata/elasticsearch-8-apache-dump-all/ilm_policies/logs.json index b98c95afa2..5dbf77aec3 100644 --- a/internal/dump/testdata/elasticsearch-8-apache-dump-all/ilm_policies/logs.json +++ b/internal/dump/testdata/elasticsearch-8-apache-dump-all/ilm_policies/logs.json @@ -1,6 +1,6 @@ { "version": 1, - "modified_date": "2023-11-27T16:35:54.053Z", + "modified_date": "2025-05-08T17:28:20.316Z", "policy": { "phases": { "hot": { @@ -20,15 +20,11 @@ }, "in_use_by": { "indices": [ - ".ds-logs-elastic_agent.metricbeat-default-2023.11.27-000001", - ".ds-logs-elastic_agent.fleet_server-default-2023.11.27-000001", - ".ds-logs-elastic_agent.filebeat-default-2023.11.27-000001", - ".ds-logs-elastic_agent-default-2023.11.27-000001" + ".ds-logs-elastic_agent-default-2025.05.08-000001", + ".ds-logs-elastic_agent.fleet_server-default-2025.05.08-000001" ], "data_streams": [ "logs-elastic_agent-default", - "logs-elastic_agent.metricbeat-default", - "logs-elastic_agent.filebeat-default", "logs-elastic_agent.fleet_server-default" ], "composable_templates": [ diff --git a/internal/dump/testdata/elasticsearch-8-apache-dump-all/ilm_policies/metrics.json b/internal/dump/testdata/elasticsearch-8-apache-dump-all/ilm_policies/metrics.json index be9141951f..8813ad582c 100644 --- a/internal/dump/testdata/elasticsearch-8-apache-dump-all/ilm_policies/metrics.json +++ b/internal/dump/testdata/elasticsearch-8-apache-dump-all/ilm_policies/metrics.json @@ -1,6 +1,6 @@ { "version": 1, - "modified_date": "2023-11-27T16:35:54.011Z", + "modified_date": "2025-05-08T17:28:20.341Z", "policy": { "phases": { "hot": { @@ -20,38 +20,12 @@ }, "in_use_by": { "indices": [ - ".ds-metrics-system.process.summary-default-2023.11.27-000001", - ".ds-metrics-system.fsstat-default-2023.11.27-000001", - ".ds-metrics-system.uptime-default-2023.11.27-000001", - ".ds-metrics-system.network-default-2023.11.27-000001", - ".ds-metrics-system.filesystem-default-2023.11.27-000001", - ".ds-metrics-elastic_agent.elastic_agent-default-2023.11.27-000001", - ".ds-metrics-system.socket_summary-default-2023.11.27-000001", - ".ds-metrics-system.diskio-default-2023.11.27-000001", - ".ds-metrics-elastic_agent.filebeat-default-2023.11.27-000001", - ".ds-metrics-system.process-default-2023.11.27-000001", - ".ds-metrics-system.cpu-default-2023.11.27-000001", - ".ds-metrics-elastic_agent.metricbeat-default-2023.11.27-000001", - ".ds-metrics-elastic_agent.fleet_server-default-2023.11.27-000001", - ".ds-metrics-system.load-default-2023.11.27-000001", - ".ds-metrics-system.memory-default-2023.11.27-000001" + ".ds-metrics-elastic_agent.elastic_agent-default-2025.05.08-000001", + ".ds-metrics-elastic_agent.fleet_server-default-2025.05.08-000001" ], "data_streams": [ - "metrics-system.filesystem-default", - "metrics-system.cpu-default", - "metrics-system.process.summary-default", - "metrics-system.memory-default", "metrics-elastic_agent.fleet_server-default", - "metrics-system.uptime-default", - "metrics-elastic_agent.elastic_agent-default", - "metrics-elastic_agent.metricbeat-default", - "metrics-system.fsstat-default", - "metrics-system.process-default", - "metrics-elastic_agent.filebeat-default", - "metrics-system.network-default", - "metrics-system.diskio-default", - "metrics-system.load-default", - "metrics-system.socket_summary-default" + "metrics-elastic_agent.elastic_agent-default" ], "composable_templates": [ "metrics-system.process", diff --git a/internal/dump/testdata/elasticsearch-8-apache-dump-all/index_templates/logs-apache.access.json b/internal/dump/testdata/elasticsearch-8-apache-dump-all/index_templates/logs-apache.access.json index 219cff1e53..5df50e1bfa 100644 --- a/internal/dump/testdata/elasticsearch-8-apache-dump-all/index_templates/logs-apache.access.json +++ b/internal/dump/testdata/elasticsearch-8-apache-dump-all/index_templates/logs-apache.access.json @@ -7,7 +7,7 @@ "template": { "settings": { "index": { - "default_pipeline": "logs-apache.access-1.3.6" + "default_pipeline": "logs-apache.access-1.8.2" } }, "mappings": { @@ -192,6 +192,13 @@ } } }, + "network": { + "properties": { + "forwarded_ip": { + "type": "ip" + } + } + }, "tags": { "ignore_above": 1024, "type": "keyword" @@ -280,6 +287,10 @@ "properties": { "access": { "properties": { + "remote_addresses": { + "ignore_above": 1024, + "type": "keyword" + }, "ssl": { "properties": { "cipher": { @@ -392,6 +403,13 @@ } } }, + "client": { + "properties": { + "ip": { + "type": "ip" + } + } + }, "http": { "properties": { "request": { diff --git a/internal/dump/testdata/elasticsearch-8-apache-dump-all/index_templates/logs-apache.error.json b/internal/dump/testdata/elasticsearch-8-apache-dump-all/index_templates/logs-apache.error.json index 78912bfdec..1434cd5f0e 100644 --- a/internal/dump/testdata/elasticsearch-8-apache-dump-all/index_templates/logs-apache.error.json +++ b/internal/dump/testdata/elasticsearch-8-apache-dump-all/index_templates/logs-apache.error.json @@ -7,7 +7,7 @@ "template": { "settings": { "index": { - "default_pipeline": "logs-apache.error-1.3.6" + "default_pipeline": "logs-apache.error-1.8.2" } }, "mappings": { @@ -411,6 +411,9 @@ }, "event": { "properties": { + "created": { + "type": "date" + }, "kind": { "ignore_above": 1024, "type": "keyword" diff --git a/internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/logs-apache.access-1.3.6-third-party.json b/internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/logs-apache.access-1.8.2-third-party.json similarity index 100% rename from internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/logs-apache.access-1.3.6-third-party.json rename to internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/logs-apache.access-1.8.2-third-party.json diff --git a/internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/logs-apache.access-1.3.6.json b/internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/logs-apache.access-1.8.2.json similarity index 64% rename from internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/logs-apache.access-1.3.6.json rename to internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/logs-apache.access-1.8.2.json index 4faa9950f5..05ef692cda 100644 --- a/internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/logs-apache.access-1.3.6.json +++ b/internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/logs-apache.access-1.8.2.json @@ -4,7 +4,7 @@ { "pipeline": { "if": "ctx.message.startsWith('{')", - "name": "logs-apache.access-1.3.6-third-party" + "name": "logs-apache.access-1.8.2-third-party" } }, { @@ -16,7 +16,7 @@ { "set": { "field": "ecs.version", - "value": "1.12.0" + "value": "8.5.1" } }, { @@ -29,14 +29,48 @@ "grok": { "field": "event.original", "patterns": [ - "%{IPORHOST:destination.domain} %{IPORHOST:source.ip} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?\" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( \"%{DATA:http.request.referrer}\")?( \"%{DATA:user_agent.original}\")?", - "%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?\" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( \"%{DATA:http.request.referrer}\")?( \"%{DATA:user_agent.original}\")?", + "(%{IPORHOST:destination.domain} )?%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?\" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( \"%{DATA:http.request.referrer}\")?( \"%{DATA:user_agent.original}\")?( X-Forwarded-For=\"%{ADDRESS_LIST:apache.access.remote_addresses}\")?", "%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"-\" %{NUMBER:http.response.status_code:long} -", "\\[%{HTTPDATE:apache.access.time}\\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol} %{DATA:apache.access.ssl.cipher} \"%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}\" (-|%{NUMBER:http.response.body.bytes:long})" ], + "ignore_missing": true, + "pattern_definitions": { + "ADDRESS_LIST": "(%{IP})(\"?,?\\s*(%{IP}))*" + } + } + }, + { + "split": { + "field": "apache.access.remote_addresses", + "separator": "\"?,\\s*", "ignore_missing": true } }, + { + "set": { + "field": "network.forwarded_ip", + "value": "{{{apache.access.remote_addresses.0}}}", + "if": "ctx.apache?.access?.remote_addresses != null && ctx.apache.access.remote_addresses.length > 0" + } + }, + { + "script": { + "if": "ctx.apache?.access?.remote_addresses != null && ctx.apache.access.remote_addresses.length > 0", + "lang": "painless", + "tag": "Get source address", + "description": "Extract from remote_addresses, the first non-private IP to ctx.client.ip", + "source": "boolean isPrivateCIDR(def ip) {\n CIDR class_a_network = new CIDR('10.0.0.0/8');\n CIDR class_b_network = new CIDR('172.16.0.0/12');\n CIDR class_c_network = new CIDR('192.168.0.0/16');\n\n try {\n return class_a_network.contains(ip) || class_b_network.contains(ip) || class_c_network.contains(ip);\n } catch (IllegalArgumentException e) {\n return false;\n }\n}\ntry {\n if (ctx.client == null) {\n Map map = new HashMap();\n ctx.put(\"client\", map);\n }\n\n def found = false;\n for (def item : ctx.apache.access.remote_addresses) {\n if (!isPrivateCIDR(item)) {\n ctx.client.ip = item;\n found = true;\n break;\n }\n }\n if (!found) {\n ctx.client.ip = ctx.apache.access.remote_addresses[0];\n }\n} catch (Exception e) {\n ctx.client.ip = null;\n}" + } + }, + { + "append": { + "field": "apache.access.remote_addresses", + "value": [ + "{{source.address}}" + ], + "if": "ctx.source?.address != null" + } + }, { "uri_parts": { "field": "_tmp.url_orig", @@ -74,14 +108,14 @@ "set": { "field": "event.outcome", "value": "success", - "if": "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400" + "if": "ctx.http?.response?.status_code != null && ctx.http.response.status_code < 400" } }, { "set": { "field": "event.outcome", "value": "failure", - "if": "ctx?.http?.response?.status_code != null && ctx.http.response.status_code > 399" + "if": "ctx.http?.response?.status_code != null && ctx.http.response.status_code > 399" } }, { @@ -165,13 +199,13 @@ "set": { "field": "tls.cipher", "value": "{{apache.access.ssl.cipher}}", - "if": "ctx?.apache?.access?.ssl?.cipher != null" + "if": "ctx.apache?.access?.ssl?.cipher != null" } }, { "script": { "lang": "painless", - "if": "ctx?.apache?.access?.ssl?.protocol != null", + "if": "ctx.apache?.access?.ssl?.protocol != null", "source": "def parts = ctx.apache.access.ssl.protocol.toLowerCase().splitOnToken(\"v\"); if (parts.length != 2) {\n return;\n} if (parts[1].contains(\".\")) {\n ctx.tls.version = parts[1];\n} else {\n ctx.tls.version = parts[1] + \".0\";\n} ctx.tls.version_protocol = parts[0];" } }, @@ -185,7 +219,7 @@ { "remove": { "field": "event.original", - "if": "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))", + "if": "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))", "ignore_failure": true, "ignore_missing": true } diff --git a/internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/logs-apache.error-1.3.6-third-party.json b/internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/logs-apache.error-1.8.2-third-party.json similarity index 100% rename from internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/logs-apache.error-1.3.6-third-party.json rename to internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/logs-apache.error-1.8.2-third-party.json diff --git a/internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/logs-apache.error-1.3.6.json b/internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/logs-apache.error-1.8.2.json similarity index 98% rename from internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/logs-apache.error-1.3.6.json rename to internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/logs-apache.error-1.8.2.json index 979cb41edf..e5ffb87175 100644 --- a/internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/logs-apache.error-1.3.6.json +++ b/internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/logs-apache.error-1.8.2.json @@ -4,7 +4,7 @@ { "pipeline": { "if": "ctx.message.startsWith('{')", - "name": "logs-apache.error-1.3.6-third-party" + "name": "logs-apache.error-1.8.2-third-party" } }, { @@ -16,7 +16,7 @@ { "set": { "field": "ecs.version", - "value": "1.12.0" + "value": "8.5.1" } }, { diff --git a/internal/dump/testdata/elasticsearch-8-mock-dump-apache.yaml b/internal/dump/testdata/elasticsearch-8-mock-dump-apache.yaml index 092fa790e6..e8f8615714 100644 --- a/internal/dump/testdata/elasticsearch-8-mock-dump-apache.yaml +++ b/internal/dump/testdata/elasticsearch-8-mock-dump-apache.yaml @@ -18,9 +18,9 @@ interactions: Authorization: - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) + - go-elasticsearch/7.17.10 (darwin arm64; Go 1.24.2) X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 + - es=7.17.10,go=1.24.2,t=7.17.10,hc=1.24.2 url: https://127.0.0.1:9200/ method: GET response: @@ -33,9 +33,9 @@ interactions: uncompressed: false body: | { - "name" : "f2d4529be9e6", + "name" : "2cb2ddb40537", "cluster_name" : "elasticsearch", - "cluster_uuid" : "jzZs7URYRremBeTRiHjDNQ", + "cluster_uuid" : "B4c296ItTAqgrlOOb6sCag", "version" : { "number" : "8.1.0", "build_flavor" : "default", @@ -58,7 +58,7 @@ interactions: - Elasticsearch status: 200 OK code: 200 - duration: 4.245924ms + duration: 4.35575ms - id: 1 request: proto: HTTP/1.1 @@ -76,9 +76,9 @@ interactions: Authorization: - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) + - go-elasticsearch/7.17.10 (darwin arm64; Go 1.24.2) X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 + - es=7.17.10,go=1.24.2,t=7.17.10,hc=1.24.2 url: https://127.0.0.1:9200/_index_template/*-apache.* method: GET response: @@ -87,19 +87,19 @@ interactions: proto_minor: 1 transfer_encoding: [] trailer: {} - content_length: 16106 + content_length: 16291 uncompressed: false - body: '{"index_templates":[{"name":"logs-apache.error","index_template":{"index_patterns":["logs-apache.error-*"],"template":{"settings":{"index":{"default_pipeline":"logs-apache.error-1.3.6"}},"mappings":{"_meta":{"package":{"name":"apache"},"managed_by":"fleet","managed":true},"dynamic_templates":[{"strings_as_keyword":{"mapping":{"ignore_above":1024,"type":"keyword"},"match_mapping_type":"string"}}],"date_detection":false,"properties":{"container":{"properties":{"image":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}},"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"},"labels":{"type":"object"}}},"process":{"properties":{"pid":{"type":"long"},"thread":{"properties":{"id":{"type":"long"}}}}},"log":{"properties":{"file":{"properties":{"path":{"ignore_above":1024,"type":"keyword"}}},"offset":{"type":"long"},"level":{"ignore_above":1024,"type":"keyword"}}},"source":{"properties":{"geo":{"properties":{"continent_name":{"ignore_above":1024,"type":"keyword"},"region_iso_code":{"ignore_above":1024,"type":"keyword"},"city_name":{"ignore_above":1024,"type":"keyword"},"country_iso_code":{"ignore_above":1024,"type":"keyword"},"country_name":{"ignore_above":1024,"type":"keyword"},"location":{"type":"geo_point"},"region_name":{"ignore_above":1024,"type":"keyword"}}},"as":{"properties":{"number":{"type":"long"},"organization":{"properties":{"name":{"ignore_above":1024,"type":"keyword","fields":{}}}}}},"address":{"ignore_above":1024,"type":"keyword"},"port":{"type":"long"},"ip":{"type":"ip"}}},"error":{"properties":{"message":{"type":"match_only_text"}}},"message":{"type":"match_only_text"},"url":{"properties":{"path":{"ignore_above":1024,"type":"wildcard"},"extension":{"ignore_above":1024,"type":"keyword"},"original":{"ignore_above":1024,"type":"wildcard","fields":{}},"domain":{"ignore_above":1024,"type":"keyword"},"query":{"ignore_above":1024,"type":"keyword"}}},"tags":{"ignore_above":1024,"type":"keyword"},"cloud":{"properties":{"availability_zone":{"ignore_above":1024,"type":"keyword"},"image":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}},"instance":{"properties":{"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"}}},"provider":{"ignore_above":1024,"type":"keyword"},"machine":{"properties":{"type":{"ignore_above":1024,"type":"keyword"}}},"project":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}},"region":{"ignore_above":1024,"type":"keyword"},"account":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}}}},"input":{"properties":{"type":{"ignore_above":1024,"type":"keyword"}}},"@timestamp":{"type":"date"},"file":{"properties":{"path":{"ignore_above":1024,"type":"keyword","fields":{}}}},"apache":{"properties":{"error":{"properties":{"module":{"ignore_above":1024,"type":"keyword"}}}}},"ecs":{"properties":{"version":{"ignore_above":1024,"type":"keyword"}}},"data_stream":{"properties":{"namespace":{"type":"constant_keyword"},"type":{"type":"constant_keyword"},"dataset":{"type":"constant_keyword"}}},"host":{"properties":{"hostname":{"ignore_above":1024,"type":"keyword"},"os":{"properties":{"build":{"ignore_above":1024,"type":"keyword"},"kernel":{"ignore_above":1024,"type":"keyword"},"codename":{"ignore_above":1024,"type":"keyword"},"name":{"ignore_above":1024,"type":"keyword","fields":{"text":{"type":"text"}}},"family":{"ignore_above":1024,"type":"keyword"},"version":{"ignore_above":1024,"type":"keyword"},"platform":{"ignore_above":1024,"type":"keyword"}}},"domain":{"ignore_above":1024,"type":"keyword"},"ip":{"type":"ip"},"containerized":{"type":"boolean"},"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"},"type":{"ignore_above":1024,"type":"keyword"},"mac":{"ignore_above":1024,"type":"keyword"},"architecture":{"ignore_above":1024,"type":"keyword"}}},"http":{"properties":{"request":{"properties":{"referrer":{"ignore_above":1024,"type":"keyword"},"method":{"ignore_above":1024,"type":"keyword"}}},"response":{"properties":{"status_code":{"type":"long"},"body":{"properties":{"bytes":{"type":"long"}}}}},"version":{"ignore_above":1024,"type":"keyword"}}},"event":{"properties":{"kind":{"ignore_above":1024,"type":"keyword"},"timezone":{"ignore_above":1024,"type":"keyword"},"module":{"type":"constant_keyword","value":"apache"},"category":{"ignore_above":1024,"type":"keyword"},"type":{"ignore_above":1024,"type":"keyword"},"dataset":{"type":"constant_keyword","value":"apache.error"}}},"user":{"properties":{"name":{"ignore_above":1024,"type":"keyword","fields":{}}}},"user_agent":{"properties":{"original":{"ignore_above":1024,"type":"keyword","fields":{}},"os":{"properties":{"name":{"ignore_above":1024,"type":"keyword","fields":{}}}},"name":{"ignore_above":1024,"type":"keyword"},"device":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}}}}}}},"composed_of":["logs-apache.error@settings","logs-apache.error@custom",".fleet_component_template-1"],"priority":200,"_meta":{"package":{"name":"apache"},"managed_by":"fleet","managed":true},"data_stream":{"hidden":false,"allow_custom_routing":false}}},{"name":"logs-apache.access","index_template":{"index_patterns":["logs-apache.access-*"],"template":{"settings":{"index":{"default_pipeline":"logs-apache.access-1.3.6"}},"mappings":{"_meta":{"package":{"name":"apache"},"managed_by":"fleet","managed":true},"dynamic_templates":[{"strings_as_keyword":{"mapping":{"ignore_above":1024,"type":"keyword"},"match_mapping_type":"string"}}],"date_detection":false,"properties":{"container":{"properties":{"image":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}},"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"},"labels":{"type":"object"}}},"process":{"properties":{"pid":{"type":"long"},"thread":{"properties":{"id":{"type":"long"}}}}},"log":{"properties":{"file":{"properties":{"path":{"ignore_above":1024,"type":"keyword"}}},"offset":{"type":"long"},"level":{"ignore_above":1024,"type":"keyword"}}},"destination":{"properties":{"domain":{"ignore_above":1024,"type":"keyword"}}},"source":{"properties":{"geo":{"properties":{"continent_name":{"ignore_above":1024,"type":"keyword"},"region_iso_code":{"ignore_above":1024,"type":"keyword"},"city_name":{"ignore_above":1024,"type":"keyword"},"country_iso_code":{"ignore_above":1024,"type":"keyword"},"country_name":{"ignore_above":1024,"type":"keyword"},"location":{"type":"geo_point"},"region_name":{"ignore_above":1024,"type":"keyword"}}},"as":{"properties":{"number":{"type":"long"},"organization":{"properties":{"name":{"ignore_above":1024,"type":"keyword","fields":{}}}}}},"address":{"ignore_above":1024,"type":"keyword"},"domain":{"ignore_above":1024,"type":"keyword"},"ip":{"type":"ip"}}},"error":{"properties":{"message":{"type":"match_only_text"}}},"message":{"type":"match_only_text"},"url":{"properties":{"path":{"ignore_above":1024,"type":"wildcard"},"extension":{"ignore_above":1024,"type":"keyword"},"original":{"ignore_above":1024,"type":"wildcard","fields":{}},"domain":{"ignore_above":1024,"type":"keyword"},"query":{"ignore_above":1024,"type":"keyword"}}},"tags":{"ignore_above":1024,"type":"keyword"},"cloud":{"properties":{"availability_zone":{"ignore_above":1024,"type":"keyword"},"image":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}},"instance":{"properties":{"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"}}},"provider":{"ignore_above":1024,"type":"keyword"},"machine":{"properties":{"type":{"ignore_above":1024,"type":"keyword"}}},"project":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}},"region":{"ignore_above":1024,"type":"keyword"},"account":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}}}},"input":{"properties":{"type":{"ignore_above":1024,"type":"keyword"}}},"@timestamp":{"type":"date"},"file":{"properties":{"path":{"ignore_above":1024,"type":"keyword","fields":{}}}},"apache":{"properties":{"access":{"properties":{"ssl":{"properties":{"cipher":{"ignore_above":1024,"type":"keyword"},"protocol":{"ignore_above":1024,"type":"keyword"}}}}}}},"ecs":{"properties":{"version":{"ignore_above":1024,"type":"keyword"}}},"data_stream":{"properties":{"namespace":{"type":"constant_keyword"},"type":{"type":"constant_keyword"},"dataset":{"type":"constant_keyword"}}},"host":{"properties":{"hostname":{"ignore_above":1024,"type":"keyword"},"os":{"properties":{"build":{"ignore_above":1024,"type":"keyword"},"kernel":{"ignore_above":1024,"type":"keyword"},"codename":{"ignore_above":1024,"type":"keyword"},"name":{"ignore_above":1024,"type":"keyword","fields":{"text":{"type":"text"}}},"family":{"ignore_above":1024,"type":"keyword"},"version":{"ignore_above":1024,"type":"keyword"},"platform":{"ignore_above":1024,"type":"keyword"}}},"domain":{"ignore_above":1024,"type":"keyword"},"ip":{"type":"ip"},"containerized":{"type":"boolean"},"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"},"type":{"ignore_above":1024,"type":"keyword"},"mac":{"ignore_above":1024,"type":"keyword"},"architecture":{"ignore_above":1024,"type":"keyword"}}},"http":{"properties":{"request":{"properties":{"referrer":{"ignore_above":1024,"type":"keyword"},"method":{"ignore_above":1024,"type":"keyword"}}},"response":{"properties":{"status_code":{"type":"long"},"body":{"properties":{"bytes":{"type":"long"}}}}},"version":{"ignore_above":1024,"type":"keyword"}}},"tls":{"properties":{"cipher":{"ignore_above":1024,"type":"keyword"},"version":{"ignore_above":1024,"type":"keyword"},"version_protocol":{"ignore_above":1024,"type":"keyword"}}},"event":{"properties":{"created":{"type":"date"},"kind":{"ignore_above":1024,"type":"keyword"},"module":{"type":"constant_keyword","value":"apache"},"category":{"ignore_above":1024,"type":"keyword"},"dataset":{"type":"constant_keyword","value":"apache.access"},"outcome":{"ignore_above":1024,"type":"keyword"}}},"user":{"properties":{"name":{"ignore_above":1024,"type":"keyword","fields":{}}}},"user_agent":{"properties":{"original":{"ignore_above":1024,"type":"keyword","fields":{}},"os":{"properties":{"name":{"ignore_above":1024,"type":"keyword","fields":{}},"version":{"ignore_above":1024,"type":"keyword"},"full":{"ignore_above":1024,"type":"keyword","fields":{}}}},"name":{"ignore_above":1024,"type":"keyword"},"device":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}},"version":{"ignore_above":1024,"type":"keyword"}}}}}},"composed_of":["logs-apache.access@settings","logs-apache.access@custom",".fleet_component_template-1"],"priority":200,"_meta":{"package":{"name":"apache"},"managed_by":"fleet","managed":true},"data_stream":{"hidden":false,"allow_custom_routing":false}}},{"name":"metrics-apache.status","index_template":{"index_patterns":["metrics-apache.status-*"],"template":{"settings":{},"mappings":{"_meta":{"package":{"name":"apache"},"managed_by":"fleet","managed":true},"dynamic_templates":[{"strings_as_keyword":{"mapping":{"ignore_above":1024,"type":"keyword"},"match_mapping_type":"string"}}],"date_detection":false,"properties":{"cloud":{"properties":{"availability_zone":{"ignore_above":1024,"type":"keyword"},"image":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}},"instance":{"properties":{"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"}}},"provider":{"ignore_above":1024,"type":"keyword"},"machine":{"properties":{"type":{"ignore_above":1024,"type":"keyword"}}},"project":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}},"region":{"ignore_above":1024,"type":"keyword"},"account":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}}}},"container":{"properties":{"image":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}},"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"},"labels":{"type":"object"}}},"@timestamp":{"type":"date"},"apache":{"properties":{"status":{"properties":{"bytes_per_request":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"load":{"properties":{"1":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"15":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"5":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"}}},"bytes_per_sec":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"total_bytes":{"meta":{"unit":"byte","metric_type":"counter"},"type":"long"},"cpu":{"properties":{"system":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"load":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"children_system":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"children_user":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"user":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"}}},"total_accesses":{"meta":{"metric_type":"counter"},"type":"long"},"scoreboard":{"properties":{"total":{"meta":{"metric_type":"gauge"},"type":"long"},"keepalive":{"meta":{"metric_type":"gauge"},"type":"long"},"idle_cleanup":{"meta":{"metric_type":"gauge"},"type":"long"},"waiting_for_connection":{"meta":{"metric_type":"gauge"},"type":"long"},"logging":{"meta":{"metric_type":"gauge"},"type":"long"},"gracefully_finishing":{"meta":{"metric_type":"gauge"},"type":"long"},"open_slot":{"meta":{"metric_type":"gauge"},"type":"long"},"dns_lookup":{"meta":{"metric_type":"gauge"},"type":"long"},"sending_reply":{"meta":{"metric_type":"gauge"},"type":"long"},"closing_connection":{"meta":{"metric_type":"gauge"},"type":"long"},"starting_up":{"meta":{"metric_type":"gauge"},"type":"long"},"reading_request":{"meta":{"metric_type":"gauge"},"type":"long"}}},"workers":{"properties":{"idle":{"meta":{"metric_type":"gauge"},"type":"long"},"busy":{"meta":{"metric_type":"gauge"},"type":"long"}}},"connections":{"properties":{"async":{"properties":{"closing":{"meta":{"metric_type":"gauge"},"type":"long"},"writing":{"meta":{"metric_type":"gauge"},"type":"long"},"keep_alive":{"meta":{"metric_type":"gauge"},"type":"long"}}},"total":{"meta":{"metric_type":"counter"},"type":"long"}}},"requests_per_sec":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"uptime":{"properties":{"server_uptime":{"meta":{"metric_type":"counter"},"type":"long"},"uptime":{"meta":{"metric_type":"counter"},"type":"long"}}}}}}},"ecs":{"properties":{"version":{"ignore_above":1024,"type":"keyword"}}},"data_stream":{"properties":{"namespace":{"type":"constant_keyword"},"type":{"type":"constant_keyword"},"dataset":{"type":"constant_keyword"}}},"service":{"properties":{"address":{"ignore_above":1024,"type":"keyword"},"type":{"ignore_above":1024,"type":"keyword"}}},"host":{"properties":{"hostname":{"ignore_above":1024,"type":"keyword"},"os":{"properties":{"build":{"ignore_above":1024,"type":"keyword"},"kernel":{"ignore_above":1024,"type":"keyword"},"codename":{"ignore_above":1024,"type":"keyword"},"name":{"ignore_above":1024,"type":"keyword","fields":{"text":{"type":"text"}}},"family":{"ignore_above":1024,"type":"keyword"},"version":{"ignore_above":1024,"type":"keyword"},"platform":{"ignore_above":1024,"type":"keyword"}}},"domain":{"ignore_above":1024,"type":"keyword"},"ip":{"type":"ip"},"containerized":{"type":"boolean"},"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"},"type":{"ignore_above":1024,"type":"keyword"},"mac":{"ignore_above":1024,"type":"keyword"},"architecture":{"ignore_above":1024,"type":"keyword"}}},"event":{"properties":{"module":{"type":"constant_keyword","value":"apache"},"dataset":{"type":"constant_keyword","value":"apache.status"}}},"error":{"properties":{"message":{"type":"match_only_text"}}}}}},"composed_of":["metrics-apache.status@settings","metrics-apache.status@custom",".fleet_component_template-1"],"priority":200,"_meta":{"package":{"name":"apache"},"managed_by":"fleet","managed":true},"data_stream":{"hidden":false,"allow_custom_routing":false}}}]}' + body: '{"index_templates":[{"name":"logs-apache.error","index_template":{"index_patterns":["logs-apache.error-*"],"template":{"settings":{"index":{"default_pipeline":"logs-apache.error-1.8.2"}},"mappings":{"_meta":{"package":{"name":"apache"},"managed_by":"fleet","managed":true},"dynamic_templates":[{"strings_as_keyword":{"mapping":{"ignore_above":1024,"type":"keyword"},"match_mapping_type":"string"}}],"date_detection":false,"properties":{"container":{"properties":{"image":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}},"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"},"labels":{"type":"object"}}},"process":{"properties":{"pid":{"type":"long"},"thread":{"properties":{"id":{"type":"long"}}}}},"log":{"properties":{"file":{"properties":{"path":{"ignore_above":1024,"type":"keyword"}}},"offset":{"type":"long"},"level":{"ignore_above":1024,"type":"keyword"}}},"source":{"properties":{"geo":{"properties":{"continent_name":{"ignore_above":1024,"type":"keyword"},"region_iso_code":{"ignore_above":1024,"type":"keyword"},"city_name":{"ignore_above":1024,"type":"keyword"},"country_iso_code":{"ignore_above":1024,"type":"keyword"},"country_name":{"ignore_above":1024,"type":"keyword"},"location":{"type":"geo_point"},"region_name":{"ignore_above":1024,"type":"keyword"}}},"as":{"properties":{"number":{"type":"long"},"organization":{"properties":{"name":{"ignore_above":1024,"type":"keyword","fields":{}}}}}},"address":{"ignore_above":1024,"type":"keyword"},"port":{"type":"long"},"ip":{"type":"ip"}}},"error":{"properties":{"message":{"type":"match_only_text"}}},"message":{"type":"match_only_text"},"url":{"properties":{"path":{"ignore_above":1024,"type":"wildcard"},"extension":{"ignore_above":1024,"type":"keyword"},"original":{"ignore_above":1024,"type":"wildcard","fields":{}},"domain":{"ignore_above":1024,"type":"keyword"},"query":{"ignore_above":1024,"type":"keyword"}}},"tags":{"ignore_above":1024,"type":"keyword"},"cloud":{"properties":{"availability_zone":{"ignore_above":1024,"type":"keyword"},"image":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}},"instance":{"properties":{"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"}}},"provider":{"ignore_above":1024,"type":"keyword"},"machine":{"properties":{"type":{"ignore_above":1024,"type":"keyword"}}},"project":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}},"region":{"ignore_above":1024,"type":"keyword"},"account":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}}}},"input":{"properties":{"type":{"ignore_above":1024,"type":"keyword"}}},"@timestamp":{"type":"date"},"file":{"properties":{"path":{"ignore_above":1024,"type":"keyword","fields":{}}}},"apache":{"properties":{"error":{"properties":{"module":{"ignore_above":1024,"type":"keyword"}}}}},"ecs":{"properties":{"version":{"ignore_above":1024,"type":"keyword"}}},"data_stream":{"properties":{"namespace":{"type":"constant_keyword"},"type":{"type":"constant_keyword"},"dataset":{"type":"constant_keyword"}}},"host":{"properties":{"hostname":{"ignore_above":1024,"type":"keyword"},"os":{"properties":{"build":{"ignore_above":1024,"type":"keyword"},"kernel":{"ignore_above":1024,"type":"keyword"},"codename":{"ignore_above":1024,"type":"keyword"},"name":{"ignore_above":1024,"type":"keyword","fields":{"text":{"type":"text"}}},"family":{"ignore_above":1024,"type":"keyword"},"version":{"ignore_above":1024,"type":"keyword"},"platform":{"ignore_above":1024,"type":"keyword"}}},"domain":{"ignore_above":1024,"type":"keyword"},"ip":{"type":"ip"},"containerized":{"type":"boolean"},"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"},"type":{"ignore_above":1024,"type":"keyword"},"mac":{"ignore_above":1024,"type":"keyword"},"architecture":{"ignore_above":1024,"type":"keyword"}}},"http":{"properties":{"request":{"properties":{"referrer":{"ignore_above":1024,"type":"keyword"},"method":{"ignore_above":1024,"type":"keyword"}}},"response":{"properties":{"status_code":{"type":"long"},"body":{"properties":{"bytes":{"type":"long"}}}}},"version":{"ignore_above":1024,"type":"keyword"}}},"event":{"properties":{"created":{"type":"date"},"kind":{"ignore_above":1024,"type":"keyword"},"timezone":{"ignore_above":1024,"type":"keyword"},"module":{"type":"constant_keyword","value":"apache"},"category":{"ignore_above":1024,"type":"keyword"},"type":{"ignore_above":1024,"type":"keyword"},"dataset":{"type":"constant_keyword","value":"apache.error"}}},"user":{"properties":{"name":{"ignore_above":1024,"type":"keyword","fields":{}}}},"user_agent":{"properties":{"original":{"ignore_above":1024,"type":"keyword","fields":{}},"os":{"properties":{"name":{"ignore_above":1024,"type":"keyword","fields":{}}}},"name":{"ignore_above":1024,"type":"keyword"},"device":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}}}}}}},"composed_of":["logs-apache.error@settings","logs-apache.error@custom",".fleet_component_template-1"],"priority":200,"_meta":{"package":{"name":"apache"},"managed_by":"fleet","managed":true},"data_stream":{"hidden":false,"allow_custom_routing":false}}},{"name":"logs-apache.access","index_template":{"index_patterns":["logs-apache.access-*"],"template":{"settings":{"index":{"default_pipeline":"logs-apache.access-1.8.2"}},"mappings":{"_meta":{"package":{"name":"apache"},"managed_by":"fleet","managed":true},"dynamic_templates":[{"strings_as_keyword":{"mapping":{"ignore_above":1024,"type":"keyword"},"match_mapping_type":"string"}}],"date_detection":false,"properties":{"container":{"properties":{"image":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}},"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"},"labels":{"type":"object"}}},"process":{"properties":{"pid":{"type":"long"},"thread":{"properties":{"id":{"type":"long"}}}}},"log":{"properties":{"file":{"properties":{"path":{"ignore_above":1024,"type":"keyword"}}},"offset":{"type":"long"},"level":{"ignore_above":1024,"type":"keyword"}}},"destination":{"properties":{"domain":{"ignore_above":1024,"type":"keyword"}}},"source":{"properties":{"geo":{"properties":{"continent_name":{"ignore_above":1024,"type":"keyword"},"region_iso_code":{"ignore_above":1024,"type":"keyword"},"city_name":{"ignore_above":1024,"type":"keyword"},"country_iso_code":{"ignore_above":1024,"type":"keyword"},"country_name":{"ignore_above":1024,"type":"keyword"},"location":{"type":"geo_point"},"region_name":{"ignore_above":1024,"type":"keyword"}}},"as":{"properties":{"number":{"type":"long"},"organization":{"properties":{"name":{"ignore_above":1024,"type":"keyword","fields":{}}}}}},"address":{"ignore_above":1024,"type":"keyword"},"domain":{"ignore_above":1024,"type":"keyword"},"ip":{"type":"ip"}}},"error":{"properties":{"message":{"type":"match_only_text"}}},"message":{"type":"match_only_text"},"url":{"properties":{"path":{"ignore_above":1024,"type":"wildcard"},"extension":{"ignore_above":1024,"type":"keyword"},"original":{"ignore_above":1024,"type":"wildcard","fields":{}},"domain":{"ignore_above":1024,"type":"keyword"},"query":{"ignore_above":1024,"type":"keyword"}}},"network":{"properties":{"forwarded_ip":{"type":"ip"}}},"tags":{"ignore_above":1024,"type":"keyword"},"cloud":{"properties":{"availability_zone":{"ignore_above":1024,"type":"keyword"},"image":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}},"instance":{"properties":{"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"}}},"provider":{"ignore_above":1024,"type":"keyword"},"machine":{"properties":{"type":{"ignore_above":1024,"type":"keyword"}}},"project":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}},"region":{"ignore_above":1024,"type":"keyword"},"account":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}}}},"input":{"properties":{"type":{"ignore_above":1024,"type":"keyword"}}},"@timestamp":{"type":"date"},"file":{"properties":{"path":{"ignore_above":1024,"type":"keyword","fields":{}}}},"apache":{"properties":{"access":{"properties":{"remote_addresses":{"ignore_above":1024,"type":"keyword"},"ssl":{"properties":{"cipher":{"ignore_above":1024,"type":"keyword"},"protocol":{"ignore_above":1024,"type":"keyword"}}}}}}},"ecs":{"properties":{"version":{"ignore_above":1024,"type":"keyword"}}},"data_stream":{"properties":{"namespace":{"type":"constant_keyword"},"type":{"type":"constant_keyword"},"dataset":{"type":"constant_keyword"}}},"host":{"properties":{"hostname":{"ignore_above":1024,"type":"keyword"},"os":{"properties":{"build":{"ignore_above":1024,"type":"keyword"},"kernel":{"ignore_above":1024,"type":"keyword"},"codename":{"ignore_above":1024,"type":"keyword"},"name":{"ignore_above":1024,"type":"keyword","fields":{"text":{"type":"text"}}},"family":{"ignore_above":1024,"type":"keyword"},"version":{"ignore_above":1024,"type":"keyword"},"platform":{"ignore_above":1024,"type":"keyword"}}},"domain":{"ignore_above":1024,"type":"keyword"},"ip":{"type":"ip"},"containerized":{"type":"boolean"},"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"},"type":{"ignore_above":1024,"type":"keyword"},"mac":{"ignore_above":1024,"type":"keyword"},"architecture":{"ignore_above":1024,"type":"keyword"}}},"client":{"properties":{"ip":{"type":"ip"}}},"http":{"properties":{"request":{"properties":{"referrer":{"ignore_above":1024,"type":"keyword"},"method":{"ignore_above":1024,"type":"keyword"}}},"response":{"properties":{"status_code":{"type":"long"},"body":{"properties":{"bytes":{"type":"long"}}}}},"version":{"ignore_above":1024,"type":"keyword"}}},"tls":{"properties":{"cipher":{"ignore_above":1024,"type":"keyword"},"version":{"ignore_above":1024,"type":"keyword"},"version_protocol":{"ignore_above":1024,"type":"keyword"}}},"event":{"properties":{"created":{"type":"date"},"kind":{"ignore_above":1024,"type":"keyword"},"module":{"type":"constant_keyword","value":"apache"},"category":{"ignore_above":1024,"type":"keyword"},"dataset":{"type":"constant_keyword","value":"apache.access"},"outcome":{"ignore_above":1024,"type":"keyword"}}},"user":{"properties":{"name":{"ignore_above":1024,"type":"keyword","fields":{}}}},"user_agent":{"properties":{"original":{"ignore_above":1024,"type":"keyword","fields":{}},"os":{"properties":{"name":{"ignore_above":1024,"type":"keyword","fields":{}},"version":{"ignore_above":1024,"type":"keyword"},"full":{"ignore_above":1024,"type":"keyword","fields":{}}}},"name":{"ignore_above":1024,"type":"keyword"},"device":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}},"version":{"ignore_above":1024,"type":"keyword"}}}}}},"composed_of":["logs-apache.access@settings","logs-apache.access@custom",".fleet_component_template-1"],"priority":200,"_meta":{"package":{"name":"apache"},"managed_by":"fleet","managed":true},"data_stream":{"hidden":false,"allow_custom_routing":false}}},{"name":"metrics-apache.status","index_template":{"index_patterns":["metrics-apache.status-*"],"template":{"settings":{},"mappings":{"_meta":{"package":{"name":"apache"},"managed_by":"fleet","managed":true},"dynamic_templates":[{"strings_as_keyword":{"mapping":{"ignore_above":1024,"type":"keyword"},"match_mapping_type":"string"}}],"date_detection":false,"properties":{"cloud":{"properties":{"availability_zone":{"ignore_above":1024,"type":"keyword"},"image":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}},"instance":{"properties":{"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"}}},"provider":{"ignore_above":1024,"type":"keyword"},"machine":{"properties":{"type":{"ignore_above":1024,"type":"keyword"}}},"project":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}},"region":{"ignore_above":1024,"type":"keyword"},"account":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}}}},"container":{"properties":{"image":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}},"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"},"labels":{"type":"object"}}},"@timestamp":{"type":"date"},"apache":{"properties":{"status":{"properties":{"bytes_per_request":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"load":{"properties":{"1":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"15":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"5":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"}}},"bytes_per_sec":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"total_bytes":{"meta":{"unit":"byte","metric_type":"counter"},"type":"long"},"cpu":{"properties":{"system":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"load":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"children_system":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"children_user":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"user":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"}}},"total_accesses":{"meta":{"metric_type":"counter"},"type":"long"},"scoreboard":{"properties":{"total":{"meta":{"metric_type":"gauge"},"type":"long"},"keepalive":{"meta":{"metric_type":"gauge"},"type":"long"},"idle_cleanup":{"meta":{"metric_type":"gauge"},"type":"long"},"waiting_for_connection":{"meta":{"metric_type":"gauge"},"type":"long"},"logging":{"meta":{"metric_type":"gauge"},"type":"long"},"gracefully_finishing":{"meta":{"metric_type":"gauge"},"type":"long"},"open_slot":{"meta":{"metric_type":"gauge"},"type":"long"},"dns_lookup":{"meta":{"metric_type":"gauge"},"type":"long"},"sending_reply":{"meta":{"metric_type":"gauge"},"type":"long"},"closing_connection":{"meta":{"metric_type":"gauge"},"type":"long"},"starting_up":{"meta":{"metric_type":"gauge"},"type":"long"},"reading_request":{"meta":{"metric_type":"gauge"},"type":"long"}}},"workers":{"properties":{"idle":{"meta":{"metric_type":"gauge"},"type":"long"},"busy":{"meta":{"metric_type":"gauge"},"type":"long"}}},"connections":{"properties":{"async":{"properties":{"closing":{"meta":{"metric_type":"gauge"},"type":"long"},"writing":{"meta":{"metric_type":"gauge"},"type":"long"},"keep_alive":{"meta":{"metric_type":"gauge"},"type":"long"}}},"total":{"meta":{"metric_type":"counter"},"type":"long"}}},"requests_per_sec":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"uptime":{"properties":{"server_uptime":{"meta":{"metric_type":"counter"},"type":"long"},"uptime":{"meta":{"metric_type":"counter"},"type":"long"}}}}}}},"ecs":{"properties":{"version":{"ignore_above":1024,"type":"keyword"}}},"data_stream":{"properties":{"namespace":{"type":"constant_keyword"},"type":{"type":"constant_keyword"},"dataset":{"type":"constant_keyword"}}},"service":{"properties":{"address":{"ignore_above":1024,"type":"keyword"},"type":{"ignore_above":1024,"type":"keyword"}}},"host":{"properties":{"hostname":{"ignore_above":1024,"type":"keyword"},"os":{"properties":{"build":{"ignore_above":1024,"type":"keyword"},"kernel":{"ignore_above":1024,"type":"keyword"},"codename":{"ignore_above":1024,"type":"keyword"},"name":{"ignore_above":1024,"type":"keyword","fields":{"text":{"type":"text"}}},"family":{"ignore_above":1024,"type":"keyword"},"version":{"ignore_above":1024,"type":"keyword"},"platform":{"ignore_above":1024,"type":"keyword"}}},"domain":{"ignore_above":1024,"type":"keyword"},"ip":{"type":"ip"},"containerized":{"type":"boolean"},"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"},"type":{"ignore_above":1024,"type":"keyword"},"mac":{"ignore_above":1024,"type":"keyword"},"architecture":{"ignore_above":1024,"type":"keyword"}}},"event":{"properties":{"module":{"type":"constant_keyword","value":"apache"},"dataset":{"type":"constant_keyword","value":"apache.status"}}},"error":{"properties":{"message":{"type":"match_only_text"}}}}}},"composed_of":["metrics-apache.status@settings","metrics-apache.status@custom",".fleet_component_template-1"],"priority":200,"_meta":{"package":{"name":"apache"},"managed_by":"fleet","managed":true},"data_stream":{"hidden":false,"allow_custom_routing":false}}}]}' headers: Content-Length: - - "16106" + - "16291" Content-Type: - application/json X-Elastic-Product: - Elasticsearch status: 200 OK code: 200 - duration: 1.497043ms + duration: 1.042625ms - id: 2 request: proto: HTTP/1.1 @@ -117,9 +117,9 @@ interactions: Authorization: - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) + - go-elasticsearch/7.17.10 (darwin arm64; Go 1.24.2) X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 + - es=7.17.10,go=1.24.2,t=7.17.10,hc=1.24.2 url: https://127.0.0.1:9200/_component_template/logs-apache.error@settings method: GET response: @@ -140,7 +140,7 @@ interactions: - Elasticsearch status: 200 OK code: 200 - duration: 513.497µs + duration: 783.708µs - id: 3 request: proto: HTTP/1.1 @@ -158,9 +158,9 @@ interactions: Authorization: - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) + - go-elasticsearch/7.17.10 (darwin arm64; Go 1.24.2) X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 + - es=7.17.10,go=1.24.2,t=7.17.10,hc=1.24.2 url: https://127.0.0.1:9200/_component_template/logs-apache.error@custom method: GET response: @@ -181,7 +181,7 @@ interactions: - Elasticsearch status: 200 OK code: 200 - duration: 369.89µs + duration: 393.5µs - id: 4 request: proto: HTTP/1.1 @@ -199,9 +199,9 @@ interactions: Authorization: - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) + - go-elasticsearch/7.17.10 (darwin arm64; Go 1.24.2) X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 + - es=7.17.10,go=1.24.2,t=7.17.10,hc=1.24.2 url: https://127.0.0.1:9200/_component_template/.fleet_component_template-1 method: GET response: @@ -222,7 +222,7 @@ interactions: - Elasticsearch status: 200 OK code: 200 - duration: 424.645µs + duration: 539.625µs - id: 5 request: proto: HTTP/1.1 @@ -240,9 +240,9 @@ interactions: Authorization: - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) + - go-elasticsearch/7.17.10 (darwin arm64; Go 1.24.2) X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 + - es=7.17.10,go=1.24.2,t=7.17.10,hc=1.24.2 url: https://127.0.0.1:9200/_component_template/logs-apache.access@settings method: GET response: @@ -251,19 +251,19 @@ interactions: proto_minor: 1 transfer_encoding: [] trailer: {} - content_length: 1595 + content_length: 1628 uncompressed: false - body: '{"component_templates":[{"name":"logs-apache.access@settings","component_template":{"template":{"settings":{"index":{"lifecycle":{"name":"logs"},"codec":"best_compression","mapping":{"total_fields":{"limit":"10000"}},"query":{"default_field":["cloud.account.id","cloud.availability_zone","cloud.instance.id","cloud.instance.name","cloud.machine.type","cloud.provider","cloud.region","cloud.project.id","cloud.image.id","container.id","container.image.name","container.name","host.architecture","host.domain","host.hostname","host.id","host.mac","host.name","host.os.family","host.os.kernel","host.os.name","host.os.platform","host.os.version","host.type","host.os.build","host.os.codename","input.type","destination.domain","ecs.version","event.category","event.kind","event.outcome","file.path","http.request.method","http.request.referrer","http.version","log.file.path","log.level","source.address","source.as.organization.name","source.domain","source.geo.city_name","source.geo.continent_name","source.geo.country_iso_code","source.geo.country_name","source.geo.region_iso_code","source.geo.region_name","tags","tls.cipher","tls.version","tls.version_protocol","url.domain","url.extension","url.query","user.name","user_agent.device.name","user_agent.device.name","user_agent.name","user_agent.name","user_agent.original","user_agent.original","user_agent.os.full","user_agent.os.name","user_agent.os.name","user_agent.os.version","user_agent.version","apache.access.ssl.protocol","apache.access.ssl.cipher"]}}}},"_meta":{"package":{"name":"apache"},"managed_by":"fleet","managed":true}}}]}' + body: '{"component_templates":[{"name":"logs-apache.access@settings","component_template":{"template":{"settings":{"index":{"lifecycle":{"name":"logs"},"codec":"best_compression","mapping":{"total_fields":{"limit":"10000"}},"query":{"default_field":["cloud.account.id","cloud.availability_zone","cloud.instance.id","cloud.instance.name","cloud.machine.type","cloud.provider","cloud.region","cloud.project.id","cloud.image.id","container.id","container.image.name","container.name","host.architecture","host.domain","host.hostname","host.id","host.mac","host.name","host.os.family","host.os.kernel","host.os.name","host.os.platform","host.os.version","host.type","host.os.build","host.os.codename","input.type","destination.domain","ecs.version","event.category","event.kind","event.outcome","file.path","http.request.method","http.request.referrer","http.version","log.file.path","log.level","source.address","source.as.organization.name","source.domain","source.geo.city_name","source.geo.continent_name","source.geo.country_iso_code","source.geo.country_name","source.geo.region_iso_code","source.geo.region_name","tags","tls.cipher","tls.version","tls.version_protocol","url.domain","url.extension","url.query","user.name","user_agent.device.name","user_agent.device.name","user_agent.name","user_agent.name","user_agent.original","user_agent.original","user_agent.os.full","user_agent.os.name","user_agent.os.name","user_agent.os.version","user_agent.version","apache.access.ssl.protocol","apache.access.ssl.cipher","apache.access.remote_addresses"]}}}},"_meta":{"package":{"name":"apache"},"managed_by":"fleet","managed":true}}}]}' headers: Content-Length: - - "1595" + - "1628" Content-Type: - application/json X-Elastic-Product: - Elasticsearch status: 200 OK code: 200 - duration: 497.616µs + duration: 526.417µs - id: 6 request: proto: HTTP/1.1 @@ -281,9 +281,9 @@ interactions: Authorization: - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) + - go-elasticsearch/7.17.10 (darwin arm64; Go 1.24.2) X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 + - es=7.17.10,go=1.24.2,t=7.17.10,hc=1.24.2 url: https://127.0.0.1:9200/_component_template/logs-apache.access@custom method: GET response: @@ -304,7 +304,7 @@ interactions: - Elasticsearch status: 200 OK code: 200 - duration: 344.216µs + duration: 396.166µs - id: 7 request: proto: HTTP/1.1 @@ -322,9 +322,9 @@ interactions: Authorization: - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) + - go-elasticsearch/7.17.10 (darwin arm64; Go 1.24.2) X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 + - es=7.17.10,go=1.24.2,t=7.17.10,hc=1.24.2 url: https://127.0.0.1:9200/_component_template/metrics-apache.status@settings method: GET response: @@ -345,7 +345,7 @@ interactions: - Elasticsearch status: 200 OK code: 200 - duration: 629.712µs + duration: 457.625µs - id: 8 request: proto: HTTP/1.1 @@ -363,9 +363,9 @@ interactions: Authorization: - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) + - go-elasticsearch/7.17.10 (darwin arm64; Go 1.24.2) X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 + - es=7.17.10,go=1.24.2,t=7.17.10,hc=1.24.2 url: https://127.0.0.1:9200/_component_template/metrics-apache.status@custom method: GET response: @@ -386,7 +386,7 @@ interactions: - Elasticsearch status: 200 OK code: 200 - duration: 438.911µs + duration: 371.125µs - id: 9 request: proto: HTTP/1.1 @@ -404,9 +404,9 @@ interactions: Authorization: - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) + - go-elasticsearch/7.17.10 (darwin arm64; Go 1.24.2) X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 + - es=7.17.10,go=1.24.2,t=7.17.10,hc=1.24.2 url: https://127.0.0.1:9200/_ilm/policy/logs method: GET response: @@ -415,19 +415,19 @@ interactions: proto_minor: 1 transfer_encoding: [] trailer: {} - content_length: 1316 + content_length: 1116 uncompressed: false - body: '{"logs":{"version":1,"modified_date":"2023-11-27T16:35:54.053Z","policy":{"phases":{"hot":{"min_age":"0ms","actions":{"rollover":{"max_primary_shard_size":"50gb","max_age":"30d"}}}},"_meta":{"managed":true,"description":"default policy for the logs index template installed by x-pack"}},"in_use_by":{"indices":[".ds-logs-elastic_agent.metricbeat-default-2023.11.27-000001",".ds-logs-elastic_agent.fleet_server-default-2023.11.27-000001",".ds-logs-elastic_agent.filebeat-default-2023.11.27-000001",".ds-logs-elastic_agent-default-2023.11.27-000001"],"data_streams":["logs-elastic_agent-default","logs-elastic_agent.metricbeat-default","logs-elastic_agent.filebeat-default","logs-elastic_agent.fleet_server-default"],"composable_templates":["logs-apache.access","logs-elastic_agent.cloudbeat","logs-elastic_agent.apm_server","logs-elastic_agent.cloud_defend","logs-system.security","logs-system.auth","logs-elastic_agent.metricbeat","logs-elastic_agent.filebeat","logs-elastic_agent.packetbeat","logs-elastic_agent.filebeat_input","logs-elastic_agent.endpoint_security","logs-elastic_agent.fleet_server","logs-apache.error","logs-system.system","logs-system.application","logs-elastic_agent.osquerybeat","logs-elastic_agent.heartbeat","logs-system.syslog","logs-elastic_agent.auditbeat","logs","logs-elastic_agent"]}}}' + body: '{"logs":{"version":1,"modified_date":"2025-05-08T17:28:20.316Z","policy":{"phases":{"hot":{"min_age":"0ms","actions":{"rollover":{"max_primary_shard_size":"50gb","max_age":"30d"}}}},"_meta":{"managed":true,"description":"default policy for the logs index template installed by x-pack"}},"in_use_by":{"indices":[".ds-logs-elastic_agent-default-2025.05.08-000001",".ds-logs-elastic_agent.fleet_server-default-2025.05.08-000001"],"data_streams":["logs-elastic_agent-default","logs-elastic_agent.fleet_server-default"],"composable_templates":["logs-apache.access","logs-elastic_agent.cloudbeat","logs-elastic_agent.apm_server","logs-elastic_agent.cloud_defend","logs-system.security","logs-system.auth","logs-elastic_agent.metricbeat","logs-elastic_agent.filebeat","logs-elastic_agent.packetbeat","logs-elastic_agent.filebeat_input","logs-elastic_agent.endpoint_security","logs-elastic_agent.fleet_server","logs-apache.error","logs-system.system","logs-system.application","logs-elastic_agent.osquerybeat","logs-elastic_agent.heartbeat","logs-system.syslog","logs-elastic_agent.auditbeat","logs","logs-elastic_agent"]}}}' headers: Content-Length: - - "1316" + - "1116" Content-Type: - application/json X-Elastic-Product: - Elasticsearch status: 200 OK code: 200 - duration: 1.782288ms + duration: 1.268875ms - id: 10 request: proto: HTTP/1.1 @@ -445,9 +445,9 @@ interactions: Authorization: - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) + - go-elasticsearch/7.17.10 (darwin arm64; Go 1.24.2) X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 + - es=7.17.10,go=1.24.2,t=7.17.10,hc=1.24.2 url: https://127.0.0.1:9200/_ilm/policy/metrics method: GET response: @@ -456,19 +456,19 @@ interactions: proto_minor: 1 transfer_encoding: [] trailer: {} - content_length: 2552 + content_length: 1358 uncompressed: false - body: '{"metrics":{"version":1,"modified_date":"2023-11-27T16:35:54.011Z","policy":{"phases":{"hot":{"min_age":"0ms","actions":{"rollover":{"max_primary_shard_size":"50gb","max_age":"30d"}}}},"_meta":{"managed":true,"description":"default policy for the metrics index template installed by x-pack"}},"in_use_by":{"indices":[".ds-metrics-system.process.summary-default-2023.11.27-000001",".ds-metrics-system.fsstat-default-2023.11.27-000001",".ds-metrics-system.uptime-default-2023.11.27-000001",".ds-metrics-system.network-default-2023.11.27-000001",".ds-metrics-system.filesystem-default-2023.11.27-000001",".ds-metrics-elastic_agent.elastic_agent-default-2023.11.27-000001",".ds-metrics-system.socket_summary-default-2023.11.27-000001",".ds-metrics-system.diskio-default-2023.11.27-000001",".ds-metrics-elastic_agent.filebeat-default-2023.11.27-000001",".ds-metrics-system.process-default-2023.11.27-000001",".ds-metrics-system.cpu-default-2023.11.27-000001",".ds-metrics-elastic_agent.metricbeat-default-2023.11.27-000001",".ds-metrics-elastic_agent.fleet_server-default-2023.11.27-000001",".ds-metrics-system.load-default-2023.11.27-000001",".ds-metrics-system.memory-default-2023.11.27-000001"],"data_streams":["metrics-system.filesystem-default","metrics-system.cpu-default","metrics-system.process.summary-default","metrics-system.memory-default","metrics-elastic_agent.fleet_server-default","metrics-system.uptime-default","metrics-elastic_agent.elastic_agent-default","metrics-elastic_agent.metricbeat-default","metrics-system.fsstat-default","metrics-system.process-default","metrics-elastic_agent.filebeat-default","metrics-system.network-default","metrics-system.diskio-default","metrics-system.load-default","metrics-system.socket_summary-default"],"composable_templates":["metrics-system.process","metrics-elastic_agent.packetbeat","metrics-system.fsstat","metrics-elastic_agent.osquerybeat","metrics-elastic_agent.endpoint_security","metrics-elastic_agent.apm_server","metrics-system.memory","metrics-system.socket_summary","metrics-apache.status","metrics-elastic_agent.elastic_agent","metrics-elastic_agent.fleet_server","metrics-system.load","metrics-system.core","metrics-elastic_agent.filebeat","metrics-elastic_agent.filebeat_input","metrics-system.uptime","metrics-system.process.summary","metrics-system.cpu","metrics-elastic_agent.heartbeat","metrics-system.diskio","metrics-elastic_agent.cloudbeat","metrics-elastic_agent.metricbeat","metrics-elastic_agent.auditbeat","metrics-system.network","metrics-system.filesystem","metrics"]}}}' + body: '{"metrics":{"version":1,"modified_date":"2025-05-08T17:28:20.341Z","policy":{"phases":{"hot":{"min_age":"0ms","actions":{"rollover":{"max_primary_shard_size":"50gb","max_age":"30d"}}}},"_meta":{"managed":true,"description":"default policy for the metrics index template installed by x-pack"}},"in_use_by":{"indices":[".ds-metrics-elastic_agent.elastic_agent-default-2025.05.08-000001",".ds-metrics-elastic_agent.fleet_server-default-2025.05.08-000001"],"data_streams":["metrics-elastic_agent.fleet_server-default","metrics-elastic_agent.elastic_agent-default"],"composable_templates":["metrics-system.process","metrics-elastic_agent.packetbeat","metrics-system.fsstat","metrics-elastic_agent.osquerybeat","metrics-elastic_agent.endpoint_security","metrics-elastic_agent.apm_server","metrics-system.memory","metrics-system.socket_summary","metrics-apache.status","metrics-elastic_agent.elastic_agent","metrics-elastic_agent.fleet_server","metrics-system.load","metrics-system.core","metrics-elastic_agent.filebeat","metrics-elastic_agent.filebeat_input","metrics-system.uptime","metrics-system.process.summary","metrics-system.cpu","metrics-elastic_agent.heartbeat","metrics-system.diskio","metrics-elastic_agent.cloudbeat","metrics-elastic_agent.metricbeat","metrics-elastic_agent.auditbeat","metrics-system.network","metrics-system.filesystem","metrics"]}}}' headers: Content-Length: - - "2552" + - "1358" Content-Type: - application/json X-Elastic-Product: - Elasticsearch status: 200 OK code: 200 - duration: 1.727045ms + duration: 1.022125ms - id: 11 request: proto: HTTP/1.1 @@ -486,10 +486,10 @@ interactions: Authorization: - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) + - go-elasticsearch/7.17.10 (darwin arm64; Go 1.24.2) X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 - url: https://127.0.0.1:9200/_ingest/pipeline/logs-apache.error-1.3.6 + - es=7.17.10,go=1.24.2,t=7.17.10,hc=1.24.2 + url: https://127.0.0.1:9200/_ingest/pipeline/logs-apache.error-1.8.2,logs-apache.access-1.8.2,.fleet_final_pipeline-1 method: GET response: proto: HTTP/1.1 @@ -497,19 +497,19 @@ interactions: proto_minor: 1 transfer_encoding: [] trailer: {} - content_length: 3767 + content_length: 12525 uncompressed: false - body: '{"logs-apache.error-1.3.6":{"description":"Pipeline for parsing apache error logs","processors":[{"pipeline":{"if":"ctx.message.startsWith(''{'')","name":"logs-apache.error-1.3.6-third-party"}},{"set":{"field":"event.ingested","value":"{{_ingest.timestamp}}"}},{"set":{"field":"ecs.version","value":"1.12.0"}},{"rename":{"field":"message","target_field":"event.original"}},{"grok":{"field":"event.original","patterns":["\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{LOGLEVEL:log.level}\\]( \\[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\\])? %{GREEDYDATA:message}","\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{DATA:apache.error.module}:%{APACHE_LOGLEVEL:log.level}\\] \\[pid %{NUMBER:process.pid:long}(:tid %{NUMBER:process.thread.id:long})?\\]( \\[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\\])? %{GREEDYDATA:message}"],"pattern_definitions":{"APACHE_LOGLEVEL":"%{LOGLEVEL}[0-9]*","APACHE_TIME":"%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"},"ignore_missing":true}},{"grok":{"field":"message","patterns":["File does not exist: %{URIPATH:file.path}, referer: %{URI:http.request.referrer}","File does not exist: %{URIPATH:file.path}"],"ignore_missing":true,"ignore_failure":true}},{"date":{"if":"ctx.event.timezone == null","field":"apache.error.timestamp","target_field":"@timestamp","formats":["EEE MMM dd H:m:s yyyy","EEE MMM dd H:m:s.SSSSSS yyyy"],"on_failure":[{"append":{"field":"error.message","value":"{{ _ingest.on_failure_message }}"}}]}},{"date":{"if":"ctx.event.timezone != null","field":"apache.error.timestamp","target_field":"@timestamp","formats":["EEE MMM dd H:m:s yyyy","EEE MMM dd H:m:s.SSSSSS yyyy"],"timezone":"{{ event.timezone }}","on_failure":[{"append":{"field":"error.message","value":"{{ _ingest.on_failure_message }}"}}]}},{"remove":{"field":"apache.error.timestamp","ignore_failure":true}},{"set":{"field":"event.kind","value":"event"}},{"set":{"field":"event.category","value":"web"}},{"script":{"if":"ctx?.log?.level != null","lang":"painless","source":"def err_levels = [\"emerg\", \"alert\", \"crit\", \"error\", \"warn\"]; if (err_levels.contains(ctx.log.level)) {\n ctx.event.type = \"error\";\n} else {\n ctx.event.type = \"info\";\n}"}},{"grok":{"field":"source.address","ignore_missing":true,"patterns":["^(%{IP:source.ip}|%{HOSTNAME:source.domain})$"]}},{"geoip":{"field":"source.ip","target_field":"source.geo","ignore_missing":true}},{"geoip":{"database_file":"GeoLite2-ASN.mmdb","field":"source.ip","target_field":"source.as","properties":["asn","organization_name"],"ignore_missing":true}},{"rename":{"field":"source.as.asn","target_field":"source.as.number","ignore_missing":true}},{"rename":{"field":"source.as.organization_name","target_field":"source.as.organization.name","ignore_missing":true}},{"convert":{"field":"source.port","type":"long","ignore_missing":true}},{"script":{"lang":"painless","description":"This script processor iterates over the whole document to remove fields with null values.","source":"void handleMap(Map map) {\n for (def x : map.values()) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n map.values().removeIf(v -> v == null);\n}\nvoid handleList(List list) {\n for (def x : list) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n}\nhandleMap(ctx);\n"}},{"remove":{"field":"event.original","if":"ctx?.tags == null || !(ctx.tags.contains(''preserve_original_event''))","ignore_failure":true,"ignore_missing":true}}],"on_failure":[{"set":{"field":"error.message","value":"{{ _ingest.on_failure_message }}"}}],"_meta":{"managed_by":"fleet","managed":true,"package":{"name":"apache"}}}}' + body: '{"logs-apache.error-1.8.2":{"description":"Pipeline for parsing apache error logs","processors":[{"pipeline":{"if":"ctx.message.startsWith(''{'')","name":"logs-apache.error-1.8.2-third-party"}},{"set":{"field":"event.ingested","value":"{{_ingest.timestamp}}"}},{"set":{"field":"ecs.version","value":"8.5.1"}},{"rename":{"field":"message","target_field":"event.original"}},{"grok":{"field":"event.original","patterns":["\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{LOGLEVEL:log.level}\\]( \\[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\\])? %{GREEDYDATA:message}","\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{DATA:apache.error.module}:%{APACHE_LOGLEVEL:log.level}\\] \\[pid %{NUMBER:process.pid:long}(:tid %{NUMBER:process.thread.id:long})?\\]( \\[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\\])? %{GREEDYDATA:message}"],"pattern_definitions":{"APACHE_LOGLEVEL":"%{LOGLEVEL}[0-9]*","APACHE_TIME":"%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"},"ignore_missing":true}},{"grok":{"field":"message","patterns":["File does not exist: %{URIPATH:file.path}, referer: %{URI:http.request.referrer}","File does not exist: %{URIPATH:file.path}"],"ignore_missing":true,"ignore_failure":true}},{"date":{"if":"ctx.event.timezone == null","field":"apache.error.timestamp","target_field":"@timestamp","formats":["EEE MMM dd H:m:s yyyy","EEE MMM dd H:m:s.SSSSSS yyyy"],"on_failure":[{"append":{"field":"error.message","value":"{{ _ingest.on_failure_message }}"}}]}},{"date":{"if":"ctx.event.timezone != null","field":"apache.error.timestamp","target_field":"@timestamp","formats":["EEE MMM dd H:m:s yyyy","EEE MMM dd H:m:s.SSSSSS yyyy"],"timezone":"{{ event.timezone }}","on_failure":[{"append":{"field":"error.message","value":"{{ _ingest.on_failure_message }}"}}]}},{"remove":{"field":"apache.error.timestamp","ignore_failure":true}},{"set":{"field":"event.kind","value":"event"}},{"set":{"field":"event.category","value":"web"}},{"script":{"if":"ctx?.log?.level != null","lang":"painless","source":"def err_levels = [\"emerg\", \"alert\", \"crit\", \"error\", \"warn\"]; if (err_levels.contains(ctx.log.level)) {\n ctx.event.type = \"error\";\n} else {\n ctx.event.type = \"info\";\n}"}},{"grok":{"field":"source.address","ignore_missing":true,"patterns":["^(%{IP:source.ip}|%{HOSTNAME:source.domain})$"]}},{"geoip":{"field":"source.ip","target_field":"source.geo","ignore_missing":true}},{"geoip":{"database_file":"GeoLite2-ASN.mmdb","field":"source.ip","target_field":"source.as","properties":["asn","organization_name"],"ignore_missing":true}},{"rename":{"field":"source.as.asn","target_field":"source.as.number","ignore_missing":true}},{"rename":{"field":"source.as.organization_name","target_field":"source.as.organization.name","ignore_missing":true}},{"convert":{"field":"source.port","type":"long","ignore_missing":true}},{"script":{"lang":"painless","description":"This script processor iterates over the whole document to remove fields with null values.","source":"void handleMap(Map map) {\n for (def x : map.values()) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n map.values().removeIf(v -> v == null);\n}\nvoid handleList(List list) {\n for (def x : list) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n}\nhandleMap(ctx);\n"}},{"remove":{"field":"event.original","if":"ctx?.tags == null || !(ctx.tags.contains(''preserve_original_event''))","ignore_failure":true,"ignore_missing":true}}],"on_failure":[{"set":{"field":"error.message","value":"{{ _ingest.on_failure_message }}"}}],"_meta":{"managed_by":"fleet","managed":true,"package":{"name":"apache"}}},"logs-apache.access-1.8.2":{"description":"Pipeline for parsing Apache HTTP Server access logs. Requires the geoip and user_agent plugins.","processors":[{"pipeline":{"if":"ctx.message.startsWith(''{'')","name":"logs-apache.access-1.8.2-third-party"}},{"set":{"field":"event.ingested","value":"{{_ingest.timestamp}}"}},{"set":{"field":"ecs.version","value":"8.5.1"}},{"rename":{"field":"message","target_field":"event.original"}},{"grok":{"field":"event.original","patterns":["(%{IPORHOST:destination.domain} )?%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?\" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( \"%{DATA:http.request.referrer}\")?( \"%{DATA:user_agent.original}\")?( X-Forwarded-For=\"%{ADDRESS_LIST:apache.access.remote_addresses}\")?","%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"-\" %{NUMBER:http.response.status_code:long} -","\\[%{HTTPDATE:apache.access.time}\\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol} %{DATA:apache.access.ssl.cipher} \"%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}\" (-|%{NUMBER:http.response.body.bytes:long})"],"ignore_missing":true,"pattern_definitions":{"ADDRESS_LIST":"(%{IP})(\"?,?\\s*(%{IP}))*"}}},{"split":{"field":"apache.access.remote_addresses","separator":"\"?,\\s*","ignore_missing":true}},{"set":{"field":"network.forwarded_ip","value":"{{{apache.access.remote_addresses.0}}}","if":"ctx.apache?.access?.remote_addresses != null && ctx.apache.access.remote_addresses.length > 0"}},{"script":{"if":"ctx.apache?.access?.remote_addresses != null && ctx.apache.access.remote_addresses.length > 0","lang":"painless","tag":"Get source address","description":"Extract from remote_addresses, the first non-private IP to ctx.client.ip","source":"boolean isPrivateCIDR(def ip) {\n CIDR class_a_network = new CIDR(''10.0.0.0/8'');\n CIDR class_b_network = new CIDR(''172.16.0.0/12'');\n CIDR class_c_network = new CIDR(''192.168.0.0/16'');\n\n try {\n return class_a_network.contains(ip) || class_b_network.contains(ip) || class_c_network.contains(ip);\n } catch (IllegalArgumentException e) {\n return false;\n }\n}\ntry {\n if (ctx.client == null) {\n Map map = new HashMap();\n ctx.put(\"client\", map);\n }\n\n def found = false;\n for (def item : ctx.apache.access.remote_addresses) {\n if (!isPrivateCIDR(item)) {\n ctx.client.ip = item;\n found = true;\n break;\n }\n }\n if (!found) {\n ctx.client.ip = ctx.apache.access.remote_addresses[0];\n }\n} catch (Exception e) {\n ctx.client.ip = null;\n}"}},{"append":{"field":"apache.access.remote_addresses","value":["{{source.address}}"],"if":"ctx.source?.address != null"}},{"uri_parts":{"field":"_tmp.url_orig","ignore_failure":true}},{"remove":{"field":["_tmp"],"ignore_missing":true}},{"set":{"field":"url.domain","value":"{{destination.domain}}","if":"ctx.url?.domain == null && ctx.destination?.domain != null"}},{"set":{"field":"event.kind","value":"event"}},{"set":{"field":"event.category","value":"web"}},{"set":{"field":"event.outcome","value":"success","if":"ctx.http?.response?.status_code != null && ctx.http.response.status_code < 400"}},{"set":{"field":"event.outcome","value":"failure","if":"ctx.http?.response?.status_code != null && ctx.http.response.status_code > 399"}},{"grok":{"field":"source.address","ignore_missing":true,"patterns":["^(%{IP:source.ip}|%{HOSTNAME:source.domain})$"]}},{"remove":{"field":"event.created","ignore_missing":true,"ignore_failure":true}},{"rename":{"field":"@timestamp","target_field":"event.created"}},{"date":{"field":"apache.access.time","target_field":"@timestamp","formats":["dd/MMM/yyyy:H:m:s Z"],"ignore_failure":true}},{"remove":{"field":"apache.access.time","ignore_failure":true}},{"user_agent":{"field":"user_agent.original","ignore_failure":true}},{"geoip":{"field":"source.ip","target_field":"source.geo","ignore_missing":true}},{"geoip":{"database_file":"GeoLite2-ASN.mmdb","field":"source.ip","target_field":"source.as","properties":["asn","organization_name"],"ignore_missing":true}},{"rename":{"field":"source.as.asn","target_field":"source.as.number","ignore_missing":true}},{"rename":{"field":"source.as.organization_name","target_field":"source.as.organization.name","ignore_missing":true}},{"set":{"field":"tls.cipher","value":"{{apache.access.ssl.cipher}}","if":"ctx.apache?.access?.ssl?.cipher != null"}},{"script":{"lang":"painless","if":"ctx.apache?.access?.ssl?.protocol != null","source":"def parts = ctx.apache.access.ssl.protocol.toLowerCase().splitOnToken(\"v\"); if (parts.length != 2) {\n return;\n} if (parts[1].contains(\".\")) {\n ctx.tls.version = parts[1];\n} else {\n ctx.tls.version = parts[1] + \".0\";\n} ctx.tls.version_protocol = parts[0];"}},{"script":{"lang":"painless","description":"This script processor iterates over the whole document to remove fields with null values.","source":"void handleMap(Map map) {\n for (def x : map.values()) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n map.values().removeIf(v -> v == null);\n}\nvoid handleList(List list) {\n for (def x : list) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n}\nhandleMap(ctx);\n"}},{"remove":{"field":"event.original","if":"ctx.tags == null || !(ctx.tags.contains(''preserve_original_event''))","ignore_failure":true,"ignore_missing":true}}],"on_failure":[{"set":{"field":"error.message","value":"{{ _ingest.on_failure_message }}"}}],"_meta":{"managed_by":"fleet","managed":true,"package":{"name":"apache"}}},".fleet_final_pipeline-1":{"version":2,"_meta":{"managed_by":"fleet","managed":true},"description":"Final pipeline for processing all incoming Fleet Agent documents.\n","processors":[{"set":{"description":"Add time when event was ingested.","field":"event.ingested","copy_from":"_ingest.timestamp"}},{"script":{"description":"Remove sub-seconds from event.ingested to improve storage efficiency.","tag":"truncate-subseconds-event-ingested","source":"ctx.event.ingested = ctx.event.ingested.withNano(0).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);","ignore_failure":true}},{"remove":{"description":"Remove any pre-existing untrusted values.","field":["event.agent_id_status","_security"],"ignore_missing":true}},{"set_security_user":{"field":"_security","properties":["authentication_type","username","realm","api_key"]}},{"script":{"description":"Add event.agent_id_status based on the API key metadata and the agent.id contained in the event.\n","tag":"agent-id-status","source":"boolean is_user_trusted(def ctx, def users) {\n if (ctx?._security?.username == null) {\n return false;\n }\n\n def user = null;\n for (def item : users) {\n if (item?.username == ctx._security.username) {\n user = item;\n break;\n }\n }\n\n if (user == null || user?.realm == null || ctx?._security?.realm?.name == null) {\n return false;\n }\n\n if (ctx._security.realm.name != user.realm) {\n return false;\n }\n\n return true;\n}\n\nString verified(def ctx, def params) {\n // No agent.id field to validate.\n if (ctx?.agent?.id == null) {\n return \"missing\";\n }\n\n // Check auth metadata from API key.\n if (ctx?._security?.authentication_type == null\n // Agents only use API keys.\n || ctx._security.authentication_type != ''API_KEY''\n // Verify the API key owner before trusting any metadata it contains.\n || !is_user_trusted(ctx, params.trusted_users)\n // Verify the API key has metadata indicating the assigned agent ID.\n || ctx?._security?.api_key?.metadata?.agent_id == null) {\n return \"auth_metadata_missing\";\n }\n\n // The API key can only be used represent the agent.id it was issued to.\n if (ctx._security.api_key.metadata.agent_id != ctx.agent.id) {\n // Potential masquerade attempt.\n return \"mismatch\";\n }\n\n return \"verified\";\n}\n\nif (ctx?.event == null) {\n ctx.event = [:];\n}\n\nctx.event.agent_id_status = verified(ctx, params);","params":{"trusted_users":[{"username":"elastic/fleet-server","realm":"_service_account"},{"username":"cloud-internal-agent-server","realm":"found"},{"username":"elastic","realm":"reserved"}]}}},{"remove":{"field":"_security","ignore_missing":true}}],"on_failure":[{"remove":{"field":"_security","ignore_missing":true,"ignore_failure":true}},{"append":{"field":"error.message","value":["failed in Fleet agent final_pipeline: {{ _ingest.on_failure_message }}"]}}]}}' headers: Content-Length: - - "3767" + - "12525" Content-Type: - application/json X-Elastic-Product: - Elasticsearch status: 200 OK code: 200 - duration: 1.026301ms + duration: 1.314ms - id: 12 request: proto: HTTP/1.1 @@ -527,10 +527,10 @@ interactions: Authorization: - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) + - go-elasticsearch/7.17.10 (darwin arm64; Go 1.24.2) X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 - url: https://127.0.0.1:9200/_ingest/pipeline/logs-apache.access-1.3.6 + - es=7.17.10,go=1.24.2,t=7.17.10,hc=1.24.2 + url: https://127.0.0.1:9200/_ingest/pipeline/logs-apache.error-1.8.2-third-party,logs-apache.access-1.8.2-third-party method: GET response: proto: HTTP/1.1 @@ -538,19 +538,19 @@ interactions: proto_minor: 1 transfer_encoding: [] trailer: {} - content_length: 4574 + content_length: 2242 uncompressed: false - body: '{"logs-apache.access-1.3.6":{"description":"Pipeline for parsing Apache HTTP Server access logs. Requires the geoip and user_agent plugins.","processors":[{"pipeline":{"if":"ctx.message.startsWith(''{'')","name":"logs-apache.access-1.3.6-third-party"}},{"set":{"field":"event.ingested","value":"{{_ingest.timestamp}}"}},{"set":{"field":"ecs.version","value":"1.12.0"}},{"rename":{"field":"message","target_field":"event.original"}},{"grok":{"field":"event.original","patterns":["%{IPORHOST:destination.domain} %{IPORHOST:source.ip} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?\" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( \"%{DATA:http.request.referrer}\")?( \"%{DATA:user_agent.original}\")?","%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?\" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( \"%{DATA:http.request.referrer}\")?( \"%{DATA:user_agent.original}\")?","%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"-\" %{NUMBER:http.response.status_code:long} -","\\[%{HTTPDATE:apache.access.time}\\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol} %{DATA:apache.access.ssl.cipher} \"%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}\" (-|%{NUMBER:http.response.body.bytes:long})"],"ignore_missing":true}},{"uri_parts":{"field":"_tmp.url_orig","ignore_failure":true}},{"remove":{"field":["_tmp"],"ignore_missing":true}},{"set":{"field":"url.domain","value":"{{destination.domain}}","if":"ctx.url?.domain == null && ctx.destination?.domain != null"}},{"set":{"field":"event.kind","value":"event"}},{"set":{"field":"event.category","value":"web"}},{"set":{"field":"event.outcome","value":"success","if":"ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400"}},{"set":{"field":"event.outcome","value":"failure","if":"ctx?.http?.response?.status_code != null && ctx.http.response.status_code > 399"}},{"grok":{"field":"source.address","ignore_missing":true,"patterns":["^(%{IP:source.ip}|%{HOSTNAME:source.domain})$"]}},{"remove":{"field":"event.created","ignore_missing":true,"ignore_failure":true}},{"rename":{"field":"@timestamp","target_field":"event.created"}},{"date":{"field":"apache.access.time","target_field":"@timestamp","formats":["dd/MMM/yyyy:H:m:s Z"],"ignore_failure":true}},{"remove":{"field":"apache.access.time","ignore_failure":true}},{"user_agent":{"field":"user_agent.original","ignore_failure":true}},{"geoip":{"field":"source.ip","target_field":"source.geo","ignore_missing":true}},{"geoip":{"database_file":"GeoLite2-ASN.mmdb","field":"source.ip","target_field":"source.as","properties":["asn","organization_name"],"ignore_missing":true}},{"rename":{"field":"source.as.asn","target_field":"source.as.number","ignore_missing":true}},{"rename":{"field":"source.as.organization_name","target_field":"source.as.organization.name","ignore_missing":true}},{"set":{"field":"tls.cipher","value":"{{apache.access.ssl.cipher}}","if":"ctx?.apache?.access?.ssl?.cipher != null"}},{"script":{"lang":"painless","if":"ctx?.apache?.access?.ssl?.protocol != null","source":"def parts = ctx.apache.access.ssl.protocol.toLowerCase().splitOnToken(\"v\"); if (parts.length != 2) {\n return;\n} if (parts[1].contains(\".\")) {\n ctx.tls.version = parts[1];\n} else {\n ctx.tls.version = parts[1] + \".0\";\n} ctx.tls.version_protocol = parts[0];"}},{"script":{"lang":"painless","description":"This script processor iterates over the whole document to remove fields with null values.","source":"void handleMap(Map map) {\n for (def x : map.values()) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n map.values().removeIf(v -> v == null);\n}\nvoid handleList(List list) {\n for (def x : list) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n}\nhandleMap(ctx);\n"}},{"remove":{"field":"event.original","if":"ctx?.tags == null || !(ctx.tags.contains(''preserve_original_event''))","ignore_failure":true,"ignore_missing":true}}],"on_failure":[{"set":{"field":"error.message","value":"{{ _ingest.on_failure_message }}"}}],"_meta":{"managed_by":"fleet","managed":true,"package":{"name":"apache"}}}}' + body: '{"logs-apache.error-1.8.2-third-party":{"description":"Pipeline for parsing Apache HTTP Server logs from third party api","processors":[{"json":{"field":"message","target_field":"json"}},{"drop":{"if":"ctx.json?.result == null"}},{"fingerprint":{"fields":["json.result._cd","json.result._indextime","json.result._raw","json.result._time","json.result.host","json.result.source"],"target_field":"_id","ignore_missing":true}},{"set":{"copy_from":"json.result._raw","field":"message","ignore_empty_value":true}},{"set":{"copy_from":"json.result.host","field":"host.name","ignore_empty_value":true}},{"set":{"copy_from":"json.result.source","field":"file.path","ignore_empty_value":true}},{"remove":{"field":["json"],"ignore_missing":true}}],"on_failure":[{"append":{"field":"error.message","value":"error in third-party pipeline: error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}} with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}} {{ _ingest.on_failure_message }}"}}],"_meta":{"managed_by":"fleet","managed":true,"package":{"name":"apache"}}},"logs-apache.access-1.8.2-third-party":{"description":"Pipeline for parsing Apache HTTP Server logs from third party api","processors":[{"json":{"field":"message","target_field":"json"}},{"drop":{"if":"ctx.json?.result == null"}},{"fingerprint":{"fields":["json.result._cd","json.result._indextime","json.result._raw","json.result._time","json.result.host","json.result.source"],"target_field":"_id","ignore_missing":true}},{"set":{"copy_from":"json.result._raw","field":"message","ignore_empty_value":true}},{"set":{"copy_from":"json.result.host","field":"host.name","ignore_empty_value":true}},{"set":{"copy_from":"json.result.source","field":"file.path","ignore_empty_value":true}},{"remove":{"field":["json"],"ignore_missing":true}}],"on_failure":[{"append":{"field":"error.message","value":"error in third-party pipeline: error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}} with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}} {{ _ingest.on_failure_message }}"}}],"_meta":{"managed_by":"fleet","managed":true,"package":{"name":"apache"}}}}' headers: Content-Length: - - "4574" + - "2242" Content-Type: - application/json X-Elastic-Product: - Elasticsearch status: 200 OK code: 200 - duration: 850.163µs + duration: 736.75µs - id: 13 request: proto: HTTP/1.1 @@ -568,132 +568,9 @@ interactions: Authorization: - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) + - go-elasticsearch/7.17.10 (darwin arm64; Go 1.24.2) X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 - url: https://127.0.0.1:9200/_ingest/pipeline/.fleet_final_pipeline-1 - method: GET - response: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - transfer_encoding: [] - trailer: {} - content_length: 2911 - uncompressed: false - body: '{".fleet_final_pipeline-1":{"version":2,"_meta":{"managed_by":"fleet","managed":true},"description":"Final pipeline for processing all incoming Fleet Agent documents.\n","processors":[{"set":{"description":"Add time when event was ingested.","field":"event.ingested","copy_from":"_ingest.timestamp"}},{"script":{"description":"Remove sub-seconds from event.ingested to improve storage efficiency.","tag":"truncate-subseconds-event-ingested","source":"ctx.event.ingested = ctx.event.ingested.withNano(0).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);","ignore_failure":true}},{"remove":{"description":"Remove any pre-existing untrusted values.","field":["event.agent_id_status","_security"],"ignore_missing":true}},{"set_security_user":{"field":"_security","properties":["authentication_type","username","realm","api_key"]}},{"script":{"description":"Add event.agent_id_status based on the API key metadata and the agent.id contained in the event.\n","tag":"agent-id-status","source":"boolean is_user_trusted(def ctx, def users) {\n if (ctx?._security?.username == null) {\n return false;\n }\n\n def user = null;\n for (def item : users) {\n if (item?.username == ctx._security.username) {\n user = item;\n break;\n }\n }\n\n if (user == null || user?.realm == null || ctx?._security?.realm?.name == null) {\n return false;\n }\n\n if (ctx._security.realm.name != user.realm) {\n return false;\n }\n\n return true;\n}\n\nString verified(def ctx, def params) {\n // No agent.id field to validate.\n if (ctx?.agent?.id == null) {\n return \"missing\";\n }\n\n // Check auth metadata from API key.\n if (ctx?._security?.authentication_type == null\n // Agents only use API keys.\n || ctx._security.authentication_type != ''API_KEY''\n // Verify the API key owner before trusting any metadata it contains.\n || !is_user_trusted(ctx, params.trusted_users)\n // Verify the API key has metadata indicating the assigned agent ID.\n || ctx?._security?.api_key?.metadata?.agent_id == null) {\n return \"auth_metadata_missing\";\n }\n\n // The API key can only be used represent the agent.id it was issued to.\n if (ctx._security.api_key.metadata.agent_id != ctx.agent.id) {\n // Potential masquerade attempt.\n return \"mismatch\";\n }\n\n return \"verified\";\n}\n\nif (ctx?.event == null) {\n ctx.event = [:];\n}\n\nctx.event.agent_id_status = verified(ctx, params);","params":{"trusted_users":[{"username":"elastic/fleet-server","realm":"_service_account"},{"username":"cloud-internal-agent-server","realm":"found"},{"username":"elastic","realm":"reserved"}]}}},{"remove":{"field":"_security","ignore_missing":true}}],"on_failure":[{"remove":{"field":"_security","ignore_missing":true,"ignore_failure":true}},{"append":{"field":"error.message","value":["failed in Fleet agent final_pipeline: {{ _ingest.on_failure_message }}"]}}]}}' - headers: - Content-Length: - - "2911" - Content-Type: - - application/json - X-Elastic-Product: - - Elasticsearch - status: 200 OK - code: 200 - duration: 718.117µs - - id: 14 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: "" - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Authorization: - - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== - User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) - X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 - url: https://127.0.0.1:9200/_ingest/pipeline/logs-apache.error-1.3.6-third-party - method: GET - response: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - transfer_encoding: [] - trailer: {} - content_length: 1121 - uncompressed: false - body: '{"logs-apache.error-1.3.6-third-party":{"description":"Pipeline for parsing Apache HTTP Server logs from third party api","processors":[{"json":{"field":"message","target_field":"json"}},{"drop":{"if":"ctx.json?.result == null"}},{"fingerprint":{"fields":["json.result._cd","json.result._indextime","json.result._raw","json.result._time","json.result.host","json.result.source"],"target_field":"_id","ignore_missing":true}},{"set":{"copy_from":"json.result._raw","field":"message","ignore_empty_value":true}},{"set":{"copy_from":"json.result.host","field":"host.name","ignore_empty_value":true}},{"set":{"copy_from":"json.result.source","field":"file.path","ignore_empty_value":true}},{"remove":{"field":["json"],"ignore_missing":true}}],"on_failure":[{"append":{"field":"error.message","value":"error in third-party pipeline: error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}} with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}} {{ _ingest.on_failure_message }}"}}],"_meta":{"managed_by":"fleet","managed":true,"package":{"name":"apache"}}}}' - headers: - Content-Length: - - "1121" - Content-Type: - - application/json - X-Elastic-Product: - - Elasticsearch - status: 200 OK - code: 200 - duration: 746.962µs - - id: 15 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: "" - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Authorization: - - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== - User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) - X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 - url: https://127.0.0.1:9200/_ingest/pipeline/logs-apache.access-1.3.6-third-party - method: GET - response: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - transfer_encoding: [] - trailer: {} - content_length: 1122 - uncompressed: false - body: '{"logs-apache.access-1.3.6-third-party":{"description":"Pipeline for parsing Apache HTTP Server logs from third party api","processors":[{"json":{"field":"message","target_field":"json"}},{"drop":{"if":"ctx.json?.result == null"}},{"fingerprint":{"fields":["json.result._cd","json.result._indextime","json.result._raw","json.result._time","json.result.host","json.result.source"],"target_field":"_id","ignore_missing":true}},{"set":{"copy_from":"json.result._raw","field":"message","ignore_empty_value":true}},{"set":{"copy_from":"json.result.host","field":"host.name","ignore_empty_value":true}},{"set":{"copy_from":"json.result.source","field":"file.path","ignore_empty_value":true}},{"remove":{"field":["json"],"ignore_missing":true}}],"on_failure":[{"append":{"field":"error.message","value":"error in third-party pipeline: error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}} with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}} {{ _ingest.on_failure_message }}"}}],"_meta":{"managed_by":"fleet","managed":true,"package":{"name":"apache"}}}}' - headers: - Content-Length: - - "1122" - Content-Type: - - application/json - X-Elastic-Product: - - Elasticsearch - status: 200 OK - code: 200 - duration: 614.104µs - - id: 16 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: "" - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Authorization: - - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== - User-Agent: - - go-elasticsearch/7.17.10 (linux amd64; Go 1.21.3) - X-Elastic-Client-Meta: - - es=7.17.10,go=1.21.3,t=7.17.10,hc=1.21.3 + - es=7.17.10,go=1.24.2,t=7.17.10,hc=1.24.2 url: https://127.0.0.1:9200/_ml/trained_models/apache_*?decompress_definition=false&include=definition%2Cfeature_importance_baseline%2Chyperparameters%2Ctotal_feature_importance method: GET response: @@ -714,4 +591,4 @@ interactions: - Elasticsearch status: 200 OK code: 200 - duration: 1.012949ms + duration: 1.0935ms diff --git a/internal/elasticsearch/client_test.go b/internal/elasticsearch/client_test.go index 40ccdcc135..1b44e65076 100644 --- a/internal/elasticsearch/client_test.go +++ b/internal/elasticsearch/client_test.go @@ -79,7 +79,7 @@ func TestClusterHealth(t *testing.T) { for _, c := range cases { t.Run(c.Record, func(t *testing.T) { - client := test.NewClient(t, c.Record) + client := test.NewClient(t, c.Record, nil) err := client.CheckHealth(context.Background()) if c.Expected != "" { @@ -94,7 +94,7 @@ func TestClusterHealth(t *testing.T) { } func TestClusterInfo(t *testing.T) { - client := test.NewClient(t, "./testdata/elasticsearch-9-info") + client := test.NewClient(t, "./testdata/elasticsearch-9-info", nil) info, err := client.Info(context.Background()) require.NoError(t, err) assert.Equal(t, "9.0.0-SNAPSHOT", info.Version.Number) diff --git a/internal/elasticsearch/ingest/pipeline.go b/internal/elasticsearch/ingest/pipeline.go index 660717c80c..fafe0062bb 100644 --- a/internal/elasticsearch/ingest/pipeline.go +++ b/internal/elasticsearch/ingest/pipeline.go @@ -11,6 +11,8 @@ import ( "fmt" "io" "net/http" + "slices" + "sort" "strings" "gopkg.in/yaml.v3" @@ -73,6 +75,169 @@ func (p *Pipeline) MarshalJSON() (asJSON []byte, err error) { return asJSON, nil } +// RemotePipeline represents resource retrieved from Elasticsearch +type RemotePipeline struct { + Processors []struct { + Pipeline *struct { + Name string `json:"name"` + } `json:"pipeline,omitempty"` + } `json:"processors"` + id string + raw []byte +} + +// Name returns the name of the ingest pipeline. +func (p RemotePipeline) Name() string { + return p.id +} + +// JSON returns the JSON representation of the ingest pipeline. +func (p RemotePipeline) JSON() []byte { + return p.raw +} + +func (p RemotePipeline) GetProcessorPipelineNames() []string { + var names []string + for _, processor := range p.Processors { + if processor.Pipeline == nil { + continue + } + name := processor.Pipeline.Name + if slices.Contains(names, name) { + continue + } + names = append(names, name) + } + return names +} + +func GetRemotePipelineNames(ctx context.Context, api *elasticsearch.API) ([]string, error) { + resp, err := api.Ingest.GetPipeline( + api.Ingest.GetPipeline.WithContext(ctx), + api.Ingest.GetPipeline.WithSummary(true), + ) + + if err != nil { + return nil, fmt.Errorf("failed to get ingest pipeline names: %w", err) + } + defer resp.Body.Close() + + if resp.IsError() { + return nil, fmt.Errorf("error getting ingest pipeline names: %s", resp) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("error reading ingest pipeline names body: %w", err) + } + + pipelineMap := map[string]struct { + Description string `json:"description"` + }{} + + if err := json.Unmarshal(body, &pipelineMap); err != nil { + return nil, fmt.Errorf("error unmarshaling ingest pipeline names: %w", err) + } + + pipelineNames := []string{} + + for name := range pipelineMap { + pipelineNames = append(pipelineNames, name) + } + + sort.Slice(pipelineNames, func(i, j int) bool { + return sort.StringsAreSorted([]string{strings.ToLower(pipelineNames[i]), strings.ToLower(pipelineNames[j])}) + }) + + return pipelineNames, nil +} + +func GetRemotePipelines(ctx context.Context, api *elasticsearch.API, ids ...string) ([]RemotePipeline, error) { + + commaSepIDs := strings.Join(ids, ",") + + resp, err := api.Ingest.GetPipeline( + api.Ingest.GetPipeline.WithContext(ctx), + api.Ingest.GetPipeline.WithPipelineID(commaSepIDs), + ) + + if err != nil { + return nil, fmt.Errorf("failed to get ingest pipelines: %w", err) + } + defer resp.Body.Close() + + // Ingest templates referenced by other templates may not exist. + if resp.StatusCode == http.StatusNotFound { + return nil, nil + } + + if resp.IsError() { + return nil, fmt.Errorf("failed to get ingest pipelines %s: %s", ids, resp.String()) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + pipelinesResponse := map[string]json.RawMessage{} + if err := json.Unmarshal(body, &pipelinesResponse); err != nil { + return nil, fmt.Errorf("failed to decode response: %w", err) + } + + var pipelines []RemotePipeline + for id, raw := range pipelinesResponse { + var pipeline RemotePipeline + err := json.Unmarshal(raw, &pipeline) + if err != nil { + return nil, fmt.Errorf("failed to decode pipeline %s: %w", id, err) + } + pipeline.id = id + pipeline.raw = raw + pipelines = append(pipelines, pipeline) + } + + return pipelines, nil +} + +func GetRemotePipelinesWithNested(ctx context.Context, api *elasticsearch.API, ids ...string) ([]RemotePipeline, error) { + var pipelines []RemotePipeline + var collected []string + pending := ids + for len(pending) > 0 { + resultPipelines, err := GetRemotePipelines(ctx, api, pending...) + if err != nil { + return nil, err + } + pipelines = append(pipelines, resultPipelines...) + collected = append(collected, pending...) + pending = pendingNestedPipelines(pipelines, collected) + } + + return pipelines, nil +} + +func pendingNestedPipelines(pipelines []RemotePipeline, collected []string) []string { + var names []string + for _, p := range pipelines { + for _, processor := range p.Processors { + if processor.Pipeline == nil { + continue + } + name := processor.Pipeline.Name + + if slices.Contains(collected, name) { + continue + } + if slices.Contains(names, name) { + continue + } + names = append(names, name) + } + } + return names +} + func SimulatePipeline(ctx context.Context, api *elasticsearch.API, pipelineName string, events []json.RawMessage, simulateDataStream string) ([]json.RawMessage, error) { var request simulatePipelineRequest for _, event := range events { diff --git a/internal/elasticsearch/test/httptest.go b/internal/elasticsearch/test/httptest.go index f4859bd405..0883dc22b3 100644 --- a/internal/elasticsearch/test/httptest.go +++ b/internal/elasticsearch/test/httptest.go @@ -22,7 +22,7 @@ import ( // responses. If responses are not found, it forwards the query to the server started by // elastic-package stack, and records the response. // Responses are recorded in the directory indicated by serverDataDir. -func NewClient(t *testing.T, recordFileName string) *elasticsearch.Client { +func NewClient(t *testing.T, recordFileName string, matcher cassette.MatcherFunc) *elasticsearch.Client { options, err := clientOptionsForRecord(recordFileName) require.NoError(t, err) @@ -35,6 +35,11 @@ func NewClient(t *testing.T, recordFileName string) *elasticsearch.Client { SkipRequestLatency: true, RealTransport: config.Transport, }) + + if matcher != nil { + rec.SetMatcher(matcher) + } + require.NoError(t, err) config.Transport = rec diff --git a/internal/export/ingest_pipelines.go b/internal/export/ingest_pipelines.go new file mode 100644 index 0000000000..0ee4d0ff9a --- /dev/null +++ b/internal/export/ingest_pipelines.go @@ -0,0 +1,136 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package export + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + + "gopkg.in/yaml.v3" + + "github.com/elastic/elastic-package/internal/elasticsearch" + "github.com/elastic/elastic-package/internal/elasticsearch/ingest" +) + +type PipelineWriteLocationType string + +const ( + PipelineWriteLocationTypeDataStream PipelineWriteLocationType = "data_stream" + PipelineWriteLocationTypeRoot PipelineWriteLocationType = "root" +) + +// Represents a target write location for exporting an ingest pipeline +type PipelineWriteLocation struct { + Type PipelineWriteLocationType + Name string + ParentPath string +} + +func (p PipelineWriteLocation) WritePath() string { + return filepath.Join(p.ParentPath, "elasticsearch", "ingest_pipeline") +} + +type PipelineWriteAssignments map[string]PipelineWriteLocation + +func IngestPipelines(ctx context.Context, api *elasticsearch.API, writeAssignments PipelineWriteAssignments) error { + var pipelineIDs []string + + for pipelineID := range writeAssignments { + pipelineIDs = append(pipelineIDs, pipelineID) + } + + pipelines, err := ingest.GetRemotePipelinesWithNested(ctx, api, pipelineIDs...) + + if err != nil { + return fmt.Errorf("exporting ingest pipelines using Elasticsearch failed: %w", err) + } + + pipelineLookup := make(map[string]ingest.RemotePipeline) + for _, pipeline := range pipelines { + pipelineLookup[pipeline.Name()] = pipeline + } + + err = writePipelinesToFiles(writeAssignments, pipelineLookup) + + if err != nil { + return err + } + + return nil +} + +func writePipelinesToFiles(writeAssignments PipelineWriteAssignments, pipelineLookup map[string]ingest.RemotePipeline) error { + if len(writeAssignments) == 0 { + return nil + } + + for name, writeLocation := range writeAssignments { + pipeline, ok := pipelineLookup[name] + if !ok { + continue + } + err := writePipelineToFile(pipeline, writeLocation) + if err != nil { + return err + } + + depPipelineWriteAssignments := createWriteAssignments(writeLocation, pipeline.GetProcessorPipelineNames()) + err = writePipelinesToFiles(depPipelineWriteAssignments, pipelineLookup) + if err != nil { + return err + } + } + + return nil +} + +func writePipelineToFile(pipeline ingest.RemotePipeline, writeLocation PipelineWriteLocation) error { + var jsonPipeline map[string]any + err := json.Unmarshal(pipeline.JSON(), &jsonPipeline) + if err != nil { + return fmt.Errorf("unmarshalling ingest pipeline failed (ID: %s): %w", pipeline.Name(), err) + } + + delete(jsonPipeline, "_meta") + delete(jsonPipeline, "version") + + var documentBytes bytes.Buffer + // requirement: https://github.com/elastic/package-spec/pull/54 + documentBytes.WriteString("---\n") + yamlEncoder := yaml.NewEncoder(&documentBytes) + yamlEncoder.SetIndent(2) + err = yamlEncoder.Encode(jsonPipeline) + if err != nil { + return fmt.Errorf("marshalling ingest pipeline json to yaml failed (ID: %s): %w", pipeline.Name(), err) + } + + err = os.MkdirAll(writeLocation.WritePath(), 0755) + if err != nil { + return fmt.Errorf("creating target directory failed (path: %s): %w", writeLocation.WritePath(), err) + } + + pipelineFilePath := filepath.Join(writeLocation.WritePath(), pipeline.Name()+".yml") + + err = os.WriteFile(pipelineFilePath, documentBytes.Bytes(), 0644) + + if err != nil { + return fmt.Errorf("writing to file '%s' failed: %w", pipelineFilePath, err) + } + + return nil +} + +func createWriteAssignments(writeLocation PipelineWriteLocation, pipelineNames []string) PipelineWriteAssignments { + writeAssignments := make(PipelineWriteAssignments) + for _, pipelineName := range pipelineNames { + writeAssignments[pipelineName] = writeLocation + } + + return writeAssignments +} diff --git a/internal/export/ingest_pipelines_test.go b/internal/export/ingest_pipelines_test.go new file mode 100644 index 0000000000..8ce72f2901 --- /dev/null +++ b/internal/export/ingest_pipelines_test.go @@ -0,0 +1,194 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package export + +import ( + "context" + "errors" + "io/fs" + "net/http" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "gopkg.in/dnaeon/go-vcr.v3/cassette" + "gopkg.in/yaml.v3" + + "github.com/elastic/elastic-package/internal/stack" + + estest "github.com/elastic/elastic-package/internal/elasticsearch/test" +) + +func TestExportIngestPipelines(t *testing.T) { + // Files for each suite are recorded automatically on first test run. + // To add a new suite: + // - Configure it here. + // - Install the package in a running stack. + // - Configure environment variables for this stack (eval "$(elastic-package stack shellinit)"). + // - Run tests. + // - Check that recorded files make sense and commit them. + // To update a suite: + // - Reproduce the scenario as described in the comments. + // - Remove the files that you want to update. + // - Follow the same steps to create a new suite. + // - Check if the changes are the expected ones and commit them. + suites := []*ingestPipelineExportSuite{ + // To reproduce the scenario: + // - Start the stack with version 8.17.4. + // - Install apache package (2.0.0). + // - Install dga package (2.3.0). + &ingestPipelineExportSuite{ + PipelineIds: []string{ + "logs-apache.access-2.0.0", + "2.3.0-ml_dga_ingest_pipeline", + }, + ExportDir: "./testdata/elasticsearch-8-export-pipelines", + RecordDir: "./testdata/elasticsearch-8-mock-export-pipelines", + Matcher: ingestPipelineRequestMatcher, + }, + } + + for _, s := range suites { + suite.Run(t, s) + } +} + +type ingestPipelineExportSuite struct { + suite.Suite + + PipelineIds []string + ExportDir string + RecordDir string + Matcher cassette.MatcherFunc +} + +func (s *ingestPipelineExportSuite) SetupTest() { + _, err := os.Stat(s.ExportDir) + if errors.Is(err, os.ErrNotExist) { + client, err := stack.NewElasticsearchClient() + s.Require().NoError(err) + + writeAssignments := createTestWriteAssignments(s.PipelineIds, s.ExportDir) + + err = IngestPipelines((context.Background()), client.API, writeAssignments) + + s.Require().NoError(err) + } else { + s.Require().NoError(err) + } +} + +func (s *ingestPipelineExportSuite) TestExportPipelines() { + client := estest.NewClient(s.T(), s.RecordDir, s.Matcher) + + outputDir := s.T().TempDir() + writeAssignments := createTestWriteAssignments(s.PipelineIds, outputDir) + err := IngestPipelines(context.Background(), client.API, writeAssignments) + s.Require().NoError(err) + + filesExpected := countFiles(s.T(), s.ExportDir) + filesExported := countFiles(s.T(), outputDir) + s.Assert().Equal(filesExpected, filesExported) + + filesFound := countFiles(s.T(), outputDir) + s.Assert().Equal(filesExpected, filesFound) + + assertEqualExports(s.T(), s.ExportDir, outputDir) +} + +func createTestWriteAssignments(pipelineIDs []string, outputDir string) PipelineWriteAssignments { + writeAssignments := make(PipelineWriteAssignments) + + for _, pipelineID := range pipelineIDs { + writeAssignments[pipelineID] = PipelineWriteLocation{ + Type: PipelineWriteLocationTypeRoot, + Name: pipelineID, + ParentPath: outputDir, + } + } + + return writeAssignments +} + +func countFiles(t *testing.T, dir string) (count int) { + t.Helper() + err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + count++ + return nil + }) + require.NoError(t, err) + return count +} + +func assertEqualExports(t *testing.T, expectedDir, resultDir string) { + t.Helper() + err := filepath.WalkDir(expectedDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + + t.Run(path, func(t *testing.T) { + relPath, err := filepath.Rel(expectedDir, path) + require.NoError(t, err) + + assertEquivalentYAML(t, path, filepath.Join(resultDir, relPath)) + }) + return nil + }) + require.NoError(t, err) +} + +func assertEquivalentYAML(t *testing.T, expectedPath, foundPath string) { + t.Helper() + readYAML := func(p string) map[string]interface{} { + d, err := os.ReadFile(p) + require.NoError(t, err) + var o map[string]interface{} + err = yaml.Unmarshal(d, &o) + require.NoError(t, err) + return o + } + + expected := readYAML(expectedPath) + found := readYAML(foundPath) + assert.EqualValues(t, expected, found) +} + +// Ingest Pipelines are requested in bulk and the param values are non-deterministic, +// which makes matching to recorded requests flaky. +// This custom cassette matcher helps match pipeline requests, and sends all others to the default matcher. +func ingestPipelineRequestMatcher(r *http.Request, cr cassette.Request) bool { + urlStartPattern := "https://127.0.0.1:9200/_ingest/pipeline/" + rSplitUrl := strings.Split(r.URL.String(), urlStartPattern) + crSplitUrl := strings.Split(cr.URL, urlStartPattern) + + isURLsPattern := len(rSplitUrl) == 2 && len(crSplitUrl) == 2 + + if !isURLsPattern { + return cassette.DefaultMatcher(r, cr) + } + + rPipelineValues := strings.Split(rSplitUrl[1], ",") + crPipelineValues := strings.Split(crSplitUrl[1], ",") + + slices.Sort(rPipelineValues) + slices.Sort(crPipelineValues) + + return slices.Equal(rPipelineValues, crPipelineValues) +} diff --git a/internal/export/testdata/elasticsearch-8-export-pipelines/elasticsearch/ingest_pipeline/2.3.0-ml_dga_inference_pipeline.yml b/internal/export/testdata/elasticsearch-8-export-pipelines/elasticsearch/ingest_pipeline/2.3.0-ml_dga_inference_pipeline.yml new file mode 100644 index 0000000000..b872429c96 --- /dev/null +++ b/internal/export/testdata/elasticsearch-8-export-pipelines/elasticsearch/ingest_pipeline/2.3.0-ml_dga_inference_pipeline.yml @@ -0,0 +1,108 @@ +--- +on_failure: + - set: + field: event.kind + value: pipeline_error + - append: + field: error.message + value: '{{{ _ingest.on_failure_message }}}' +processors: + - script: + description: Add ngram features for ML DGA model + lang: painless + params: + dynamic_domains: + avsvmcloud.com: 0 + co.cc: 0 + cz.cc: 0 + ddns.net: 0 + dyndns.org: 0 + dynserv.com: 0 + github.io: 0 + mooo.com: 0 + mynumber.org: 0 + yi.org: 0 + source: | + String nGramAtPosition(String text, int fieldcount, int n){ + if (fieldcount+n>text.length()){ + return null; + } else { + return text.substring(fieldcount, fieldcount+n); + } + } + + String[] secondLevelDomain(Map dynamic_domains, String domain, String subdomain, String registered_domain, String top_level_domain){ + if (registered_domain == null || registered_domain == '.') { + return new String[] {domain, ''}; + } + + if (dynamic_domains.containsKey(registered_domain) == true) { + if (subdomain != null) { + return new String[] {subdomain, registered_domain}; + } + } + + return new String[] {registered_domain.substring(0, registered_domain.length()-top_level_domain.length()-1), top_level_domain}; + } + + String domain = ctx['dns']['question']['name']; + String subdomain = ctx['dns']['question']['subdomain']; + String registered_domain = ctx['dns']['question']['registered_domain']; + String top_level_domain = ctx['dns']['question']['top_level_domain']; + + String[] ret = secondLevelDomain(params.dynamic_domains, domain, subdomain, registered_domain, top_level_domain); + + String sld = ret[0]; + String tld = ret[1]; + + ctx['f'] = new HashMap(); + ctx['f']['tld'] = tld; + + for (int i=0;i 0 + value: '{{{apache.access.remote_addresses.0}}}' + - script: + description: Extract from remote_addresses, the first non-private IP to ctx.client.ip + if: ctx.apache?.access?.remote_addresses != null && ctx.apache.access.remote_addresses.length > 0 + lang: painless + source: |- + boolean isPrivateCIDR(def ip) { + CIDR class_a_network = new CIDR('10.0.0.0/8'); + CIDR class_b_network = new CIDR('172.16.0.0/12'); + CIDR class_c_network = new CIDR('192.168.0.0/16'); + + try { + return class_a_network.contains(ip) || class_b_network.contains(ip) || class_c_network.contains(ip); + } catch (IllegalArgumentException e) { + return false; + } + } + try { + if (ctx.client == null) { + Map map = new HashMap(); + ctx.put("client", map); + } + + def found = false; + for (def item : ctx.apache.access.remote_addresses) { + if (!isPrivateCIDR(item)) { + ctx.client.ip = item; + found = true; + break; + } + } + if (!found) { + ctx.client.ip = ctx.apache.access.remote_addresses[0]; + } + } catch (Exception e) { + ctx.client.ip = null; + } + tag: Get source address + - append: + field: apache.access.remote_addresses + if: ctx.source?.address != null + value: + - '{{source.address}}' + - uri_parts: + field: _tmp.url_orig + ignore_failure: true + - remove: + field: + - _tmp + ignore_missing: true + - set: + field: url.domain + if: ctx.url?.domain == null && ctx.destination?.domain != null + value: '{{destination.domain}}' + - set: + field: event.kind + value: event + - append: + field: event.category + value: web + - set: + field: event.outcome + if: ctx.http?.response?.status_code != null && ctx.http.response.status_code < 400 + value: success + - set: + field: event.outcome + if: ctx.http?.response?.status_code != null && ctx.http.response.status_code > 399 + value: failure + - grok: + field: source.address + ignore_missing: true + patterns: + - ^(%{IP:source.ip}|%{HOSTNAME:source.domain})$ + - remove: + field: event.created + ignore_failure: true + ignore_missing: true + - rename: + field: '@timestamp' + target_field: event.created + - date: + field: apache.access.time + formats: + - dd/MMM/yyyy:H:m:s Z + ignore_failure: true + target_field: '@timestamp' + - remove: + field: apache.access.time + ignore_failure: true + - user_agent: + field: user_agent.original + ignore_failure: true + - geoip: + field: source.ip + ignore_missing: true + target_field: source.geo + - geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + ignore_missing: true + properties: + - asn + - organization_name + target_field: source.as + - rename: + field: source.as.asn + ignore_missing: true + target_field: source.as.number + - rename: + field: source.as.organization_name + ignore_missing: true + target_field: source.as.organization.name + - set: + field: tls.cipher + if: ctx.apache?.access?.ssl?.cipher != null + value: '{{apache.access.ssl.cipher}}' + - script: + if: ctx.apache?.access?.ssl?.protocol != null + lang: painless + source: |- + def parts = ctx.apache.access.ssl.protocol.toLowerCase().splitOnToken("v"); if (parts.length != 2) { + return; + } if (parts[1].contains(".")) { + ctx.tls.version = parts[1]; + } else { + ctx.tls.version = parts[1] + ".0"; + } ctx.tls.version_protocol = parts[0]; + - convert: + field: source.address + ignore_missing: true + on_failure: + - set: + copy_from: source.address + field: tmp_host + - append: + allow_duplicates: false + field: related.hosts + if: ctx.tmp_host != null + value: '{{{tmp_host}}}' + - set: + field: tmp_host + if: ctx.tmp_host != null + value: "" + type: ip + - append: + allow_duplicates: false + field: related.ip + if: ctx.tmp_host == null + value: '{{{source.address}}}' + - convert: + field: destination.domain + ignore_missing: true + on_failure: + - set: + copy_from: destination.domain + field: tmp_host + - append: + allow_duplicates: false + field: related.hosts + if: ctx.tmp_host != null + value: '{{{tmp_host}}}' + - set: + field: tmp_host + if: ctx.tmp_host != null + value: "" + type: ip + - convert: + field: destination.port + if: ctx.destination?.port != null + type: long + - convert: + field: source.port + if: ctx.source?.port != null + type: long + - append: + allow_duplicates: false + field: related.ip + if: ctx.tmp_host == null + value: '{{{destination.domain}}}' + - convert: + field: apache.access.response_time + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag fail-{{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + tag: convert_response_time_to_long + type: long + - script: + description: Drops null and empty values recursively from the Elasticsearch document context. + lang: painless + source: |- + boolean drop(Object o) { + if (o == null || o == '' || o == '-') { + return true; + } else if (o instanceof Map) { + ((Map) o).values().removeIf(v -> drop(v)); + return (((Map) o).size() == 0); + } else if (o instanceof List) { + ((List) o).removeIf(v -> drop(v)); + return (((List) o).size() == 0); + } + return false; + } + drop(ctx); + - pipeline: + description: '[Fleet] Global pipeline for all data streams' + ignore_missing_pipeline: true + name: global@custom + - pipeline: + description: '[Fleet] Pipeline for all data streams of type `logs`' + ignore_missing_pipeline: true + name: logs@custom + - pipeline: + description: '[Fleet] Pipeline for all data streams of type `logs` defined by the `apache` integration' + ignore_missing_pipeline: true + name: logs-apache.integration@custom + - pipeline: + description: '[Fleet] Pipeline for the `apache.access` dataset' + ignore_missing_pipeline: true + name: logs-apache.access@custom diff --git a/internal/export/testdata/elasticsearch-8-mock-export-pipelines.yaml b/internal/export/testdata/elasticsearch-8-mock-export-pipelines.yaml new file mode 100644 index 0000000000..bbde0dbb9d --- /dev/null +++ b/internal/export/testdata/elasticsearch-8-mock-export-pipelines.yaml @@ -0,0 +1,143 @@ +--- +version: 2 +interactions: + - id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: "" + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Authorization: + - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== + User-Agent: + - go-elasticsearch/7.17.10 (darwin arm64; Go 1.24.2) + X-Elastic-Client-Meta: + - es=7.17.10,go=1.24.2,t=7.17.10,hc=1.24.2 + url: https://127.0.0.1:9200/ + method: GET + response: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + transfer_encoding: [] + trailer: {} + content_length: 540 + uncompressed: false + body: | + { + "name" : "c6180cf277a0", + "cluster_name" : "elasticsearch", + "cluster_uuid" : "fZKFi-3CSDu53Sl7YGS_5Q", + "version" : { + "number" : "8.18.0", + "build_flavor" : "default", + "build_type" : "docker", + "build_hash" : "04e979aa50b657bebd4a0937389308de82c2bdad", + "build_date" : "2025-04-10T10:09:16.444104780Z", + "build_snapshot" : false, + "lucene_version" : "9.12.1", + "minimum_wire_compatibility_version" : "7.17.0", + "minimum_index_compatibility_version" : "7.0.0" + }, + "tagline" : "You Know, for Search" + } + headers: + Content-Length: + - "540" + Content-Type: + - application/json + X-Elastic-Product: + - Elasticsearch + status: 200 OK + code: 200 + duration: 4.848ms + - id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: "" + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Authorization: + - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== + User-Agent: + - go-elasticsearch/7.17.10 (darwin arm64; Go 1.24.2) + X-Elastic-Client-Meta: + - es=7.17.10,go=1.24.2,t=7.17.10,hc=1.24.2 + url: https://127.0.0.1:9200/_ingest/pipeline/logs-apache.access-2.0.0,2.3.0-ml_dga_ingest_pipeline + method: GET + response: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + transfer_encoding: [] + trailer: {} + content_length: 8694 + uncompressed: false + body: '{"logs-apache.access-2.0.0":{"description":"Pipeline for parsing Apache HTTP Server access logs. Requires the geoip and user_agent plugins.","processors":[{"pipeline":{"if":"ctx.message.startsWith(''{'')","name":"logs-apache.access-2.0.0-third-party"}},{"set":{"field":"event.ingested","value":"{{_ingest.timestamp}}"}},{"set":{"field":"ecs.version","value":"8.11.0"}},{"rename":{"field":"message","target_field":"event.original","ignore_missing":true,"if":"ctx.event?.original == null"}},{"remove":{"field":"message","ignore_missing":true,"if":"ctx.event?.original != null","description":"The `message` field is no longer required if the document has an `event.original` field."}},{"grok":{"field":"event.original","patterns":["(%{IPORHOST:destination.domain}:?%{POSINT:destination.port}? )?(%{IPORHOST:source.address}:?%{POSINT:source.port}? )%{DATA:apache.access.identity} %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"(%{DATA:apache.access.tls_handshake.error})?((%{WORD:http.request.method}?) %{DATA:_tmp.url_orig}? HTTP/%{NUMBER:http.version})?(%{WORD:http.request.method}? ?%{DATA:apache.access.http.request_headers}?(-)? %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version})?(-)?\" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( %{NUMBER:apache.access.response_time})?( \"%{DATA:http.request.referrer}\")?( \"%{DATA:user_agent.original}\")?( X-Forwarded-For=\"%{ADDRESS_LIST:apache.access.remote_addresses}\")?","%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"-\" %{NUMBER:http.response.status_code:long} -","\\[%{HTTPDATE:apache.access.time}\\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol} %{DATA:apache.access.ssl.cipher} \"%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}\" (-|%{NUMBER:http.response.body.bytes:long})"],"ignore_missing":true,"pattern_definitions":{"ADDRESS_LIST":"(%{IP})(\"?,?\\s*(%{IP}))*"}}},{"split":{"field":"apache.access.remote_addresses","separator":"\"?,\\s*","ignore_missing":true}},{"set":{"field":"network.forwarded_ip","value":"{{{apache.access.remote_addresses.0}}}","if":"ctx.apache?.access?.remote_addresses != null && ctx.apache.access.remote_addresses.length > 0"}},{"script":{"if":"ctx.apache?.access?.remote_addresses != null && ctx.apache.access.remote_addresses.length > 0","lang":"painless","tag":"Get source address","description":"Extract from remote_addresses, the first non-private IP to ctx.client.ip","source":"boolean isPrivateCIDR(def ip) {\n CIDR class_a_network = new CIDR(''10.0.0.0/8'');\n CIDR class_b_network = new CIDR(''172.16.0.0/12'');\n CIDR class_c_network = new CIDR(''192.168.0.0/16'');\n\n try {\n return class_a_network.contains(ip) || class_b_network.contains(ip) || class_c_network.contains(ip);\n } catch (IllegalArgumentException e) {\n return false;\n }\n}\ntry {\n if (ctx.client == null) {\n Map map = new HashMap();\n ctx.put(\"client\", map);\n }\n\n def found = false;\n for (def item : ctx.apache.access.remote_addresses) {\n if (!isPrivateCIDR(item)) {\n ctx.client.ip = item;\n found = true;\n break;\n }\n }\n if (!found) {\n ctx.client.ip = ctx.apache.access.remote_addresses[0];\n }\n} catch (Exception e) {\n ctx.client.ip = null;\n}"}},{"append":{"field":"apache.access.remote_addresses","value":["{{source.address}}"],"if":"ctx.source?.address != null"}},{"uri_parts":{"field":"_tmp.url_orig","ignore_failure":true}},{"remove":{"field":["_tmp"],"ignore_missing":true}},{"set":{"field":"url.domain","value":"{{destination.domain}}","if":"ctx.url?.domain == null && ctx.destination?.domain != null"}},{"set":{"field":"event.kind","value":"event"}},{"append":{"field":"event.category","value":"web"}},{"set":{"field":"event.outcome","value":"success","if":"ctx.http?.response?.status_code != null && ctx.http.response.status_code < 400"}},{"set":{"field":"event.outcome","value":"failure","if":"ctx.http?.response?.status_code != null && ctx.http.response.status_code > 399"}},{"grok":{"field":"source.address","ignore_missing":true,"patterns":["^(%{IP:source.ip}|%{HOSTNAME:source.domain})$"]}},{"remove":{"field":"event.created","ignore_missing":true,"ignore_failure":true}},{"rename":{"field":"@timestamp","target_field":"event.created"}},{"date":{"field":"apache.access.time","target_field":"@timestamp","formats":["dd/MMM/yyyy:H:m:s Z"],"ignore_failure":true}},{"remove":{"field":"apache.access.time","ignore_failure":true}},{"user_agent":{"field":"user_agent.original","ignore_failure":true}},{"geoip":{"field":"source.ip","target_field":"source.geo","ignore_missing":true}},{"geoip":{"database_file":"GeoLite2-ASN.mmdb","field":"source.ip","target_field":"source.as","properties":["asn","organization_name"],"ignore_missing":true}},{"rename":{"field":"source.as.asn","target_field":"source.as.number","ignore_missing":true}},{"rename":{"field":"source.as.organization_name","target_field":"source.as.organization.name","ignore_missing":true}},{"set":{"field":"tls.cipher","value":"{{apache.access.ssl.cipher}}","if":"ctx.apache?.access?.ssl?.cipher != null"}},{"script":{"lang":"painless","if":"ctx.apache?.access?.ssl?.protocol != null","source":"def parts = ctx.apache.access.ssl.protocol.toLowerCase().splitOnToken(\"v\"); if (parts.length != 2) {\n return;\n} if (parts[1].contains(\".\")) {\n ctx.tls.version = parts[1];\n} else {\n ctx.tls.version = parts[1] + \".0\";\n} ctx.tls.version_protocol = parts[0];"}},{"convert":{"field":"source.address","type":"ip","ignore_missing":true,"on_failure":[{"set":{"field":"tmp_host","copy_from":"source.address"}},{"append":{"field":"related.hosts","value":"{{{tmp_host}}}","allow_duplicates":false,"if":"ctx.tmp_host != null"}},{"set":{"field":"tmp_host","value":"","if":"ctx.tmp_host != null"}}]}},{"append":{"field":"related.ip","value":"{{{source.address}}}","allow_duplicates":false,"if":"ctx.tmp_host == null"}},{"convert":{"field":"destination.domain","type":"ip","ignore_missing":true,"on_failure":[{"set":{"field":"tmp_host","copy_from":"destination.domain"}},{"append":{"field":"related.hosts","value":"{{{tmp_host}}}","allow_duplicates":false,"if":"ctx.tmp_host != null"}},{"set":{"field":"tmp_host","value":"","if":"ctx.tmp_host != null"}}]}},{"convert":{"field":"destination.port","type":"long","if":"ctx.destination?.port != null"}},{"convert":{"field":"source.port","type":"long","if":"ctx.source?.port != null"}},{"append":{"field":"related.ip","value":"{{{destination.domain}}}","allow_duplicates":false,"if":"ctx.tmp_host == null"}},{"convert":{"field":"apache.access.response_time","tag":"convert_response_time_to_long","type":"long","ignore_missing":true,"on_failure":[{"append":{"field":"error.message","value":"Processor {{{_ingest.on_failure_processor_type}}} with tag fail-{{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}"}}]}},{"script":{"lang":"painless","source":"boolean drop(Object o) {\n if (o == null || o == '''' || o == ''-'') {\n return true;\n } else if (o instanceof Map) {\n ((Map) o).values().removeIf(v -> drop(v));\n return (((Map) o).size() == 0);\n } else if (o instanceof List) {\n ((List) o).removeIf(v -> drop(v));\n return (((List) o).size() == 0);\n }\n return false;\n}\ndrop(ctx);","description":"Drops null and empty values recursively from the Elasticsearch document context."}},{"pipeline":{"name":"global@custom","ignore_missing_pipeline":true,"description":"[Fleet] Global pipeline for all data streams"}},{"pipeline":{"name":"logs@custom","ignore_missing_pipeline":true,"description":"[Fleet] Pipeline for all data streams of type `logs`"}},{"pipeline":{"name":"logs-apache.integration@custom","ignore_missing_pipeline":true,"description":"[Fleet] Pipeline for all data streams of type `logs` defined by the `apache` integration"}},{"pipeline":{"name":"logs-apache.access@custom","ignore_missing_pipeline":true,"description":"[Fleet] Pipeline for the `apache.access` dataset"}}],"on_failure":[{"set":{"field":"error.message","value":"{{ _ingest.on_failure_message }}"}}],"_meta":{"managed_by":"fleet","managed":true,"package":{"name":"apache"}}},"2.3.0-ml_dga_ingest_pipeline":{"description":"Pipelines for enriching DNS data. Ignores non-DNS data.","processors":[{"pipeline":{"if":"ctx.network?.protocol == ''dns'' && ctx.dns?.question?.type != ''PTR''","name":"2.3.0-ml_dga_inference_pipeline"}}],"on_failure":[{"set":{"field":"event.kind","value":"pipeline_error"}},{"append":{"field":"error.message","value":"{{{ _ingest.on_failure_message }}}"}}],"_meta":{"managed_by":"fleet","managed":true,"package":{"name":"dga"}}}}' + headers: + Content-Length: + - "8694" + Content-Type: + - application/json + X-Elastic-Product: + - Elasticsearch + status: 200 OK + code: 200 + duration: 1.365916ms + - id: 2 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: "" + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Authorization: + - Basic ZWxhc3RpYzpjaGFuZ2VtZQ== + User-Agent: + - go-elasticsearch/7.17.10 (darwin arm64; Go 1.24.2) + X-Elastic-Client-Meta: + - es=7.17.10,go=1.24.2,t=7.17.10,hc=1.24.2 + url: https://127.0.0.1:9200/_ingest/pipeline/logs-apache.access-2.0.0-third-party,global@custom,logs@custom,logs-apache.integration@custom,logs-apache.access@custom,2.3.0-ml_dga_inference_pipeline + method: GET + response: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + transfer_encoding: [] + trailer: {} + content_length: 4087 + uncompressed: false + body: '{"logs-apache.access-2.0.0-third-party":{"description":"Pipeline for parsing Apache HTTP Server logs from third party api","processors":[{"json":{"field":"message","target_field":"json"}},{"drop":{"if":"ctx.json?.result == null"}},{"fingerprint":{"fields":["json.result._cd","json.result._indextime","json.result._raw","json.result._time","json.result.host","json.result.source"],"target_field":"_id","ignore_missing":true}},{"set":{"copy_from":"json.result._raw","field":"message","ignore_empty_value":true}},{"set":{"copy_from":"json.result.host","field":"host.name","ignore_empty_value":true}},{"set":{"copy_from":"json.result.source","field":"file.path","ignore_empty_value":true}},{"remove":{"field":["json"],"ignore_missing":true}}],"on_failure":[{"append":{"field":"error.message","value":"error in third-party pipeline: error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}} with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}} {{ _ingest.on_failure_message }}"}}],"_meta":{"managed_by":"fleet","managed":true,"package":{"name":"apache"}}},"2.3.0-ml_dga_inference_pipeline":{"processors":[{"script":{"description":"Add ngram features for ML DGA model","lang":"painless","source":"String nGramAtPosition(String text, int fieldcount, int n){\n if (fieldcount+n>text.length()){\n return null;\n } else {\n return text.substring(fieldcount, fieldcount+n);\n }\n}\n\nString[] secondLevelDomain(Map dynamic_domains, String domain, String subdomain, String registered_domain, String top_level_domain){\n if (registered_domain == null || registered_domain == ''.'') {\n return new String[] {domain, ''''};\n }\n\n if (dynamic_domains.containsKey(registered_domain) == true) {\n if (subdomain != null) {\n return new String[] {subdomain, registered_domain};\n }\n }\n\n return new String[] {registered_domain.substring(0, registered_domain.length()-top_level_domain.length()-1), top_level_domain};\n }\n\nString domain = ctx[''dns''][''question''][''name''];\nString subdomain = ctx[''dns''][''question''][''subdomain''];\nString registered_domain = ctx[''dns''][''question''][''registered_domain''];\nString top_level_domain = ctx[''dns''][''question''][''top_level_domain''];\n\nString[] ret = secondLevelDomain(params.dynamic_domains, domain, subdomain, registered_domain, top_level_domain);\n\nString sld = ret[0];\nString tld = ret[1];\n\nctx[''f''] = new HashMap();\nctx[''f''][''tld''] = tld;\n\nfor (int i=0;i