Skip to content

Add export ingest-pipelines command #2574

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 24 commits into from
May 16, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
2b8dd4b
Move export dashboards cmd into own file
MichelLosier May 1, 2025
f1ffa7f
Add cmd interface for export ingest-pipelines and init es client
MichelLosier May 1, 2025
5851641
Implement ingest pipeline list selection
MichelLosier May 2, 2025
dd5c688
Have ingest pipelines export fetch pipelines by IDs
MichelLosier May 2, 2025
f65865c
Refactor dump pipelines to share functionality with export
MichelLosier May 5, 2025
d33c88e
Get nested pipelines
MichelLosier May 5, 2025
ec296da
Filter out system pipelines for export selection
MichelLosier May 6, 2025
6b8c718
Add GetProcessorPipelineNames method to RemotePipeline struct
MichelLosier May 6, 2025
a6e1c14
Add package root as export option, and implement PipelineWriteLocation
MichelLosier May 7, 2025
721eba5
Write parent pipelines to yaml files
MichelLosier May 7, 2025
748c597
Fix ingest_pipeline dir name
MichelLosier May 7, 2025
5ec05c4
Write pipeline processor pipeline deps
MichelLosier May 7, 2025
1ff73e5
Remove logging lines
MichelLosier May 7, 2025
11a4494
Update installedobjects_test to handle bulk pipeline requests
MichelLosier May 8, 2025
d5a089c
Apply formatting
MichelLosier May 9, 2025
5e8cbb0
Add triple-dash document start to pipeline yaml
MichelLosier May 9, 2025
db80e29
Remove log line
MichelLosier May 9, 2025
cb13e0d
Update readme
MichelLosier May 9, 2025
d9ddd09
Formatting and clean up
MichelLosier May 15, 2025
c69162a
Remove _meta and version fields from exported pipelines
MichelLosier May 15, 2025
b1925a3
Formatting
MichelLosier May 15, 2025
2f2de41
Set pipeline yaml export with 2 space indent
MichelLosier May 15, 2025
7969335
Add test for export.IngestPipelines
MichelLosier May 16, 2025
c741674
Add matcher to export ingest_pipelines test
MichelLosier May 16, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -354,6 +354,14 @@ Use this command to export dashboards with referenced objects from the Kibana in

Use this command to download selected dashboards and other associated saved objects from Kibana. This command adjusts the downloaded saved objects according to package naming conventions (prefixes, unique IDs) and writes them locally into folders corresponding to saved object types (dashboard, visualization, map, etc.).

### `elastic-package export ingest-pipelines`

_Context: package_

Use this command to export ingest pipelines with referenced pipelines from the Elasticsearch instance.

Use this command to download selected ingest pipelines and its referenced processor pipelines from Elasticsearch. Select data stream or the package root directories to download the pipelines. Pipelines are downloaded as is and will need adjustment to meet your package needs.

### `elastic-package format`

_Context: package_
Expand Down
122 changes: 13 additions & 109 deletions cmd/export.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,27 +5,16 @@
package cmd

import (
"context"
"fmt"

"github.com/AlecAivazis/survey/v2"

"github.com/spf13/cobra"

"github.com/elastic/elastic-package/internal/cobraext"
"github.com/elastic/elastic-package/internal/common"
"github.com/elastic/elastic-package/internal/export"
"github.com/elastic/elastic-package/internal/install"
"github.com/elastic/elastic-package/internal/kibana"
"github.com/elastic/elastic-package/internal/stack"
)

const exportLongDescription = `Use this command to export assets relevant for the package, e.g. Kibana dashboards.`

const exportDashboardsLongDescription = `Use this command to export dashboards with referenced objects from the Kibana instance.

Use this command to download selected dashboards and other associated saved objects from Kibana. This command adjusts the downloaded saved objects according to package naming conventions (prefixes, unique IDs) and writes them locally into folders corresponding to saved object types (dashboard, visualization, map, etc.).`

func setupExportCommand() *cobraext.Command {
exportDashboardCmd := &cobra.Command{
Use: "dashboards",
Expand All @@ -38,111 +27,26 @@ func setupExportCommand() *cobraext.Command {
exportDashboardCmd.Flags().Bool(cobraext.TLSSkipVerifyFlagName, false, cobraext.TLSSkipVerifyFlagDescription)
exportDashboardCmd.Flags().Bool(cobraext.AllowSnapshotFlagName, false, cobraext.AllowSnapshotDescription)

exportIngestPipelinesCmd := &cobra.Command{
Use: "ingest-pipelines",
Short: "Export ingest pipelines from Elasticsearch",
Long: exportIngestPipelinesLongDescription,
Args: cobra.NoArgs,
RunE: exportIngestPipelinesCmd,
}

exportIngestPipelinesCmd.Flags().StringSliceP(cobraext.IngestPipelineIDsFlagName, "d", nil, cobraext.IngestPipelineIDsFlagDescription)
exportIngestPipelinesCmd.Flags().Bool(cobraext.TLSSkipVerifyFlagName, false, cobraext.TLSSkipVerifyFlagDescription)
exportIngestPipelinesCmd.Flags().Bool(cobraext.AllowSnapshotFlagName, false, cobraext.AllowSnapshotDescription)

cmd := &cobra.Command{
Use: "export",
Short: "Export package assets",
Long: exportLongDescription,
}
cmd.AddCommand(exportDashboardCmd)
cmd.AddCommand(exportIngestPipelinesCmd)
cmd.PersistentFlags().StringP(cobraext.ProfileFlagName, "p", "", fmt.Sprintf(cobraext.ProfileFlagDescription, install.ProfileNameEnvVar))

return cobraext.NewCommand(cmd, cobraext.ContextPackage)
}

func exportDashboardsCmd(cmd *cobra.Command, args []string) error {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Moves export dashboards functionality into own file now that we are adding support for ingest-pipelines

cmd.Println("Export Kibana dashboards")

dashboardIDs, err := cmd.Flags().GetStringSlice(cobraext.DashboardIDsFlagName)
if err != nil {
return cobraext.FlagParsingError(err, cobraext.DashboardIDsFlagName)
}

common.TrimStringSlice(dashboardIDs)

var opts []kibana.ClientOption
tlsSkipVerify, _ := cmd.Flags().GetBool(cobraext.TLSSkipVerifyFlagName)
if tlsSkipVerify {
opts = append(opts, kibana.TLSSkipVerify())
}

allowSnapshot, _ := cmd.Flags().GetBool(cobraext.AllowSnapshotFlagName)
if err != nil {
return cobraext.FlagParsingError(err, cobraext.AllowSnapshotFlagName)
}

profile, err := cobraext.GetProfileFlag(cmd)
if err != nil {
return err
}

kibanaClient, err := stack.NewKibanaClientFromProfile(profile, opts...)
if err != nil {
return fmt.Errorf("can't create Kibana client: %w", err)
}

kibanaVersion, err := kibanaClient.Version()
if err != nil {
return fmt.Errorf("can't get Kibana status information: %w", err)
}

if kibanaVersion.IsSnapshot() {
message := fmt.Sprintf("exporting dashboards from a SNAPSHOT version of Kibana (%s) is discouraged. It could lead to invalid dashboards (for example if they use features that are reverted or modified before the final release)", kibanaVersion.Version())
if !allowSnapshot {
return fmt.Errorf("%s. --%s flag can be used to ignore this error", message, cobraext.AllowSnapshotFlagName)
}
fmt.Printf("Warning: %s\n", message)
}

if len(dashboardIDs) == 0 {
dashboardIDs, err = promptDashboardIDs(cmd.Context(), kibanaClient)
if err != nil {
return fmt.Errorf("prompt for dashboard selection failed: %w", err)
}

if len(dashboardIDs) == 0 {
fmt.Println("No dashboards were found in Kibana.")
return nil
}
}

err = export.Dashboards(cmd.Context(), kibanaClient, dashboardIDs)
if err != nil {
return fmt.Errorf("dashboards export failed: %w", err)
}

cmd.Println("Done")
return nil
}

func promptDashboardIDs(ctx context.Context, kibanaClient *kibana.Client) ([]string, error) {
savedDashboards, err := kibanaClient.FindDashboards(ctx)
if err != nil {
return nil, fmt.Errorf("finding dashboards failed: %w", err)
}

if len(savedDashboards) == 0 {
return []string{}, nil
}

dashboardsPrompt := &survey.MultiSelect{
Message: "Which dashboards would you like to export?",
Options: savedDashboards.Strings(),
PageSize: 100,
}

var selectedOptions []string
err = survey.AskOne(dashboardsPrompt, &selectedOptions, survey.WithValidator(survey.Required))
if err != nil {
return nil, err
}

var selected []string
for _, option := range selectedOptions {
for _, sd := range savedDashboards {
if sd.String() == option {
selected = append(selected, sd.ID)
}
}
}
return selected, nil
}
122 changes: 122 additions & 0 deletions cmd/export_dashboards.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This file was extracted from cmd/export.go

// or more contributor license agreements. Licensed under the Elastic License;
// you may not use this file except in compliance with the Elastic License.

package cmd

import (
"context"
"fmt"

"github.com/AlecAivazis/survey/v2"

"github.com/spf13/cobra"

"github.com/elastic/elastic-package/internal/cobraext"
"github.com/elastic/elastic-package/internal/common"
"github.com/elastic/elastic-package/internal/export"
"github.com/elastic/elastic-package/internal/kibana"
"github.com/elastic/elastic-package/internal/stack"
)

const exportDashboardsLongDescription = `Use this command to export dashboards with referenced objects from the Kibana instance.

Use this command to download selected dashboards and other associated saved objects from Kibana. This command adjusts the downloaded saved objects according to package naming conventions (prefixes, unique IDs) and writes them locally into folders corresponding to saved object types (dashboard, visualization, map, etc.).`

func exportDashboardsCmd(cmd *cobra.Command, args []string) error {
cmd.Println("Export Kibana dashboards")

dashboardIDs, err := cmd.Flags().GetStringSlice(cobraext.DashboardIDsFlagName)
if err != nil {
return cobraext.FlagParsingError(err, cobraext.DashboardIDsFlagName)
}

common.TrimStringSlice(dashboardIDs)

var opts []kibana.ClientOption
tlsSkipVerify, _ := cmd.Flags().GetBool(cobraext.TLSSkipVerifyFlagName)
if tlsSkipVerify {
opts = append(opts, kibana.TLSSkipVerify())
}

allowSnapshot, _ := cmd.Flags().GetBool(cobraext.AllowSnapshotFlagName)
if err != nil {
return cobraext.FlagParsingError(err, cobraext.AllowSnapshotFlagName)
}

profile, err := cobraext.GetProfileFlag(cmd)
if err != nil {
return err
}

kibanaClient, err := stack.NewKibanaClientFromProfile(profile, opts...)
if err != nil {
return fmt.Errorf("can't create Kibana client: %w", err)
}

kibanaVersion, err := kibanaClient.Version()
if err != nil {
return fmt.Errorf("can't get Kibana status information: %w", err)
}

if kibanaVersion.IsSnapshot() {
message := fmt.Sprintf("exporting dashboards from a SNAPSHOT version of Kibana (%s) is discouraged. It could lead to invalid dashboards (for example if they use features that are reverted or modified before the final release)", kibanaVersion.Version())
if !allowSnapshot {
return fmt.Errorf("%s. --%s flag can be used to ignore this error", message, cobraext.AllowSnapshotFlagName)
}
fmt.Printf("Warning: %s\n", message)
}

if len(dashboardIDs) == 0 {
dashboardIDs, err = promptDashboardIDs(cmd.Context(), kibanaClient)
if err != nil {
return fmt.Errorf("prompt for dashboard selection failed: %w", err)
}

if len(dashboardIDs) == 0 {
fmt.Println("No dashboards were found in Kibana.")
return nil
}
}

err = export.Dashboards(cmd.Context(), kibanaClient, dashboardIDs)
if err != nil {
return fmt.Errorf("dashboards export failed: %w", err)
}

cmd.Println("Done")
return nil
}

func promptDashboardIDs(ctx context.Context, kibanaClient *kibana.Client) ([]string, error) {
savedDashboards, err := kibanaClient.FindDashboards(ctx)
if err != nil {
return nil, fmt.Errorf("finding dashboards failed: %w", err)
}

if len(savedDashboards) == 0 {
return []string{}, nil
}

dashboardsPrompt := &survey.MultiSelect{
Message: "Which dashboards would you like to export?",
Options: savedDashboards.Strings(),
PageSize: 100,
}

var selectedOptions []string
err = survey.AskOne(dashboardsPrompt, &selectedOptions, survey.WithValidator(survey.Required))
if err != nil {
return nil, err
}

var selected []string
for _, option := range selectedOptions {
for _, sd := range savedDashboards {
if sd.String() == option {
selected = append(selected, sd.ID)
}
}
}
return selected, nil
}
Loading