diff --git a/cmd/testrunner.go b/cmd/testrunner.go index dae556c3e..6773435bf 100644 --- a/cmd/testrunner.go +++ b/cmd/testrunner.go @@ -417,6 +417,8 @@ func getTestRunnerSystemCommand() *cobra.Command { cmd.Flags().Bool(cobraext.TearDownFlagName, false, cobraext.TearDownFlagDescription) cmd.Flags().Bool(cobraext.NoProvisionFlagName, false, cobraext.NoProvisionFlagDescription) + cmd.Flags().String(cobraext.TestDumpPrefixFlagName, "", cobraext.TestDumpPrefixFlagDescription) + cmd.MarkFlagsMutuallyExclusive(cobraext.SetupFlagName, cobraext.TearDownFlagName, cobraext.NoProvisionFlagName) cmd.MarkFlagsRequiredTogether(cobraext.ConfigFileFlagName, cobraext.SetupFlagName) @@ -472,6 +474,11 @@ func testRunnerSystemCommandAction(cmd *cobra.Command, args []string) error { return cobraext.FlagParsingError(err, cobraext.TestCoverageFormatFlagName) } + dumpPrefix, err := cmd.Flags().GetString(cobraext.TestDumpPrefixFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.TestDumpPrefixFlagName) + } + if !slices.Contains(testrunner.CoverageFormatsList(), testCoverageFormat) { return cobraext.FlagParsingError(fmt.Errorf("coverage format not available: %s", testCoverageFormat), cobraext.TestCoverageFormatFlagName) } @@ -578,6 +585,7 @@ func testRunnerSystemCommandAction(cmd *cobra.Command, args []string) error { GlobalTestConfig: globalTestConfig.System, WithCoverage: testCoverage, CoverageType: testCoverageFormat, + DumpPrefix: dumpPrefix, }) logger.Debugf("Running suite...") diff --git a/internal/cobraext/flags.go b/internal/cobraext/flags.go index 1838404fd..2f6911064 100644 --- a/internal/cobraext/flags.go +++ b/internal/cobraext/flags.go @@ -209,6 +209,9 @@ const ( TestCoverageFormatFlagName = "coverage-format" TestCoverageFormatFlagDescription = "set format for coverage reports: %s" + TestDumpPrefixFlagName = "dump-docs" + TestDumpPrefixFlagDescription = "prefix for system test document output dump file" + VariantFlagName = "variant" VariantFlagDescription = "service variant" diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 6ba712511..4dbd0eee2 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -39,6 +39,7 @@ type runner struct { generateTestResult bool withCoverage bool coverageType string + dumpPrefix string configFilePath string runSetup bool @@ -76,6 +77,7 @@ type SystemTestRunnerOptions struct { DeferCleanup time.Duration WithCoverage bool CoverageType string + DumpPrefix string } func NewSystemTestRunner(options SystemTestRunnerOptions) *runner { @@ -97,6 +99,7 @@ func NewSystemTestRunner(options SystemTestRunnerOptions) *runner { globalTestConfig: options.GlobalTestConfig, withCoverage: options.WithCoverage, coverageType: options.CoverageType, + dumpPrefix: options.DumpPrefix, } r.resourcesManager = resources.NewManager() @@ -262,6 +265,7 @@ func (r *runner) GetTests(ctx context.Context) ([]testrunner.Tester, error) { GlobalTestConfig: r.globalTestConfig, WithCoverage: r.withCoverage, CoverageType: r.coverageType, + DumpPrefix: r.dumpPrefix, }) if err != nil { return nil, fmt.Errorf( diff --git a/internal/testrunner/runners/system/tester.go b/internal/testrunner/runners/system/tester.go index be0b34668..63a839d29 100644 --- a/internal/testrunner/runners/system/tester.go +++ b/internal/testrunner/runners/system/tester.go @@ -186,6 +186,7 @@ type tester struct { dataStreamManifest *packages.DataStreamManifest withCoverage bool coverageType string + dumpPrefix string serviceStateFilePath string @@ -218,6 +219,7 @@ type SystemTesterOptions struct { GlobalTestConfig testrunner.GlobalRunnerTestConfig WithCoverage bool CoverageType string + DumpPrefix string RunSetup bool RunTearDown bool @@ -672,7 +674,6 @@ func (r *tester) runTestPerVariant(ctx context.Context, stackConfig stack.Config logger.Debugf("Using config: %q", testConfig.Name()) partial, err := r.runTest(ctx, testConfig, stackConfig, svcInfo) - tdErr := r.tearDownTest(ctx) if err != nil { return partial, err @@ -1814,32 +1815,26 @@ func (r *tester) runTest(ctx context.Context, config *testConfig, stackConfig st return results, nil } - if dump, ok := os.LookupEnv(dumpScenarioDocsEnv); ok && dump != "" { - err := dumpScenarioDocs(scenario.docs) + var dumpPath string + switch { + case r.dumpPrefix != "": + dumpPath = fmt.Sprintf("%s-%s.json", r.dumpPrefix, time.Now().Format("20060102150405")) + case os.Getenv(dumpScenarioDocsEnv) != "": + dumpPath = filepath.Join(os.TempDir(), fmt.Sprintf("elastic-package-test-docs-dump-%s.json", time.Now().Format("20060102150405"))) + } + var dumpErr error + if dumpPath != "" { + err := dumpScenarioDocs(scenario.docs, dumpPath) if err != nil { - return nil, fmt.Errorf("failed to dump scenario docs: %w", err) + dumpErr = fmt.Errorf("failed to dump scenario docs: %w", err) } } - return r.validateTestScenario(ctx, result, scenario, config) -} - -func (r *tester) isTestUsingOTELCollectorInput(policyTemplateInput string) bool { - // Just supported for input packages currently - if r.pkgManifest.Type != "input" { - return false - } - - if policyTemplateInput != otelCollectorInputName { - return false - } - - return true + results, err := r.validateTestScenario(ctx, result, scenario, config) + return results, errors.Join(err, dumpErr) } -func dumpScenarioDocs(docs any) error { - timestamp := time.Now().Format("20060102150405") - path := filepath.Join(os.TempDir(), fmt.Sprintf("elastic-package-test-docs-dump-%s.json", timestamp)) +func dumpScenarioDocs(docs any, path string) error { f, err := os.Create(path) if err != nil { return fmt.Errorf("failed to create dump file: %w", err) @@ -1857,6 +1852,19 @@ func dumpScenarioDocs(docs any) error { return nil } +func (r *tester) isTestUsingOTELCollectorInput(policyTemplateInput string) bool { + // Just supported for input packages currently + if r.pkgManifest.Type != "input" { + return false + } + + if policyTemplateInput != otelCollectorInputName { + return false + } + + return true +} + func (r *tester) checkEnrolledAgents(ctx context.Context, agentInfo agentdeployer.AgentInfo, svcInfo servicedeployer.ServiceInfo) (*kibana.Agent, error) { var agents []kibana.Agent