From c61ecf2f7ffbf1cc684b5ae77ab3aa60aa5312d8 Mon Sep 17 00:00:00 2001 From: frjcomp <107982661+frjcomp@users.noreply.github.com> Date: Wed, 1 Oct 2025 14:26:47 +0000 Subject: [PATCH 01/13] test --- .github/workflows/actions.yml | 30 ++++++++ src/pipeleak/cmd/github/action.go | 110 ++++++++++++++++++++++++++++++ src/pipeleak/cmd/github/github.go | 1 + src/pipeleak/cmd/github/helper.go | 52 ++++++++++++++ src/pipeleak/cmd/github/scan.go | 46 +------------ 5 files changed, 194 insertions(+), 45 deletions(-) create mode 100644 .github/workflows/actions.yml create mode 100644 src/pipeleak/cmd/github/action.go create mode 100644 src/pipeleak/cmd/github/helper.go diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml new file mode 100644 index 00000000..535229fe --- /dev/null +++ b/.github/workflows/actions.yml @@ -0,0 +1,30 @@ + +name: Wait for Other Runs testing + +on: + workflow_dispatch: + +jobs: + scan-secrets: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.22' + - name: Wait for other runs on same commit + working-directory: src/pipeleak + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_SHA: ${{ github.sha }} + GITHUB_RUN_ID: ${{ github.run_id }} + run: go run main.go gh action + dummy-wait: + runs-on: ubuntu-latest + steps: + - name: Wait 3 minutes, echo test, then wait 2 more minutes + run: | + sleep 180 + echo test + sleep 120 \ No newline at end of file diff --git a/src/pipeleak/cmd/github/action.go b/src/pipeleak/cmd/github/action.go new file mode 100644 index 00000000..15c84bb5 --- /dev/null +++ b/src/pipeleak/cmd/github/action.go @@ -0,0 +1,110 @@ +package github + +import ( + "context" + "os" + "strconv" + "strings" + "time" + + "github.com/CompassSecurity/pipeleak/helper" + "github.com/google/go-github/v69/github" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" +) + +func NewActionScanCmd() *cobra.Command { + scanActionCmd := &cobra.Command{ + Use: "action", + Short: "Scan all jobs of the action workflow running", + Long: `Scan GitHub Actions workflow runs and artifacts for secrets`, + Example: `pipeleak gh action -t $GH_TOKEN`, + Run: ScanAction, + } + + scanActionCmd.PersistentFlags().BoolVarP(&options.Verbose, "verbose", "v", false, "Verbose logging") + + return scanActionCmd +} + +func ScanAction(cmd *cobra.Command, args []string) { + helper.SetLogLevel(options.Verbose) + scanWorkflowRuns() + log.Info().Msg("Scan Finished, Bye Bye 🏳️‍🌈🔥") +} + +func scanWorkflowRuns() { + log.Info().Msg("Scanning GitHub Actions workflow runs for secrets") + ctx := context.WithValue(context.Background(), github.BypassRateLimitCheck, true) + client := setupClient(options.AccessToken) + + token := os.Getenv("GITHUB_TOKEN") + if token == "" { + log.Fatal().Msg("GITHUB_TOKEN not set") + } + + repoFull := os.Getenv("GITHUB_REPOSITORY") + if token == "" { + log.Fatal().Msg("GITHUB_REPOSITORY not set") + } + + parts := strings.Split(repoFull, "/") + if len(parts) != 2 { + log.Fatal().Str("repository", repoFull).Msg("invalid GITHUB_REPOSITORY") + } + + owner, repo := parts[0], parts[1] + + sha := os.Getenv("GITHUB_SHA") + if sha == "" { + log.Fatal().Msg("GITHUB_SHA not set") + } + + runIDStr := os.Getenv("GITHUB_RUN_ID") + if runIDStr == "" { + log.Fatal().Msg("GITHUB_RUN_ID not set") + } + + currentRunID, _ := strconv.ParseInt(runIDStr, 10, 64) + + for { + allCompleted := true + + opts := &github.ListWorkflowRunsOptions{ + ListOptions: github.ListOptions{PerPage: 100}, + } + + for { + runs, resp, err := client.Actions.ListRepositoryWorkflowRuns(ctx, owner, repo, opts) + if err != nil { + log.Fatal().Stack().Err(err).Msg("Failed listing workflow runs") + } + + for _, run := range runs.WorkflowRuns { + log.Info().Int64("run", run.GetID()).Str("commit", run.GetHeadSHA()).Str("name", run.GetName()).Msg("Check run") + if run.GetHeadSHA() == sha && run.GetID() != currentRunID { + status := run.GetStatus() + log.Info().Int64("run", run.GetID()).Str("status", status).Str("name", run.GetName()).Msgf("Run") + + if status != "completed" { + allCompleted = false + } + } + } + + if resp.NextPage == 0 { + break + } + + opts.Page = resp.NextPage + } + + if allCompleted { + log.Info().Msg("✅ All *other* runs for this commit are completed") + break + } + + log.Info().Msg("⏳ Still waiting... retrying in 30s") + time.Sleep(30 * time.Second) + } +} diff --git a/src/pipeleak/cmd/github/github.go b/src/pipeleak/cmd/github/github.go index 802fdb77..85d98755 100644 --- a/src/pipeleak/cmd/github/github.go +++ b/src/pipeleak/cmd/github/github.go @@ -12,6 +12,7 @@ func NewGitHubRootCmd() *cobra.Command { } ghCmd.AddCommand(NewScanCmd()) + ghCmd.AddCommand(NewActionScanCmd()) return ghCmd } diff --git a/src/pipeleak/cmd/github/helper.go b/src/pipeleak/cmd/github/helper.go new file mode 100644 index 00000000..05e80639 --- /dev/null +++ b/src/pipeleak/cmd/github/helper.go @@ -0,0 +1,52 @@ +package github + +import ( + "context" + "net/http" + "time" + + "github.com/gofri/go-github-ratelimit/v2/github_ratelimit" + "github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_primary_ratelimit" + "github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_secondary_ratelimit" + "github.com/google/go-github/v69/github" + "github.com/rs/zerolog/log" +) + +type GitHubScanOptions struct { + AccessToken string + Verbose bool + ConfidenceFilter []string + MaxScanGoRoutines int + TruffleHogVerification bool + MaxWorkflows int + Organization string + Owned bool + User string + Public bool + SearchQuery string + Artifacts bool + Context context.Context + Client *github.Client + HttpClient *http.Client +} + +var options = GitHubScanOptions{} + +func setupClient(accessToken string) *github.Client { + rateLimiter := github_ratelimit.New(nil, + github_primary_ratelimit.WithLimitDetectedCallback(func(ctx *github_primary_ratelimit.CallbackContext) { + resetTime := ctx.ResetTime.Add(time.Duration(time.Second * 30)) + log.Info().Str("category", string(ctx.Category)).Time("reset", resetTime).Msg("Primary rate limit detected, will resume automatically") + time.Sleep(time.Until(resetTime)) + log.Info().Str("category", string(ctx.Category)).Msg("Resuming") + }), + github_secondary_ratelimit.WithLimitDetectedCallback(func(ctx *github_secondary_ratelimit.CallbackContext) { + resetTime := ctx.ResetTime.Add(time.Duration(time.Second * 30)) + log.Info().Time("reset", *ctx.ResetTime).Dur("totalSleep", *ctx.TotalSleepTime).Msg("Secondary rate limit detected, will resume automatically") + time.Sleep(time.Until(resetTime)) + log.Info().Msg("Resuming") + }), + ) + + return github.NewClient(&http.Client{Transport: rateLimiter}).WithAuthToken(accessToken) +} \ No newline at end of file diff --git a/src/pipeleak/cmd/github/scan.go b/src/pipeleak/cmd/github/scan.go index 1b6b86ae..96f708dd 100644 --- a/src/pipeleak/cmd/github/scan.go +++ b/src/pipeleak/cmd/github/scan.go @@ -5,15 +5,10 @@ import ( "bytes" "context" "io" - "net/http" "sort" - "time" "github.com/CompassSecurity/pipeleak/helper" "github.com/CompassSecurity/pipeleak/scanner" - "github.com/gofri/go-github-ratelimit/v2/github_ratelimit" - "github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_primary_ratelimit" - "github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_secondary_ratelimit" "github.com/google/go-github/v69/github" "github.com/h2non/filetype" "github.com/rs/zerolog" @@ -22,29 +17,9 @@ import ( "github.com/wandb/parallel" ) -type GitHubScanOptions struct { - AccessToken string - Verbose bool - ConfidenceFilter []string - MaxScanGoRoutines int - TruffleHogVerification bool - MaxWorkflows int - Organization string - Owned bool - User string - Public bool - SearchQuery string - Artifacts bool - Context context.Context - Client *github.Client - HttpClient *http.Client -} - -var options = GitHubScanOptions{} - func NewScanCmd() *cobra.Command { scanCmd := &cobra.Command{ - Use: "scan [no options!]", + Use: "scan", Short: "Scan GitHub Actions", Long: `Scan GitHub Actions workflow runs and artifacts for secrets`, Example: ` @@ -99,25 +74,6 @@ func Scan(cmd *cobra.Command, args []string) { log.Info().Msg("Scan Finished, Bye Bye 🏳️‍🌈🔥") } -func setupClient(accessToken string) *github.Client { - rateLimiter := github_ratelimit.New(nil, - github_primary_ratelimit.WithLimitDetectedCallback(func(ctx *github_primary_ratelimit.CallbackContext) { - resetTime := ctx.ResetTime.Add(time.Duration(time.Second * 30)) - log.Info().Str("category", string(ctx.Category)).Time("reset", resetTime).Msg("Primary rate limit detected, will resume automatically") - time.Sleep(time.Until(resetTime)) - log.Info().Str("category", string(ctx.Category)).Msg("Resuming") - }), - github_secondary_ratelimit.WithLimitDetectedCallback(func(ctx *github_secondary_ratelimit.CallbackContext) { - resetTime := ctx.ResetTime.Add(time.Duration(time.Second * 30)) - log.Info().Time("reset", *ctx.ResetTime).Dur("totalSleep", *ctx.TotalSleepTime).Msg("Secondary rate limit detected, will resume automatically") - time.Sleep(time.Until(resetTime)) - log.Info().Msg("Resuming") - }), - ) - - return github.NewClient(&http.Client{Transport: rateLimiter}).WithAuthToken(accessToken) -} - func scan(client *github.Client) { if options.Owned { log.Info().Msg("Scanning authenticated user's owned repositories actions") From 5b6caf9610bee6ef66381e730c2f7da91d9c8a8b Mon Sep 17 00:00:00 2001 From: frjcomp <107982661+frjcomp@users.noreply.github.com> Date: Wed, 1 Oct 2025 14:28:51 +0000 Subject: [PATCH 02/13] trigger test joib --- .github/workflows/actions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 535229fe..83597749 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -2,7 +2,7 @@ name: Wait for Other Runs testing on: - workflow_dispatch: + push: jobs: scan-secrets: From 698b2f20758d36abe59684a6bcfd814736046c9e Mon Sep 17 00:00:00 2001 From: frjcomp <107982661+frjcomp@users.noreply.github.com> Date: Wed, 1 Oct 2025 14:32:12 +0000 Subject: [PATCH 03/13] added workflow permissions --- .github/workflows/actions.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 83597749..57619f0f 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -4,6 +4,10 @@ name: Wait for Other Runs testing on: push: +permissions: + actions: read + contents: read + jobs: scan-secrets: runs-on: ubuntu-latest From 59f88e5cad7860c4dee4c2ba68cced90bed9302f Mon Sep 17 00:00:00 2001 From: frjcomp <107982661+frjcomp@users.noreply.github.com> Date: Wed, 1 Oct 2025 14:38:15 +0000 Subject: [PATCH 04/13] debug --- src/pipeleak/cmd/github/action.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/pipeleak/cmd/github/action.go b/src/pipeleak/cmd/github/action.go index 15c84bb5..2934bb9f 100644 --- a/src/pipeleak/cmd/github/action.go +++ b/src/pipeleak/cmd/github/action.go @@ -54,15 +54,20 @@ func scanWorkflowRuns() { } owner, repo := parts[0], parts[1] + log.Info().Str("owner", owner).Str("repo", repo).Msg("Repository to scan") sha := os.Getenv("GITHUB_SHA") if sha == "" { log.Fatal().Msg("GITHUB_SHA not set") + } else { + log.Info().Str("sha", sha).Msg("Current commit sha") } runIDStr := os.Getenv("GITHUB_RUN_ID") if runIDStr == "" { log.Fatal().Msg("GITHUB_RUN_ID not set") + } else { + log.Info().Str("runID", runIDStr).Msg("Current run ID") } currentRunID, _ := strconv.ParseInt(runIDStr, 10, 64) From af1ec6722c9ca3f6fd9281941f7ce03c30cf47c9 Mon Sep 17 00:00:00 2001 From: frjcomp <107982661+frjcomp@users.noreply.github.com> Date: Wed, 1 Oct 2025 14:43:39 +0000 Subject: [PATCH 05/13] fix client --- src/pipeleak/cmd/github/action.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pipeleak/cmd/github/action.go b/src/pipeleak/cmd/github/action.go index 2934bb9f..130c76a7 100644 --- a/src/pipeleak/cmd/github/action.go +++ b/src/pipeleak/cmd/github/action.go @@ -36,15 +36,16 @@ func ScanAction(cmd *cobra.Command, args []string) { func scanWorkflowRuns() { log.Info().Msg("Scanning GitHub Actions workflow runs for secrets") ctx := context.WithValue(context.Background(), github.BypassRateLimitCheck, true) - client := setupClient(options.AccessToken) token := os.Getenv("GITHUB_TOKEN") if token == "" { log.Fatal().Msg("GITHUB_TOKEN not set") } + client := setupClient(token) + repoFull := os.Getenv("GITHUB_REPOSITORY") - if token == "" { + if repoFull == "" { log.Fatal().Msg("GITHUB_REPOSITORY not set") } From 903507ac705227fc6ede9b165289b2c6ff89e63e Mon Sep 17 00:00:00 2001 From: frjcomp <107982661+frjcomp@users.noreply.github.com> Date: Thu, 2 Oct 2025 07:32:58 +0000 Subject: [PATCH 06/13] wip --- .github/workflows/actions.yml | 2 +- src/pipeleak/cmd/github/action.go | 32 ++++- src/pipeleak/cmd/github/helper.go | 232 +++++++++++++++++++++++++++++- src/pipeleak/cmd/github/scan.go | 231 +---------------------------- 4 files changed, 263 insertions(+), 234 deletions(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 57619f0f..87494596 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -29,6 +29,6 @@ jobs: steps: - name: Wait 3 minutes, echo test, then wait 2 more minutes run: | - sleep 180 + sleep 20 echo test sleep 120 \ No newline at end of file diff --git a/src/pipeleak/cmd/github/action.go b/src/pipeleak/cmd/github/action.go index 130c76a7..bb559739 100644 --- a/src/pipeleak/cmd/github/action.go +++ b/src/pipeleak/cmd/github/action.go @@ -8,6 +8,7 @@ import ( "time" "github.com/CompassSecurity/pipeleak/helper" + "github.com/CompassSecurity/pipeleak/scanner" "github.com/google/go-github/v69/github" "github.com/rs/zerolog/log" "github.com/spf13/cobra" @@ -28,7 +29,9 @@ func NewActionScanCmd() *cobra.Command { } func ScanAction(cmd *cobra.Command, args []string) { + options.HttpClient = helper.GetPipeleakHTTPClient() helper.SetLogLevel(options.Verbose) + scanner.InitRules(options.ConfidenceFilter) scanWorkflowRuns() log.Info().Msg("Scan Finished, Bye Bye 🏳️‍🌈🔥") } @@ -43,6 +46,7 @@ func scanWorkflowRuns() { } client := setupClient(token) + options.Client = client repoFull := os.Getenv("GITHUB_REPOSITORY") if repoFull == "" { @@ -57,6 +61,11 @@ func scanWorkflowRuns() { owner, repo := parts[0], parts[1] log.Info().Str("owner", owner).Str("repo", repo).Msg("Repository to scan") + repository, _, err := client.Repositories.Get(ctx, owner, repo) + if err != nil { + log.Fatal().Err(err).Msg("Failed to fetch repository") + } + sha := os.Getenv("GITHUB_SHA") if sha == "" { log.Fatal().Msg("GITHUB_SHA not set") @@ -78,22 +87,36 @@ func scanWorkflowRuns() { opts := &github.ListWorkflowRunsOptions{ ListOptions: github.ListOptions{PerPage: 100}, + HeadSHA: sha, } for { runs, resp, err := client.Actions.ListRepositoryWorkflowRuns(ctx, owner, repo, opts) + + log.Info().Int("count", len(runs.WorkflowRuns)).Msg("Fetched workflow runs") + if err != nil { log.Fatal().Stack().Err(err).Msg("Failed listing workflow runs") } for _, run := range runs.WorkflowRuns { - log.Info().Int64("run", run.GetID()).Str("commit", run.GetHeadSHA()).Str("name", run.GetName()).Msg("Check run") - if run.GetHeadSHA() == sha && run.GetID() != currentRunID { + if run.GetID() != currentRunID { + log.Info().Int64("run", run.GetID()).Str("commit", run.GetHeadSHA()).Str("name", run.GetName()).Msg("Check run") status := run.GetStatus() log.Info().Int64("run", run.GetID()).Str("status", status).Str("name", run.GetName()).Msgf("Run") if status != "completed" { allCompleted = false + if _, scanned := scannedRuns[run.GetID()]; !scanned { + if status == "completed" { + scannedRuns[run.GetID()] = struct{}{} + wg.Add(1) + go func(runCopy *github.WorkflowRun) { + defer wg.Done() + scanRun(client, repository, runCopy) + }(run) + } + } } } } @@ -114,3 +137,8 @@ func scanWorkflowRuns() { time.Sleep(30 * time.Second) } } + +func scanRun(client *github.Client, repo *github.Repository, workflowRun *github.WorkflowRun) { + downloadWorkflowRunLog(client, repo, workflowRun) + listArtifacts(client, workflowRun) +} diff --git a/src/pipeleak/cmd/github/helper.go b/src/pipeleak/cmd/github/helper.go index 05e80639..c9d194f9 100644 --- a/src/pipeleak/cmd/github/helper.go +++ b/src/pipeleak/cmd/github/helper.go @@ -1,15 +1,21 @@ package github import ( + "archive/zip" + "bytes" "context" + "io" "net/http" "time" + "github.com/CompassSecurity/pipeleak/scanner" "github.com/gofri/go-github-ratelimit/v2/github_ratelimit" "github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_primary_ratelimit" "github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_secondary_ratelimit" "github.com/google/go-github/v69/github" + "github.com/h2non/filetype" "github.com/rs/zerolog/log" + "github.com/wandb/parallel" ) type GitHubScanOptions struct { @@ -49,4 +55,228 @@ func setupClient(accessToken string) *github.Client { ) return github.NewClient(&http.Client{Transport: rateLimiter}).WithAuthToken(accessToken) -} \ No newline at end of file +} + +func downloadWorkflowRunLog(client *github.Client, repo *github.Repository, workflowRun *github.WorkflowRun) { + logURL, resp, err := client.Actions.GetWorkflowRunLogs(options.Context, *repo.Owner.Login, *repo.Name, *workflowRun.ID, 5) + + if resp == nil { + log.Trace().Msg("downloadWorkflowRunLog Empty response") + return + } + + // already deleted, skip + if resp.StatusCode == 410 { + log.Debug().Str("workflowRunName", *workflowRun.Name).Msg("Skipped expired") + return + } else if resp.StatusCode == 404 { + return + } + + if err != nil { + log.Error().Stack().Err(err).Msg("Failed getting workflow run log URL") + return + } + + log.Trace().Msg("Downloading run log") + logs := downloadRunLogZIP(logURL.String()) + log.Trace().Msg("Finished downloading run log") + findings, err := scanner.DetectHits(logs, options.MaxScanGoRoutines, options.TruffleHogVerification) + if err != nil { + log.Debug().Err(err).Str("workflowRun", *workflowRun.HTMLURL).Msg("Failed detecting secrets") + return + } + + for _, finding := range findings { + log.Warn().Str("confidence", finding.Pattern.Pattern.Confidence).Str("ruleName", finding.Pattern.Pattern.Name).Str("value", finding.Text).Str("workflowRun", *workflowRun.HTMLURL).Msg("HIT") + } + log.Trace().Msg("Finished scannig run log") +} + +func iterateWorkflowRuns(client *github.Client, repo *github.Repository) { + opt := github.ListWorkflowRunsOptions{ + ListOptions: github.ListOptions{PerPage: 100}, + } + wfCount := 0 + for { + workflowRuns, resp, err := client.Actions.ListRepositoryWorkflowRuns(options.Context, *repo.Owner.Login, *repo.Name, &opt) + + if resp == nil { + log.Trace().Msg("Empty response due to rate limit, resume now<") + continue + } + + if resp.StatusCode == 404 { + return + } + + if err != nil { + log.Error().Stack().Err(err).Msg("Failed fetching workflow runs") + return + } + + for _, workflowRun := range workflowRuns.WorkflowRuns { + log.Debug().Str("name", *workflowRun.DisplayTitle).Str("url", *workflowRun.HTMLURL).Msg("Workflow run") + downloadWorkflowRunLog(client, repo, workflowRun) + + if options.Artifacts { + listArtifacts(client, workflowRun) + } + + wfCount = wfCount + 1 + if wfCount >= options.MaxWorkflows && options.MaxWorkflows > 0 { + log.Debug().Str("name", *workflowRun.DisplayTitle).Str("url", *workflowRun.HTMLURL).Msg("Reached MaxWorkflow runs, skip remaining") + return + } + } + + if resp.NextPage == 0 { + break + } + opt.Page = resp.NextPage + } +} + +func downloadRunLogZIP(url string) []byte { + res, err := options.HttpClient.Get(url) + logLines := make([]byte, 0) + + if err != nil { + return logLines + } + + if res.StatusCode == 200 { + body, err := io.ReadAll(res.Body) + if err != nil { + log.Err(err).Msg("Failed reading response log body") + return logLines + } + + zipReader, err := zip.NewReader(bytes.NewReader(body), int64(len(body))) + if err != nil { + log.Err(err).Msg("Failed creating zip reader") + return logLines + } + + for _, zipFile := range zipReader.File { + log.Trace().Str("zipFile", zipFile.Name).Msg("Zip file") + unzippedFileBytes, err := readZipFile(zipFile) + if err != nil { + log.Err(err).Msg("Failed reading zip file") + continue + } + + logLines = append(logLines, unzippedFileBytes...) + } + } + + return logLines +} + +func readZipFile(zf *zip.File) ([]byte, error) { + f, err := zf.Open() + if err != nil { + return nil, err + } + defer f.Close() + return io.ReadAll(f) +} + +func listArtifacts(client *github.Client, workflowRun *github.WorkflowRun) { + listOpt := github.ListOptions{PerPage: 100} + for { + artifactList, resp, err := client.Actions.ListWorkflowRunArtifacts(options.Context, *workflowRun.Repository.Owner.Login, *workflowRun.Repository.Name, *workflowRun.ID, &listOpt) + if resp == nil { + return + } + + if resp.StatusCode == 404 { + return + } + + if err != nil { + log.Error().Stack().Err(err).Msg("Failed fetching artifacts list") + return + } + + for _, artifact := range artifactList.Artifacts { + log.Debug().Str("name", *artifact.Name).Str("url", *artifact.ArchiveDownloadURL).Msg("Scan") + analyzeArtifact(client, workflowRun, artifact) + } + + if resp.NextPage == 0 { + break + } + listOpt.Page = resp.NextPage + } +} + +func analyzeArtifact(client *github.Client, workflowRun *github.WorkflowRun, artifact *github.Artifact) { + + url, resp, err := client.Actions.DownloadArtifact(options.Context, *workflowRun.Repository.Owner.Login, *workflowRun.Repository.Name, *artifact.ID, 5) + + if resp == nil { + log.Trace().Msg("analyzeArtifact Empty response") + return + } + + // already deleted, skip + if resp.StatusCode == 410 { + log.Debug().Str("workflowRunName", *workflowRun.Name).Msg("Skipped expired artifact") + return + } + + if err != nil { + log.Err(err).Msg("Failed getting artifact download URL") + return + } + + res, err := options.HttpClient.Get(url.String()) + + if err != nil { + log.Err(err).Str("workflow", url.String()).Msg("Failed downloading artifacts zip") + return + } + + if res.StatusCode == 200 { + body, err := io.ReadAll(res.Body) + if err != nil { + log.Err(err).Msg("Failed reading response log body") + return + } + zipListing, err := zip.NewReader(bytes.NewReader(body), int64(len(body))) + if err != nil { + log.Err(err).Str("url", url.String()).Msg("Failed creating zip reader") + return + } + + ctx := options.Context + group := parallel.Limited(ctx, options.MaxScanGoRoutines) + for _, file := range zipListing.File { + group.Go(func(ctx context.Context) { + fc, err := file.Open() + if err != nil { + log.Error().Stack().Err(err).Msg("Unable to open raw artifact zip file") + return + } + + content, err := io.ReadAll(fc) + if err != nil { + log.Error().Stack().Err(err).Msg("Unable to readAll artifact zip file") + return + } + + kind, _ := filetype.Match(content) + // do not scan https://pkg.go.dev/github.com/h2non/filetype#readme-supported-types + if kind == filetype.Unknown { + scanner.DetectFileHits(content, *workflowRun.HTMLURL, *workflowRun.Name, file.Name, "", options.TruffleHogVerification) + } else if filetype.IsArchive(content) { + scanner.HandleArchiveArtifact(file.Name, content, *workflowRun.HTMLURL, *workflowRun.Name, options.TruffleHogVerification) + } + fc.Close() + }) + } + + group.Wait() + } +} diff --git a/src/pipeleak/cmd/github/scan.go b/src/pipeleak/cmd/github/scan.go index 96f708dd..05b0c194 100644 --- a/src/pipeleak/cmd/github/scan.go +++ b/src/pipeleak/cmd/github/scan.go @@ -1,20 +1,15 @@ package github import ( - "archive/zip" - "bytes" "context" - "io" "sort" "github.com/CompassSecurity/pipeleak/helper" "github.com/CompassSecurity/pipeleak/scanner" "github.com/google/go-github/v69/github" - "github.com/h2non/filetype" "github.com/rs/zerolog" "github.com/rs/zerolog/log" "github.com/spf13/cobra" - "github.com/wandb/parallel" ) func NewScanCmd() *cobra.Command { @@ -255,131 +250,6 @@ func scanRepositories(client *github.Client) { } } -func iterateWorkflowRuns(client *github.Client, repo *github.Repository) { - opt := github.ListWorkflowRunsOptions{ - ListOptions: github.ListOptions{PerPage: 100}, - } - wfCount := 0 - for { - workflowRuns, resp, err := client.Actions.ListRepositoryWorkflowRuns(options.Context, *repo.Owner.Login, *repo.Name, &opt) - - if resp == nil { - log.Trace().Msg("Empty response due to rate limit, resume now<") - continue - } - - if resp.StatusCode == 404 { - return - } - - if err != nil { - log.Error().Stack().Err(err).Msg("Failed fetching workflow runs") - return - } - - for _, workflowRun := range workflowRuns.WorkflowRuns { - log.Debug().Str("name", *workflowRun.DisplayTitle).Str("url", *workflowRun.HTMLURL).Msg("Workflow run") - downloadWorkflowRunLog(client, repo, workflowRun) - - if options.Artifacts { - listArtifacts(client, workflowRun) - } - - wfCount = wfCount + 1 - if wfCount >= options.MaxWorkflows && options.MaxWorkflows > 0 { - log.Debug().Str("name", *workflowRun.DisplayTitle).Str("url", *workflowRun.HTMLURL).Msg("Reached MaxWorkflow runs, skip remaining") - return - } - } - - if resp.NextPage == 0 { - break - } - opt.Page = resp.NextPage - } -} - -func downloadWorkflowRunLog(client *github.Client, repo *github.Repository, workflowRun *github.WorkflowRun) { - logURL, resp, err := client.Actions.GetWorkflowRunLogs(options.Context, *repo.Owner.Login, *repo.Name, *workflowRun.ID, 5) - - if resp == nil { - log.Trace().Msg("downloadWorkflowRunLog Empty response") - return - } - - // already deleted, skip - if resp.StatusCode == 410 { - log.Debug().Str("workflowRunName", *workflowRun.Name).Msg("Skipped expired") - return - } else if resp.StatusCode == 404 { - return - } - - if err != nil { - log.Error().Stack().Err(err).Msg("Failed getting workflow run log URL") - return - } - - log.Trace().Msg("Downloading run log") - logs := downloadRunLogZIP(logURL.String()) - log.Trace().Msg("Finished downloading run log") - findings, err := scanner.DetectHits(logs, options.MaxScanGoRoutines, options.TruffleHogVerification) - if err != nil { - log.Debug().Err(err).Str("workflowRun", *workflowRun.HTMLURL).Msg("Failed detecting secrets") - return - } - - for _, finding := range findings { - log.Warn().Str("confidence", finding.Pattern.Pattern.Confidence).Str("ruleName", finding.Pattern.Pattern.Name).Str("value", finding.Text).Str("workflowRun", *workflowRun.HTMLURL).Msg("HIT") - } - log.Trace().Msg("Finished scannig run log") -} - -func downloadRunLogZIP(url string) []byte { - res, err := options.HttpClient.Get(url) - logLines := make([]byte, 0) - - if err != nil { - return logLines - } - - if res.StatusCode == 200 { - body, err := io.ReadAll(res.Body) - if err != nil { - log.Err(err).Msg("Failed reading response log body") - return logLines - } - - zipReader, err := zip.NewReader(bytes.NewReader(body), int64(len(body))) - if err != nil { - log.Err(err).Msg("Failed creating zip reader") - return logLines - } - - for _, zipFile := range zipReader.File { - log.Trace().Str("zipFile", zipFile.Name).Msg("Zip file") - unzippedFileBytes, err := readZipFile(zipFile) - if err != nil { - log.Err(err).Msg("Failed reading zip file") - continue - } - - logLines = append(logLines, unzippedFileBytes...) - } - } - - return logLines -} - -func readZipFile(zf *zip.File) ([]byte, error) { - f, err := zf.Open() - if err != nil { - return nil, err - } - defer f.Close() - return io.ReadAll(f) -} - func identifyNewestPublicProjectId(client *github.Client) int64 { for { listOpts := github.ListOptions{PerPage: 1000} @@ -408,103 +278,4 @@ func identifyNewestPublicProjectId(client *github.Client) int64 { log.Fatal().Msg("Failed finding a CreateEvent and thus no rerpository id") return -1 -} - -func listArtifacts(client *github.Client, workflowRun *github.WorkflowRun) { - listOpt := github.ListOptions{PerPage: 100} - for { - artifactList, resp, err := client.Actions.ListWorkflowRunArtifacts(options.Context, *workflowRun.Repository.Owner.Login, *workflowRun.Repository.Name, *workflowRun.ID, &listOpt) - if resp == nil { - return - } - - if resp.StatusCode == 404 { - return - } - - if err != nil { - log.Error().Stack().Err(err).Msg("Failed fetching artifacts list") - return - } - - for _, artifact := range artifactList.Artifacts { - log.Debug().Str("name", *artifact.Name).Str("url", *artifact.ArchiveDownloadURL).Msg("Scan") - analyzeArtifact(client, workflowRun, artifact) - } - - if resp.NextPage == 0 { - break - } - listOpt.Page = resp.NextPage - } -} - -func analyzeArtifact(client *github.Client, workflowRun *github.WorkflowRun, artifact *github.Artifact) { - - url, resp, err := client.Actions.DownloadArtifact(options.Context, *workflowRun.Repository.Owner.Login, *workflowRun.Repository.Name, *artifact.ID, 5) - - if resp == nil { - log.Trace().Msg("analyzeArtifact Empty response") - return - } - - // already deleted, skip - if resp.StatusCode == 410 { - log.Debug().Str("workflowRunName", *workflowRun.Name).Msg("Skipped expired artifact") - return - } - - if err != nil { - log.Err(err).Msg("Failed getting artifact download URL") - return - } - - res, err := options.HttpClient.Get(url.String()) - - if err != nil { - log.Err(err).Str("workflow", url.String()).Msg("Failed downloading artifacts zip") - return - } - - if res.StatusCode == 200 { - body, err := io.ReadAll(res.Body) - if err != nil { - log.Err(err).Msg("Failed reading response log body") - return - } - zipListing, err := zip.NewReader(bytes.NewReader(body), int64(len(body))) - if err != nil { - log.Err(err).Str("url", url.String()).Msg("Failed creating zip reader") - return - } - - ctx := options.Context - group := parallel.Limited(ctx, options.MaxScanGoRoutines) - for _, file := range zipListing.File { - group.Go(func(ctx context.Context) { - fc, err := file.Open() - if err != nil { - log.Error().Stack().Err(err).Msg("Unable to open raw artifact zip file") - return - } - - content, err := io.ReadAll(fc) - if err != nil { - log.Error().Stack().Err(err).Msg("Unable to readAll artifact zip file") - return - } - - kind, _ := filetype.Match(content) - // do not scan https://pkg.go.dev/github.com/h2non/filetype#readme-supported-types - if kind == filetype.Unknown { - scanner.DetectFileHits(content, *workflowRun.HTMLURL, *workflowRun.Name, file.Name, "", options.TruffleHogVerification) - } else if filetype.IsArchive(content) { - scanner.HandleArchiveArtifact(file.Name, content, *workflowRun.HTMLURL, *workflowRun.Name, options.TruffleHogVerification) - } - fc.Close() - }) - } - - group.Wait() - } -} +} \ No newline at end of file From 37e2052ebbc3082f5a82558c535e7d7faedfd16f Mon Sep 17 00:00:00 2001 From: frjcomp <107982661+frjcomp@users.noreply.github.com> Date: Thu, 2 Oct 2025 12:26:13 +0000 Subject: [PATCH 07/13] run scan --- src/pipeleak/cmd/github/action.go | 39 ++++++++++++++----------------- src/pipeleak/cmd/github/scan.go | 2 +- 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/src/pipeleak/cmd/github/action.go b/src/pipeleak/cmd/github/action.go index bb559739..b5a4fb00 100644 --- a/src/pipeleak/cmd/github/action.go +++ b/src/pipeleak/cmd/github/action.go @@ -5,6 +5,7 @@ import ( "os" "strconv" "strings" + "sync" "time" "github.com/CompassSecurity/pipeleak/helper" @@ -40,6 +41,9 @@ func scanWorkflowRuns() { log.Info().Msg("Scanning GitHub Actions workflow runs for secrets") ctx := context.WithValue(context.Background(), github.BypassRateLimitCheck, true) + var wg sync.WaitGroup + scannedRuns := make(map[int64]struct{}) + token := os.Getenv("GITHUB_TOKEN") if token == "" { log.Fatal().Msg("GITHUB_TOKEN not set") @@ -83,8 +87,6 @@ func scanWorkflowRuns() { currentRunID, _ := strconv.ParseInt(runIDStr, 10, 64) for { - allCompleted := true - opts := &github.ListWorkflowRunsOptions{ ListOptions: github.ListOptions{PerPage: 100}, HeadSHA: sha, @@ -101,21 +103,18 @@ func scanWorkflowRuns() { for _, run := range runs.WorkflowRuns { if run.GetID() != currentRunID { - log.Info().Int64("run", run.GetID()).Str("commit", run.GetHeadSHA()).Str("name", run.GetName()).Msg("Check run") status := run.GetStatus() - log.Info().Int64("run", run.GetID()).Str("status", status).Str("name", run.GetName()).Msgf("Run") + log.Info().Int64("run", run.GetID()).Str("status", status).Str("name", run.GetName()).Msgf("Running workflow run") - if status != "completed" { - allCompleted = false + if status == "completed" { if _, scanned := scannedRuns[run.GetID()]; !scanned { - if status == "completed" { - scannedRuns[run.GetID()] = struct{}{} - wg.Add(1) - go func(runCopy *github.WorkflowRun) { - defer wg.Done() - scanRun(client, repository, runCopy) - }(run) - } + scannedRuns[run.GetID()] = struct{}{} + wg.Add(1) + go func(runCopy *github.WorkflowRun) { + defer wg.Done() + log.Warn().Int64("run", run.GetID()).Str("status", status).Str("name", run.GetName()).Msgf("Running scan for workflow run") + scanRun(client, repository, runCopy) + }(run) } } } @@ -125,16 +124,12 @@ func scanWorkflowRuns() { break } - opts.Page = resp.NextPage - } - - if allCompleted { - log.Info().Msg("✅ All *other* runs for this commit are completed") - break + log.Info().Msg("⏳ Some runs are still running") + time.Sleep(3 * time.Second) } - log.Info().Msg("⏳ Still waiting... retrying in 30s") - time.Sleep(30 * time.Second) + log.Info().Msg("⏳ Waiting for any remaining scans to finish...") + wg.Wait() } } diff --git a/src/pipeleak/cmd/github/scan.go b/src/pipeleak/cmd/github/scan.go index 05b0c194..188cfccd 100644 --- a/src/pipeleak/cmd/github/scan.go +++ b/src/pipeleak/cmd/github/scan.go @@ -278,4 +278,4 @@ func identifyNewestPublicProjectId(client *github.Client) int64 { log.Fatal().Msg("Failed finding a CreateEvent and thus no rerpository id") return -1 -} \ No newline at end of file +} From e7d004cde7d2a05cb2242f98096adf5ba8df8eae Mon Sep 17 00:00:00 2001 From: frjcomp <107982661+frjcomp@users.noreply.github.com> Date: Thu, 2 Oct 2025 12:30:47 +0000 Subject: [PATCH 08/13] fix nil pinter --- src/pipeleak/cmd/github/action.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pipeleak/cmd/github/action.go b/src/pipeleak/cmd/github/action.go index b5a4fb00..375c7833 100644 --- a/src/pipeleak/cmd/github/action.go +++ b/src/pipeleak/cmd/github/action.go @@ -39,7 +39,7 @@ func ScanAction(cmd *cobra.Command, args []string) { func scanWorkflowRuns() { log.Info().Msg("Scanning GitHub Actions workflow runs for secrets") - ctx := context.WithValue(context.Background(), github.BypassRateLimitCheck, true) + options.Context = context.WithValue(context.Background(), github.BypassRateLimitCheck, true) var wg sync.WaitGroup scannedRuns := make(map[int64]struct{}) @@ -65,7 +65,7 @@ func scanWorkflowRuns() { owner, repo := parts[0], parts[1] log.Info().Str("owner", owner).Str("repo", repo).Msg("Repository to scan") - repository, _, err := client.Repositories.Get(ctx, owner, repo) + repository, _, err := client.Repositories.Get(options.Context, owner, repo) if err != nil { log.Fatal().Err(err).Msg("Failed to fetch repository") } @@ -93,7 +93,7 @@ func scanWorkflowRuns() { } for { - runs, resp, err := client.Actions.ListRepositoryWorkflowRuns(ctx, owner, repo, opts) + runs, resp, err := client.Actions.ListRepositoryWorkflowRuns(options.Context, owner, repo, opts) log.Info().Int("count", len(runs.WorkflowRuns)).Msg("Fetched workflow runs") @@ -104,7 +104,7 @@ func scanWorkflowRuns() { for _, run := range runs.WorkflowRuns { if run.GetID() != currentRunID { status := run.GetStatus() - log.Info().Int64("run", run.GetID()).Str("status", status).Str("name", run.GetName()).Msgf("Running workflow run") + log.Info().Int64("run", run.GetID()).Str("status", status).Str("name", run.GetName()).Str("url", *run.URL).Msgf("Workflow run") if status == "completed" { if _, scanned := scannedRuns[run.GetID()]; !scanned { From 5968daf4e829d2465513b939551fa68b24dd0039 Mon Sep 17 00:00:00 2001 From: frjcomp <107982661+frjcomp@users.noreply.github.com> Date: Thu, 2 Oct 2025 12:41:18 +0000 Subject: [PATCH 09/13] end the game --- src/pipeleak/cmd/github/action.go | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/pipeleak/cmd/github/action.go b/src/pipeleak/cmd/github/action.go index 375c7833..a5c4a58c 100644 --- a/src/pipeleak/cmd/github/action.go +++ b/src/pipeleak/cmd/github/action.go @@ -85,8 +85,10 @@ func scanWorkflowRuns() { } currentRunID, _ := strconv.ParseInt(runIDStr, 10, 64) + allRunsCompleted := true for { + opts := &github.ListWorkflowRunsOptions{ ListOptions: github.ListOptions{PerPage: 100}, HeadSHA: sha, @@ -102,9 +104,10 @@ func scanWorkflowRuns() { } for _, run := range runs.WorkflowRuns { + if run.GetID() != currentRunID { status := run.GetStatus() - log.Info().Int64("run", run.GetID()).Str("status", status).Str("name", run.GetName()).Str("url", *run.URL).Msgf("Workflow run") + log.Info().Int64("run", run.GetID()).Str("status", status).Str("name", run.GetName()).Str("url", *run.HTMLURL).Msgf("Workflow run") if status == "completed" { if _, scanned := scannedRuns[run.GetID()]; !scanned { @@ -112,25 +115,35 @@ func scanWorkflowRuns() { wg.Add(1) go func(runCopy *github.WorkflowRun) { defer wg.Done() - log.Warn().Int64("run", run.GetID()).Str("status", status).Str("name", run.GetName()).Msgf("Running scan for workflow run") + log.Warn().Int64("run", run.GetID()).Str("status", status).Str("name", run.GetName()).Str("url", *run.HTMLURL).Msgf("Running scan for workflow run") scanRun(client, repository, runCopy) }(run) } } } + + if *run.Status != "completed" { + allRunsCompleted = false + } } if resp.NextPage == 0 { break } + opts.Page = resp.NextPage + } + + if allRunsCompleted { + log.Info().Msg("⏳ Waiting for any remaining scans to finish...") + break + } else { log.Info().Msg("⏳ Some runs are still running") time.Sleep(3 * time.Second) } - - log.Info().Msg("⏳ Waiting for any remaining scans to finish...") - wg.Wait() } + + wg.Wait() } func scanRun(client *github.Client, repo *github.Repository, workflowRun *github.WorkflowRun) { From f2e6d574505de6329e37bf5cfadcb28f656b45f0 Mon Sep 17 00:00:00 2001 From: frjcomp <107982661+frjcomp@users.noreply.github.com> Date: Thu, 2 Oct 2025 13:55:50 +0000 Subject: [PATCH 10/13] end --- src/pipeleak/cmd/github/action.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/pipeleak/cmd/github/action.go b/src/pipeleak/cmd/github/action.go index a5c4a58c..2af1553a 100644 --- a/src/pipeleak/cmd/github/action.go +++ b/src/pipeleak/cmd/github/action.go @@ -104,9 +104,12 @@ func scanWorkflowRuns() { } for _, run := range runs.WorkflowRuns { + status := run.GetStatus() + if status != "completed" { + allRunsCompleted = false + } if run.GetID() != currentRunID { - status := run.GetStatus() log.Info().Int64("run", run.GetID()).Str("status", status).Str("name", run.GetName()).Str("url", *run.HTMLURL).Msgf("Workflow run") if status == "completed" { @@ -121,10 +124,6 @@ func scanWorkflowRuns() { } } } - - if *run.Status != "completed" { - allRunsCompleted = false - } } if resp.NextPage == 0 { From 8ee92ee1f6871b9478ca742d71b9dbbc821c55bc Mon Sep 17 00:00:00 2001 From: frjcomp <107982661+frjcomp@users.noreply.github.com> Date: Thu, 2 Oct 2025 14:06:05 +0000 Subject: [PATCH 11/13] end correctly --- src/pipeleak/cmd/github/action.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipeleak/cmd/github/action.go b/src/pipeleak/cmd/github/action.go index 2af1553a..e1050f15 100644 --- a/src/pipeleak/cmd/github/action.go +++ b/src/pipeleak/cmd/github/action.go @@ -85,9 +85,9 @@ func scanWorkflowRuns() { } currentRunID, _ := strconv.ParseInt(runIDStr, 10, 64) - allRunsCompleted := true for { + allRunsCompleted := true opts := &github.ListWorkflowRunsOptions{ ListOptions: github.ListOptions{PerPage: 100}, From 1f7d89d0553bde1b9b51967e2e45e6285171936c Mon Sep 17 00:00:00 2001 From: frjcomp <107982661+frjcomp@users.noreply.github.com> Date: Thu, 2 Oct 2025 14:17:20 +0000 Subject: [PATCH 12/13] try again --- src/pipeleak/cmd/github/action.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/pipeleak/cmd/github/action.go b/src/pipeleak/cmd/github/action.go index e1050f15..9aff4f2c 100644 --- a/src/pipeleak/cmd/github/action.go +++ b/src/pipeleak/cmd/github/action.go @@ -85,15 +85,14 @@ func scanWorkflowRuns() { } currentRunID, _ := strconv.ParseInt(runIDStr, 10, 64) + opts := &github.ListWorkflowRunsOptions{ + ListOptions: github.ListOptions{PerPage: 100}, + HeadSHA: sha, + } for { allRunsCompleted := true - opts := &github.ListWorkflowRunsOptions{ - ListOptions: github.ListOptions{PerPage: 100}, - HeadSHA: sha, - } - for { runs, resp, err := client.Actions.ListRepositoryWorkflowRuns(options.Context, owner, repo, opts) @@ -129,8 +128,7 @@ func scanWorkflowRuns() { if resp.NextPage == 0 { break } - - opts.Page = resp.NextPage + opt.Page = resp.NextPage } if allRunsCompleted { From 37dcbd8db8f3137cb59b89499b50ca5b02854eab Mon Sep 17 00:00:00 2001 From: frjcomp <107982661+frjcomp@users.noreply.github.com> Date: Thu, 2 Oct 2025 14:19:16 +0000 Subject: [PATCH 13/13] brr --- src/pipeleak/cmd/github/action.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipeleak/cmd/github/action.go b/src/pipeleak/cmd/github/action.go index 9aff4f2c..0f01298d 100644 --- a/src/pipeleak/cmd/github/action.go +++ b/src/pipeleak/cmd/github/action.go @@ -128,7 +128,7 @@ func scanWorkflowRuns() { if resp.NextPage == 0 { break } - opt.Page = resp.NextPage + opts.Page = resp.NextPage } if allRunsCompleted {