feat: add gitlab/github provider API retries

config/302-pac-configmap.yaml

@@ -160,6 +160,27 @@ data:
   # Default: false
   require-ok-to-test-sha: "false"
 
+  # Enable retrying Git provider API requests when hitting rate limits (429,
+  # or 403 with rate limit headers on GitHub) or transient server errors.
+  # Retries use exponential backoff with jitter and honor the Retry-After and
+  # X-RateLimit-Reset headers, giving up when the reset is further away than
+  # api-retry-max-wait-seconds.
+  # When false, the GitLab client keeps the retry behaviour of the upstream
+  # GitLab Go client and the GitHub client does not retry.
+  # Default: false
+  enable-api-retry: "false"
+
+  # Maximum number of attempts (initial request included) when enable-api-retry
+  # is true. Values lower than 1 fall back to the default.
+  # Default: 4
+  api-retry-max-attempts: "4"
+
+  # Maximum time in seconds to wait between retries when enable-api-retry is
+  # true. If the provider rate limit reset is further away than this, the
+  # request fails instead of blocking.
+  # Default: 120
+  api-retry-max-wait-seconds: "120"
+
   # When enabled, this option prevents duplicate pipeline runs when a commit appears in
   # both a push event and a pull request. If a push event comes from a commit that is
   # part of an open pull request, the push event will be skipped as it would create

docs/content/docs/api/configmap.md

@@ -339,6 +339,72 @@ skip-push-event-for-pr-commits: "true"
 
 {{< /param >}}
 
+### API Retry
+
+{{< tech_preview "Provider API Retry for GitHub and GitLab" >}}
+
+{{< param name="enable-api-retry" type="boolean" default="false" id="param-enable-api-retry" >}}
+Enables retrying GitHub and GitLab API requests when Pipelines-as-Code
+encounters a temporary provider failure. This includes rate limits, temporary
+server errors, and selected network failures.
+
+Retries use backoff with jitter so multiple requests do not all retry at the
+same time. When the provider supplies a retry or reset time, Pipelines-as-Code
+uses that information as long as it does not exceed
+`api-retry-max-wait-seconds`.
+
+The setting is disabled by default. Enabling it affects API operations made
+while processing an event, including temporary clients and GitHub App setup.
+
+When disabled, the GitLab client keeps the retry behaviour built into the
+upstream GitLab Go client, and the GitHub client performs no retries.
+
+Pipelines-as-Code only repeats an operation when it can do so safely. It does
+not repeat provider changes after an uncertain network or server failure when
+doing so could create duplicate comments, statuses, or other mutations.
+
+```yaml
+enable-api-retry: "false"
+```
+
+{{< /param >}}
+
+{{< param name="api-retry-max-attempts" type="integer" default="4" id="param-api-retry-max-attempts" >}}
+Sets the maximum number of attempts when `enable-api-retry` is `true`. The
+initial request counts as the first attempt. For example, a value of `4`
+allows the initial request followed by up to three retries.
+
+```yaml
+api-retry-max-attempts: "4"
+```
+
+{{< /param >}}
+
+{{< param name="api-retry-max-wait-seconds" type="integer" default="120" id="param-api-retry-max-wait-seconds" >}}
+Sets the maximum time in seconds that Pipelines-as-Code waits between
+attempts. If a provider asks the client to wait longer than this value,
+Pipelines-as-Code stops retrying rather than holding the event for a long
+cooling period.
+
+```yaml
+api-retry-max-wait-seconds: "120"
+```
+
+{{< /param >}}
+
+#### Failure reporting and limitations
+
+If all attempts fail, Pipelines-as-Code reports the original provider error
+through its normal logs, events, and provider status handling. Reporting a
+failure status to GitHub or GitLab is best effort because a fully exhausted or
+unavailable provider API might also reject the status update.
+
+These settings are intended for short, temporary provider failures. They do
+not provide persistent queueing for long rate-limit windows and do not control
+how quickly a large backlog of PipelineRuns is admitted to the cluster.
+Long-duration queueing and workload admission should be handled separately at
+the pipeline or platform level.
+
 ## Complete Example
 
 ```yaml

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/google/go-github/v85 v85.0.0
 	github.com/hako/durafmt v0.0.0-20210608085754-5c1018a4e16b
 	github.com/jenkins-x/go-scm v1.15.32
+	github.com/hashicorp/go-retryablehttp v0.7.8
 	github.com/jonboulle/clockwork v0.5.0
 	github.com/juju/ansiterm v1.0.0
 	github.com/ktrysmt/go-bitbucket v0.10.0
@@ -127,7 +128,6 @@ require (
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
-	github.com/hashicorp/go-retryablehttp v0.7.8 // indirect
 	github.com/hashicorp/go-version v1.9.0 // indirect
 	github.com/hashicorp/golang-lru v1.0.2 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect

pkg/adapter/incoming.go

@@ -176,6 +176,10 @@ func (l *listener) detectIncoming(ctx context.Context, event *info.Event, req *h
 	if repo.Spec.GitProvider == nil || repo.Spec.GitProvider.Type == "" {
 		gh := github.New()
 		gh.Run = l.run
+		if l.run.Info.Pac != nil {
+			pacInfo := l.run.Info.GetPacOpts()
+			gh.SetPacInfo(&pacInfo)
+		}
 		ns := info.GetNS(ctx)
 		ip := app.NewInstallation(req, l.run, repo, gh, ns)
 		enterpriseURL, token, installationID, err := ip.GetAndUpdateInstallationID(ctx)

pkg/params/settings/config.go

@@ -25,6 +25,13 @@ const (
 	CustomConsoleNamespaceURLKey = "custom-console-url-namespace"
 
 	SecretGhAppTokenRepoScopedKey = "secret-github-app-token-scoped" //nolint: gosec
+
+	// DefaultAPIRetryMaxAttempts is the fallback number of attempts (initial
+	// request included) used when api-retry-max-attempts is unset or invalid.
+	DefaultAPIRetryMaxAttempts = 4
+	// DefaultAPIRetryMaxWaitSeconds is the fallback cap on the wait between
+	// attempts used when api-retry-max-wait-seconds is unset or invalid.
+	DefaultAPIRetryMaxWaitSeconds = 120
 )
 
 var (
@@ -78,6 +85,12 @@ type Settings struct {
 	RememberOKToTest   bool `json:"remember-ok-to-test"`
 	RequireOkToTestSHA bool `json:"require-ok-to-test-sha"`
 
+	// Retry Git provider API requests on rate limits and transient errors.
+	// Disabled by default.
+	EnableAPIRetry         bool `default:"false" json:"enable-api-retry"`
+	APIRetryMaxAttempts    int  `default:"4"     json:"api-retry-max-attempts"`
+	APIRetryMaxWaitSeconds int  `default:"120"   json:"api-retry-max-wait-seconds"`
+
 	// Tracing label names. Defaults in config/302-pac-configmap.yaml.
 	TracingLabelAction      string `json:"tracing-label-action"`
 	TracingLabelApplication string `json:"tracing-label-application"`

pkg/params/settings/config_test.go

@@ -48,6 +48,8 @@ func TestSyncConfig(t *testing.T) {
 				CustomConsolePRTaskLog:               "",
 				CustomConsoleNamespaceURL:            "",
 				RememberOKToTest:                     false,
+				APIRetryMaxAttempts:                  4,
+				APIRetryMaxWaitSeconds:               120,
 			},
 		},
 		{
@@ -109,6 +111,8 @@ func TestSyncConfig(t *testing.T) {
 				CustomConsoleNamespaceURL:            "https://custom-console-namespace",
 				RememberOKToTest:                     false,
 				RequireOkToTestSHA:                   true,
+				APIRetryMaxAttempts:                  4,
+				APIRetryMaxWaitSeconds:               120,
 			},
 		},
 		{

pkg/provider/github/app/token.go

@@ -74,7 +74,7 @@ func (ip *Install) GetAndUpdateInstallationID(ctx context.Context) (string, stri
 		}
 	}
 
-	client, _, _, err := github.MakeClient(ctx, apiURL, jwtToken)
+	client, _, _, err := ip.ghClient.MakeClient(ctx, apiURL, jwtToken)
 	if err != nil {
 		return "", "", 0, err
 	}

pkg/provider/github/github.go

@@ -26,6 +26,7 @@ import (
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/params/info"
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/params/triggertype"
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/provider"
+	"github.com/openshift-pipelines/pipelines-as-code/pkg/provider/retryhttp"
 	"go.uber.org/zap"
 	"golang.org/x/oauth2"
 	"k8s.io/client-go/kubernetes"
@@ -120,6 +121,19 @@ func (v *Provider) SetPacInfo(pacInfo *info.PacOpts) {
 	v.pacInfo = pacInfo
 }
 
+// retryOptions returns retry transport options built from the pac settings,
+// or nil when API retries are disabled (the default).
+func (v *Provider) retryOptions() *retryhttp.Options {
+	if v.pacInfo == nil || !v.pacInfo.EnableAPIRetry {
+		return nil
+	}
+	return &retryhttp.Options{
+		MaxAttempts: v.pacInfo.APIRetryMaxAttempts,
+		MaxWait:     time.Duration(v.pacInfo.APIRetryMaxWaitSeconds) * time.Second,
+		Logger:      v.Logger,
+	}
+}
+
 // detectGHERawURL Detect if we have a raw URL in GHE.
 func detectGHERawURL(event *info.Event, taskHost string) bool {
 	gheURL, err := url.Parse(event.GHEURL)
@@ -235,13 +249,16 @@ func (v *Provider) GetConfig() *info.ProviderConfig {
 	}
 }
 
-func MakeClient(ctx context.Context, apiURL, token string) (*github.Client, string, *string, error) {
+func MakeClient(ctx context.Context, apiURL, token string, retryOpts ...*retryhttp.Options) (*github.Client, string, *string, error) {
 	var client *github.Client
 	ts := oauth2.StaticTokenSource(
 		&oauth2.Token{AccessToken: token},
 	)
 
 	tc := oauth2.NewClient(ctx, ts)
+	if len(retryOpts) > 0 && retryOpts[0] != nil {
+		tc.Transport = retryhttp.Wrap(tc.Transport, *retryOpts[0])
+	}
 	if apiURL != "" {
 		if !strings.HasPrefix(apiURL, "https") && !strings.HasPrefix(apiURL, "http") {
 			apiURL = "https://" + apiURL
@@ -265,6 +282,11 @@ func MakeClient(ctx context.Context, apiURL, token string) (*github.Client, stri
 	return client, providerName, github.Ptr(apiURL), nil
 }
 
+// MakeClient creates a GitHub API client using the provider retry settings.
+func (v *Provider) MakeClient(ctx context.Context, apiURL, token string) (*github.Client, string, *string, error) {
+	return MakeClient(ctx, apiURL, token, v.retryOptions())
+}
+
 func parseTS(headerTS string) (time.Time, error) {
 	ts := time.Time{}
 	// Normal UTC: 2023-01-31 23:00:00 UTC
@@ -326,7 +348,7 @@ func (v *Provider) checkWebhookSecretValidity(ctx context.Context, cw clockwork.
 }
 
 func (v *Provider) SetClient(ctx context.Context, run *params.Run, event *info.Event, repo *v1alpha1.Repository, eventsEmitter *events.EventEmitter) error {
-	client, providerName, apiURL, err := MakeClient(ctx, event.Provider.URL, event.Provider.Token)
+	client, providerName, apiURL, err := v.MakeClient(ctx, event.Provider.URL, event.Provider.Token)
 	if err != nil {
 		return err
 	}
@@ -1244,7 +1266,7 @@ func (v *Provider) fetchAppSlug(ctx context.Context, apiURL string) (string, err
 		return "", err
 	}
 
-	client, _, _, err := MakeClient(ctx, apiURL, tokenString)
+	client, _, _, err := v.MakeClient(ctx, apiURL, tokenString)
 	if err != nil {
 		return "", err
 	}

pkg/provider/github/parse_payload.go

@@ -25,6 +25,7 @@ import (
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/params/info"
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/params/triggertype"
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/provider"
+	"github.com/openshift-pipelines/pipelines-as-code/pkg/provider/retryhttp"
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/secrets"
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/sort"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -90,15 +91,21 @@ func (v *Provider) GetAppToken(ctx context.Context, kube kubernetes.Interface, g
 		gheURL = strings.TrimSuffix(reqTokenURL, "/api/v3")
 	}
 
+	// wrap the installation transport with the retry transport when enabled
+	var apiTransport http.RoundTripper = itr
+	if retryOpts := v.retryOptions(); retryOpts != nil {
+		apiTransport = retryhttp.Wrap(itr, *retryOpts)
+	}
+
 	if gheURL != "" {
 		if !strings.HasPrefix(gheURL, "https://") && !strings.HasPrefix(gheURL, "http://") {
 			gheURL = "https://" + gheURL
 		}
 		uploadURL := gheURL + "/api/uploads"
-		v.ghClient, _ = github.NewClient(&http.Client{Transport: itr}).WithEnterpriseURLs(gheURL, uploadURL)
+		v.ghClient, _ = github.NewClient(&http.Client{Transport: apiTransport}).WithEnterpriseURLs(gheURL, uploadURL)
 		itr.BaseURL = strings.TrimSuffix(v.Client().BaseURL.String(), "/")
 	} else {
-		v.ghClient = github.NewClient(&http.Client{Transport: itr})
+		v.ghClient = github.NewClient(&http.Client{Transport: apiTransport})
 	}
 
 	// Get a token ASAP because we need it for setting private repos

pkg/provider/github/retry_test.go

@@ -0,0 +1,56 @@
+package github
+
+import (
+	"testing"
+	"time"
+
+	"github.com/openshift-pipelines/pipelines-as-code/pkg/params/info"
+	"github.com/openshift-pipelines/pipelines-as-code/pkg/params/settings"
+	"gotest.tools/v3/assert"
+)
+
+func TestRetryOptions(t *testing.T) {
+	tests := []struct {
+		name            string
+		pacInfo         *info.PacOpts
+		wantNil         bool
+		wantMaxAttempts int
+		wantMaxWait     time.Duration
+	}{
+		{
+			name:    "nil pacinfo",
+			pacInfo: nil,
+			wantNil: true,
+		},
+		{
+			name:    "disabled by default",
+			pacInfo: &info.PacOpts{Settings: settings.DefaultSettings()},
+			wantNil: true,
+		},
+		{
+			name: "enabled with settings",
+			pacInfo: &info.PacOpts{
+				Settings: settings.Settings{
+					EnableAPIRetry:         true,
+					APIRetryMaxAttempts:    7,
+					APIRetryMaxWaitSeconds: 42,
+				},
+			},
+			wantMaxAttempts: 7,
+			wantMaxWait:     42 * time.Second,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			v := &Provider{pacInfo: tt.pacInfo}
+			opts := v.retryOptions()
+			if tt.wantNil {
+				assert.Assert(t, opts == nil)
+				return
+			}
+			assert.Assert(t, opts != nil)
+			assert.Equal(t, tt.wantMaxAttempts, opts.MaxAttempts)
+			assert.Equal(t, tt.wantMaxWait, opts.MaxWait)
+		})
+	}
+}