backend: ignore requests for etf2l user 0, add ratelimiting and retries to rgl client

Author: CondensedTea

backend/cmd/offi/serve.go

@@ -54,7 +54,7 @@ func serveAction(ctx context.Context, _ *cli.Command) error {
 	retryTransport := internalHTTP.Transport(true)
 
 	etf2lClient := etf2l.New(retryTransport)
-	rglClient := rgl.NewClient(defaultTransport)
+	rglClient := rgl.NewClient(retryTransport)
 
 	logsClient := logstf.NewClient(defaultTransport)
 	demosClient := demostf.NewClient(defaultTransport)

backend/internal/cache/redis_players.go

@@ -58,7 +58,7 @@ func (r Redis) GetPlayers(ctx context.Context, league string, playerIDs []int64)
 		} else {
 			var player Player
 			if err = json.Unmarshal([]byte(result.(string)), &player); err != nil {
-				return nil, fmt.Errorf("unmarshalling player: %w", err)
+				return nil, fmt.Errorf("unmarshaling player: %w", err)
 			}
 
 			players[playerIDs[i]] = &player

backend/internal/etf2l/etf2l.go

@@ -20,7 +20,7 @@ func New(rt http.RoundTripper) *Client {
 	return &Client{
 		apiURL:     "https://api-v2.etf2l.org",
 		httpClient: &http.Client{Transport: rt},
-		limiter:    rate.NewLimiter(rate.Every(time.Second), 6),
+		limiter:    rate.NewLimiter(rate.Every(time.Second), 5),
 		tracer:     otel.Tracer("etf2l"),
 	}
 }

backend/internal/http/transport.go

@@ -1,9 +1,13 @@
 package http
 
 import (
+	"log/slog"
+	"math"
 	"net/http"
 	info "offi/internal/build_info"
 	"offi/internal/tracing"
+	"strconv"
+	"time"
 
 	"github.com/go-chi/transport"
 )
@@ -17,8 +21,87 @@ func Transport(withRetries bool) http.RoundTripper {
 	}
 
 	if withRetries {
-		base = append(base, transport.Retry(transport.Chain(http.DefaultTransport, uaTransport), 3))
+		base = append(base, Retry(transport.Chain(http.DefaultTransport, uaTransport), 3))
 	}
 
 	return transport.Chain(http.DefaultTransport, base...)
 }
+
+func Retry(baseTransport http.RoundTripper, maxRetries int) func(http.RoundTripper) http.RoundTripper {
+	return func(next http.RoundTripper) http.RoundTripper {
+		return transport.RoundTripFunc(func(req *http.Request) (resp *http.Response, err error) {
+			defer func() {
+				if isRetryable(resp) {
+					for i := 1; i <= maxRetries; i++ {
+						wait := backOff(resp, i)
+
+						timer := time.NewTimer(wait)
+
+						slog.DebugContext(req.Context(), "waiting before retrying request", "wait_time", wait.String())
+
+						select {
+						case <-req.Context().Done():
+							timer.Stop()
+							err = req.Context().Err()
+							break
+						case <-timer.C:
+						}
+
+						resp, err = baseTransport.RoundTrip(req)
+						if !isRetryable(resp) {
+							break
+						}
+
+						slog.InfoContext(req.Context(), "retrying request",
+							"target_host", req.URL.Host,
+							"attempt", i,
+						)
+					}
+				}
+			}()
+
+			return next.RoundTrip(req)
+		})
+	}
+}
+
+func backOff(resp *http.Response, attempt int) time.Duration {
+	minDuration := 1 * time.Second
+	maxDuration := 16 * time.Second
+
+	if resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode == http.StatusServiceUnavailable {
+		if s, ok := resp.Header["Retry-After"]; ok {
+			if sleep, err := strconv.ParseInt(s[0], 10, 64); err == nil {
+				return time.Second * time.Duration(sleep)
+			}
+		}
+	}
+
+	// simple exp. backoff
+	mult := math.Pow(2, float64(attempt)) * float64(minDuration)
+	sleep := time.Duration(mult)
+	if float64(sleep) != mult || sleep > maxDuration {
+		sleep = maxDuration
+	}
+	return sleep
+}
+
+func isRetryable(resp *http.Response) bool {
+	if resp == nil {
+		return false
+	}
+
+	// 429 Too Many Requests is recoverable. Sometimes the server puts
+	// Retry-After response header to indicate when the server is will be available again
+	if resp.StatusCode == http.StatusTooManyRequests {
+		return true
+	}
+
+	// We retry on 500-range responses to allow the server time to recover, as 500's are typically not permanent
+	// errors and may relate to outages on the server side.
+	if resp.StatusCode == 0 || (resp.StatusCode > 500 && resp.StatusCode != http.StatusNotImplemented) {
+		return true
+	}
+
+	return false
+}

backend/internal/rgl/client.go

@@ -7,20 +7,25 @@ import (
 	"fmt"
 	"net/http"
 	"strconv"
+	"time"
 
 	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
+	"golang.org/x/time/rate"
 )
 
 type Client struct {
-	client *http.Client
-	tracer trace.Tracer
+	client  *http.Client
+	limiter *rate.Limiter
+	tracer  trace.Tracer
 }
 
 func NewClient(rt http.RoundTripper) *Client {
 	return &Client{
-		client: &http.Client{Transport: rt},
-		tracer: otel.Tracer("rgl"),
+		client:  &http.Client{Transport: rt},
+		limiter: rate.NewLimiter(rate.Every(time.Second), 5),
+		tracer:  otel.Tracer("rgl"),
 	}
 }
 
@@ -33,14 +38,22 @@ func (c *Client) GetPlayers(ctx context.Context, playerIDs []int64) ([]Player, e
 	ctx, span := c.tracer.Start(ctx, "rgl.GetPlayers")
 	defer span.End()
 
+	t := time.Now()
+
+	if err := c.limiter.Wait(ctx); err != nil {
+		return nil, err
+	}
+
+	span.SetAttributes(attribute.Float64("rate_limit_wait", float64(time.Since(t).Milliseconds())))
+
 	stringPlayerIDs := make([]string, len(playerIDs))
 	for i, id := range playerIDs {
 		stringPlayerIDs[i] = strconv.FormatInt(id, 10)
 	}
 
 	reqBytes, err := json.Marshal(stringPlayerIDs)
 	if err != nil {
-		return nil, fmt.Errorf("marshalling player IDs: %w", err)
+		return nil, fmt.Errorf("marshaling player IDs: %w", err)
 	}
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodPost, "https://api.rgl.gg/v0/profile/getmany", bytes.NewReader(reqBytes))

backend/internal/service/get_etf2l_players.go

@@ -26,6 +26,10 @@ func (s *Service) getETF2LPlayers(ctx context.Context, playerIDs []int64, withRe
 	var players []gen.ETF2LPlayer
 
 	for _, playerID := range playerIDs {
+		if playerID == 0 {
+			continue
+		}
+
 		player, err := s.cache.GetPlayer(ctx, cache.LeagueETF2L, playerID)
 		switch {
 		case errors.Is(err, redis.Nil):

backend/internal/service/get_logs_for_match.go

@@ -63,6 +63,7 @@ func (s *Service) GetLogsForMatch(ctx context.Context, params gen.GetLogsForMatc
 }
 
 func (s *Service) getLogsForMatch(ctx context.Context, matchID int) (logs []db.Log, err error) {
+	// TODO: use cache for storing information that match exists
 	exists, err := s.db.MatchExists(ctx, matchID)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get match %d from cache: %w", matchID, err)

backend/internal/service/get_players.go

@@ -29,6 +29,10 @@ func (s *Service) getPlayers(ctx context.Context, playerIDs []int64, withRecruit
 	var players []gen.Player
 
 	for _, playerID := range playerIDs {
+		if playerID == 0 {
+			continue
+		}
+
 		player, err := s.cache.GetPlayer(ctx, cache.LeagueETF2L, playerID)
 		switch {
 		case errors.Is(err, redis.Nil):