diff --git a/internal/apirouter/log_handlers.go b/internal/apirouter/log_handlers.go index e26e3a91f..d0ee22d5e 100644 --- a/internal/apirouter/log_handlers.go +++ b/internal/apirouter/log_handlers.go @@ -1,12 +1,14 @@ package apirouter import ( + "errors" "net/http" "strconv" "strings" "time" "github.com/gin-gonic/gin" + "github.com/hookdeck/outpost/internal/cursor" "github.com/hookdeck/outpost/internal/logging" "github.com/hookdeck/outpost/internal/logstore" "github.com/hookdeck/outpost/internal/models" @@ -269,6 +271,10 @@ func (h *LogHandlers) listDeliveriesInternal(c *gin.Context, tenantID string) { response, err := h.logStore.ListDeliveryEvent(c.Request.Context(), req) if err != nil { + if errors.Is(err, cursor.ErrInvalidCursor) || errors.Is(err, cursor.ErrVersionMismatch) { + AbortWithError(c, http.StatusBadRequest, NewErrBadRequest(err)) + return + } AbortWithError(c, http.StatusInternalServerError, NewErrInternalServer(err)) return } @@ -431,6 +437,10 @@ func (h *LogHandlers) listEventsInternal(c *gin.Context, tenantID string) { response, err := h.logStore.ListEvent(c.Request.Context(), req) if err != nil { + if errors.Is(err, cursor.ErrInvalidCursor) || errors.Is(err, cursor.ErrVersionMismatch) { + AbortWithError(c, http.StatusBadRequest, NewErrBadRequest(err)) + return + } AbortWithError(c, http.StatusInternalServerError, NewErrInternalServer(err)) return } diff --git a/internal/logstore/chlogstore/chlogstore.go b/internal/logstore/chlogstore/chlogstore.go index d2b366df3..12cb677a2 100644 --- a/internal/logstore/chlogstore/chlogstore.go +++ b/internal/logstore/chlogstore/chlogstore.go @@ -3,7 +3,6 @@ package chlogstore import ( "context" "encoding/json" - "errors" "fmt" "strconv" "strings" @@ -13,6 +12,7 @@ import ( "github.com/hookdeck/outpost/internal/cursor" "github.com/hookdeck/outpost/internal/logstore/driver" "github.com/hookdeck/outpost/internal/models" + "github.com/hookdeck/outpost/internal/pagination" ) const ( @@ -41,6 +41,12 @@ func NewLogStore(chDB clickhouse.DB, deploymentID string) driver.LogStore { } } +// eventWithPosition wraps an event with its cursor position data. +type eventWithPosition struct { + *models.Event + eventTime time.Time +} + func (s *logStoreImpl) ListEvent(ctx context.Context, req driver.ListEventRequest) (driver.ListEventResponse, error) { sortOrder := req.SortOrder if sortOrder != "asc" && sortOrder != "desc" { @@ -52,33 +58,48 @@ func (s *logStoreImpl) ListEvent(ctx context.Context, req driver.ListEventReques limit = 100 } - nextPosition, err := cursor.Decode(req.Next, cursorResourceEvent, cursorVersion) - if err != nil { - return driver.ListEventResponse{}, convertCursorError(err) - } - prevPosition, err := cursor.Decode(req.Prev, cursorResourceEvent, cursorVersion) + res, err := pagination.Run(ctx, pagination.Config[eventWithPosition]{ + Limit: limit, + Order: sortOrder, + Next: req.Next, + Prev: req.Prev, + Fetch: func(ctx context.Context, q pagination.QueryInput) ([]eventWithPosition, error) { + query, args := buildEventQuery(s.eventsTable, req, q) + rows, err := s.chDB.Query(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("query failed: %w", err) + } + defer rows.Close() + return scanEvents(rows) + }, + Cursor: pagination.Cursor[eventWithPosition]{ + Encode: func(e eventWithPosition) string { + position := fmt.Sprintf("%d::%s", e.eventTime.UnixMilli(), e.Event.ID) + return cursor.Encode(cursorResourceEvent, cursorVersion, position) + }, + Decode: func(c string) (string, error) { + return cursor.Decode(c, cursorResourceEvent, cursorVersion) + }, + }, + }) if err != nil { - return driver.ListEventResponse{}, convertCursorError(err) + return driver.ListEventResponse{}, err } - goingBackward := prevPosition != "" - - // Multi-column ORDER BY for deterministic pagination - var orderByClause string - if sortOrder == "desc" { - if goingBackward { - orderByClause = "ORDER BY event_time ASC, event_id ASC" - } else { - orderByClause = "ORDER BY event_time DESC, event_id DESC" - } - } else { - if goingBackward { - orderByClause = "ORDER BY event_time DESC, event_id DESC" - } else { - orderByClause = "ORDER BY event_time ASC, event_id ASC" - } + // Extract events from results + data := make([]*models.Event, len(res.Items)) + for i, item := range res.Items { + data[i] = item.Event } + return driver.ListEventResponse{ + Data: data, + Next: res.Next, + Prev: res.Prev, + }, nil +} + +func buildEventQuery(table string, req driver.ListEventRequest, q pagination.QueryInput) (string, []interface{}) { var conditions []string var args []interface{} @@ -106,12 +127,8 @@ func (s *logStoreImpl) ListEvent(ctx context.Context, req driver.ListEventReques args = append(args, *req.EventEnd) } - if nextPosition != "" { - cursorCond, cursorArgs := buildEventCursorCondition(sortOrder, nextPosition, false) - conditions = append(conditions, cursorCond) - args = append(args, cursorArgs...) - } else if prevPosition != "" { - cursorCond, cursorArgs := buildEventCursorCondition(sortOrder, prevPosition, true) + if q.CursorPos != "" { + cursorCond, cursorArgs := buildEventCursorCondition(q.Compare, q.CursorPos) conditions = append(conditions, cursorCond) args = append(args, cursorArgs...) } @@ -121,6 +138,9 @@ func (s *logStoreImpl) ListEvent(ctx context.Context, req driver.ListEventReques whereClause = "1=1" } + orderByClause := fmt.Sprintf("ORDER BY event_time %s, event_id %s", + strings.ToUpper(q.SortDir), strings.ToUpper(q.SortDir)) + // Note: We intentionally omit FINAL to avoid forcing ClickHouse to merge all parts // before returning results. The events table uses ReplacingMergeTree, so duplicates // may briefly appear before background merges consolidate them. This is acceptable @@ -139,20 +159,13 @@ func (s *logStoreImpl) ListEvent(ctx context.Context, req driver.ListEventReques WHERE %s %s LIMIT %d - `, s.eventsTable, whereClause, orderByClause, limit+1) - - rows, err := s.chDB.Query(ctx, query, args...) - if err != nil { - return driver.ListEventResponse{}, fmt.Errorf("query failed: %w", err) - } - defer rows.Close() + `, table, whereClause, orderByClause, q.Limit) - type rowData struct { - event *models.Event - eventTime time.Time - } - var results []rowData + return query, args +} +func scanEvents(rows clickhouse.Rows) ([]eventWithPosition, error) { + var results []eventWithPosition for rows.Next() { var ( eventID string @@ -176,7 +189,7 @@ func (s *logStoreImpl) ListEvent(ctx context.Context, req driver.ListEventReques &dataStr, ) if err != nil { - return driver.ListEventResponse{}, fmt.Errorf("scan failed: %w", err) + return nil, fmt.Errorf("scan failed: %w", err) } var metadata map[string]string @@ -184,86 +197,38 @@ func (s *logStoreImpl) ListEvent(ctx context.Context, req driver.ListEventReques if metadataStr != "" { if err := json.Unmarshal([]byte(metadataStr), &metadata); err != nil { - return driver.ListEventResponse{}, fmt.Errorf("failed to unmarshal metadata: %w", err) + return nil, fmt.Errorf("failed to unmarshal metadata: %w", err) } } if dataStr != "" { if err := json.Unmarshal([]byte(dataStr), &data); err != nil { - return driver.ListEventResponse{}, fmt.Errorf("failed to unmarshal data: %w", err) + return nil, fmt.Errorf("failed to unmarshal data: %w", err) } } - event := &models.Event{ - ID: eventID, - TenantID: tenantID, - DestinationID: destinationID, - Topic: topic, - EligibleForRetry: eligibleForRetry, - Time: eventTime, - Data: data, - Metadata: metadata, - } - - results = append(results, rowData{event: event, eventTime: eventTime}) + results = append(results, eventWithPosition{ + Event: &models.Event{ + ID: eventID, + TenantID: tenantID, + DestinationID: destinationID, + Topic: topic, + EligibleForRetry: eligibleForRetry, + Time: eventTime, + Data: data, + Metadata: metadata, + }, + eventTime: eventTime, + }) } if err := rows.Err(); err != nil { - return driver.ListEventResponse{}, fmt.Errorf("rows error: %w", err) - } - - var hasMore bool - if len(results) > limit { - hasMore = true - results = results[:limit] // Always keep the first `limit` items, remove the extra - } - - // When going backward, we queried in reverse order, so reverse results back to normal order - if goingBackward { - for i, j := 0, len(results)-1; i < j; i, j = i+1, j-1 { - results[i], results[j] = results[j], results[i] - } - } - - data := make([]*models.Event, len(results)) - for i, r := range results { - data[i] = r.event + return nil, fmt.Errorf("rows error: %w", err) } - var nextEncoded, prevEncoded string - if len(results) > 0 { - getPosition := func(r rowData) string { - return fmt.Sprintf("%d::%s", r.eventTime.UnixMilli(), r.event.ID) - } - - encodeCursor := func(position string) string { - return cursor.Encode(cursorResourceEvent, cursorVersion, position) - } - - if prevPosition != "" { - nextEncoded = encodeCursor(getPosition(results[len(results)-1])) - if hasMore { - prevEncoded = encodeCursor(getPosition(results[0])) - } - } else if nextPosition != "" { - prevEncoded = encodeCursor(getPosition(results[0])) - if hasMore { - nextEncoded = encodeCursor(getPosition(results[len(results)-1])) - } - } else { - if hasMore { - nextEncoded = encodeCursor(getPosition(results[len(results)-1])) - } - } - } - - return driver.ListEventResponse{ - Data: data, - Next: nextEncoded, - Prev: prevEncoded, - }, nil + return results, nil } -func buildEventCursorCondition(sortOrder, position string, isBackward bool) (string, []interface{}) { +func buildEventCursorCondition(compare, position string) (string, []interface{}) { parts := strings.SplitN(position, "::", 2) if len(parts) != 2 { return "1=1", nil // invalid cursor, return always true @@ -274,29 +239,20 @@ func buildEventCursorCondition(sortOrder, position string, isBackward bool) (str } eventID := parts[1] - var cmp string - if sortOrder == "desc" { - if isBackward { - cmp = ">" - } else { - cmp = "<" - } - } else { - if isBackward { - cmp = "<" - } else { - cmp = ">" - } - } - condition := fmt.Sprintf(`( event_time %s fromUnixTimestamp64Milli(?) OR (event_time = fromUnixTimestamp64Milli(?) AND event_id %s ?) - )`, cmp, cmp) + )`, compare, compare) return condition, []interface{}{eventTimeMs, eventTimeMs, eventID} } +// deliveryEventWithPosition wraps a delivery event with its cursor position data. +type deliveryEventWithPosition struct { + *models.DeliveryEvent + deliveryTime time.Time +} + func (s *logStoreImpl) ListDeliveryEvent(ctx context.Context, req driver.ListDeliveryEventRequest) (driver.ListDeliveryEventResponse, error) { sortOrder := req.SortOrder if sortOrder != "asc" && sortOrder != "desc" { @@ -308,32 +264,48 @@ func (s *logStoreImpl) ListDeliveryEvent(ctx context.Context, req driver.ListDel limit = 100 } - nextPosition, err := cursor.Decode(req.Next, cursorResourceDelivery, cursorVersion) - if err != nil { - return driver.ListDeliveryEventResponse{}, convertCursorError(err) - } - prevPosition, err := cursor.Decode(req.Prev, cursorResourceDelivery, cursorVersion) + res, err := pagination.Run(ctx, pagination.Config[deliveryEventWithPosition]{ + Limit: limit, + Order: sortOrder, + Next: req.Next, + Prev: req.Prev, + Fetch: func(ctx context.Context, q pagination.QueryInput) ([]deliveryEventWithPosition, error) { + query, args := buildDeliveryQuery(s.deliveriesTable, req, q) + rows, err := s.chDB.Query(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("query failed: %w", err) + } + defer rows.Close() + return scanDeliveryEvents(rows) + }, + Cursor: pagination.Cursor[deliveryEventWithPosition]{ + Encode: func(de deliveryEventWithPosition) string { + position := fmt.Sprintf("%d::%s", de.deliveryTime.UnixMilli(), de.Delivery.ID) + return cursor.Encode(cursorResourceDelivery, cursorVersion, position) + }, + Decode: func(c string) (string, error) { + return cursor.Decode(c, cursorResourceDelivery, cursorVersion) + }, + }, + }) if err != nil { - return driver.ListDeliveryEventResponse{}, convertCursorError(err) + return driver.ListDeliveryEventResponse{}, err } - goingBackward := prevPosition != "" - - var orderByClause string - if sortOrder == "desc" { - if goingBackward { - orderByClause = "ORDER BY delivery_time ASC, delivery_id ASC" - } else { - orderByClause = "ORDER BY delivery_time DESC, delivery_id DESC" - } - } else { - if goingBackward { - orderByClause = "ORDER BY delivery_time DESC, delivery_id DESC" - } else { - orderByClause = "ORDER BY delivery_time ASC, delivery_id ASC" - } + // Extract delivery events from results + data := make([]*models.DeliveryEvent, len(res.Items)) + for i, item := range res.Items { + data[i] = item.DeliveryEvent } + return driver.ListDeliveryEventResponse{ + Data: data, + Next: res.Next, + Prev: res.Prev, + }, nil +} + +func buildDeliveryQuery(table string, req driver.ListDeliveryEventRequest, q pagination.QueryInput) (string, []interface{}) { var conditions []string var args []interface{} @@ -371,12 +343,8 @@ func (s *logStoreImpl) ListDeliveryEvent(ctx context.Context, req driver.ListDel args = append(args, *req.End) } - if nextPosition != "" { - cursorCond, cursorArgs := buildCursorCondition(sortOrder, nextPosition, false) - conditions = append(conditions, cursorCond) - args = append(args, cursorArgs...) - } else if prevPosition != "" { - cursorCond, cursorArgs := buildCursorCondition(sortOrder, prevPosition, true) + if q.CursorPos != "" { + cursorCond, cursorArgs := buildDeliveryCursorCondition(q.Compare, q.CursorPos) conditions = append(conditions, cursorCond) args = append(args, cursorArgs...) } @@ -386,6 +354,9 @@ func (s *logStoreImpl) ListDeliveryEvent(ctx context.Context, req driver.ListDel whereClause = "1=1" } + orderByClause := fmt.Sprintf("ORDER BY delivery_time %s, delivery_id %s", + strings.ToUpper(q.SortDir), strings.ToUpper(q.SortDir)) + query := fmt.Sprintf(` SELECT event_id, @@ -408,21 +379,13 @@ func (s *logStoreImpl) ListDeliveryEvent(ctx context.Context, req driver.ListDel WHERE %s %s LIMIT %d - `, s.deliveriesTable, whereClause, orderByClause, limit+1) - - rows, err := s.chDB.Query(ctx, query, args...) - if err != nil { - return driver.ListDeliveryEventResponse{}, fmt.Errorf("query failed: %w", err) - } - defer rows.Close() + `, table, whereClause, orderByClause, q.Limit) - type rowData struct { - de *models.DeliveryEvent - eventTime time.Time - deliveryTime time.Time - } - var results []rowData + return query, args +} +func scanDeliveryEvents(rows clickhouse.Rows) ([]deliveryEventWithPosition, error) { + var results []deliveryEventWithPosition for rows.Next() { var ( eventID string @@ -462,7 +425,7 @@ func (s *logStoreImpl) ListDeliveryEvent(ctx context.Context, req driver.ListDel &attempt, ) if err != nil { - return driver.ListDeliveryEventResponse{}, fmt.Errorf("scan failed: %w", err) + return nil, fmt.Errorf("scan failed: %w", err) } var metadata map[string]string @@ -471,102 +434,55 @@ func (s *logStoreImpl) ListDeliveryEvent(ctx context.Context, req driver.ListDel if metadataStr != "" { if err := json.Unmarshal([]byte(metadataStr), &metadata); err != nil { - return driver.ListDeliveryEventResponse{}, fmt.Errorf("failed to unmarshal metadata: %w", err) + return nil, fmt.Errorf("failed to unmarshal metadata: %w", err) } } if dataStr != "" { if err := json.Unmarshal([]byte(dataStr), &data); err != nil { - return driver.ListDeliveryEventResponse{}, fmt.Errorf("failed to unmarshal data: %w", err) + return nil, fmt.Errorf("failed to unmarshal data: %w", err) } } if responseDataStr != "" { if err := json.Unmarshal([]byte(responseDataStr), &responseData); err != nil { - return driver.ListDeliveryEventResponse{}, fmt.Errorf("failed to unmarshal response_data: %w", err) + return nil, fmt.Errorf("failed to unmarshal response_data: %w", err) } } - de := &models.DeliveryEvent{ - ID: deliveryEventID, - DestinationID: destinationID, - Manual: manual, - Attempt: int(attempt), - Event: models.Event{ - ID: eventID, - TenantID: tenantID, - DestinationID: destinationID, - Topic: topic, - EligibleForRetry: eligibleForRetry, - Time: eventTime, - Data: data, - Metadata: metadata, - }, - Delivery: &models.Delivery{ - ID: deliveryID, - EventID: eventID, + results = append(results, deliveryEventWithPosition{ + DeliveryEvent: &models.DeliveryEvent{ + ID: deliveryEventID, DestinationID: destinationID, - Status: status, - Time: deliveryTime, - Code: code, - ResponseData: responseData, + Manual: manual, + Attempt: int(attempt), + Event: models.Event{ + ID: eventID, + TenantID: tenantID, + DestinationID: destinationID, + Topic: topic, + EligibleForRetry: eligibleForRetry, + Time: eventTime, + Data: data, + Metadata: metadata, + }, + Delivery: &models.Delivery{ + ID: deliveryID, + EventID: eventID, + DestinationID: destinationID, + Status: status, + Time: deliveryTime, + Code: code, + ResponseData: responseData, + }, }, - } - - results = append(results, rowData{de: de, eventTime: eventTime, deliveryTime: deliveryTime}) + deliveryTime: deliveryTime, + }) } if err := rows.Err(); err != nil { - return driver.ListDeliveryEventResponse{}, fmt.Errorf("rows error: %w", err) + return nil, fmt.Errorf("rows error: %w", err) } - var hasMore bool - if len(results) > limit { - hasMore = true - results = results[:limit] - } - - if goingBackward { - for i, j := 0, len(results)-1; i < j; i, j = i+1, j-1 { - results[i], results[j] = results[j], results[i] - } - } - - data := make([]*models.DeliveryEvent, len(results)) - for i, r := range results { - data[i] = r.de - } - - var nextEncoded, prevEncoded string - if len(results) > 0 { - getPosition := func(r rowData) string { - return fmt.Sprintf("%d::%s", r.deliveryTime.UnixMilli(), r.de.Delivery.ID) - } - - encodeCursor := func(position string) string { - return cursor.Encode(cursorResourceDelivery, cursorVersion, position) - } - - if prevPosition != "" { - nextEncoded = encodeCursor(getPosition(results[len(results)-1])) - if hasMore { - prevEncoded = encodeCursor(getPosition(results[0])) - } - } else if nextPosition != "" { - prevEncoded = encodeCursor(getPosition(results[0])) - if hasMore { - nextEncoded = encodeCursor(getPosition(results[len(results)-1])) - } - } else { - if hasMore { - nextEncoded = encodeCursor(getPosition(results[len(results)-1])) - } - } - } - - return driver.ListDeliveryEventResponse{ - Data: data, - Next: nextEncoded, - Prev: prevEncoded, - }, nil + return results, nil } func (s *logStoreImpl) RetrieveEvent(ctx context.Context, req driver.RetrieveEventRequest) (*models.Event, error) { @@ -892,7 +808,7 @@ func parseTimestampMs(s string) (int64, error) { return strconv.ParseInt(s, 10, 64) } -func buildCursorCondition(sortOrder, position string, isBackward bool) (string, []interface{}) { +func buildDeliveryCursorCondition(compare, position string) (string, []interface{}) { parts := strings.SplitN(position, "::", 2) if len(parts) != 2 { return "1=1", nil @@ -903,33 +819,10 @@ func buildCursorCondition(sortOrder, position string, isBackward bool) (string, } deliveryID := parts[1] - var cmp string - if sortOrder == "desc" { - if isBackward { - cmp = ">" - } else { - cmp = "<" - } - } else { - if isBackward { - cmp = "<" - } else { - cmp = ">" - } - } - condition := fmt.Sprintf(`( delivery_time %s fromUnixTimestamp64Milli(?) OR (delivery_time = fromUnixTimestamp64Milli(?) AND delivery_id %s ?) - )`, cmp, cmp) + )`, compare, compare) return condition, []interface{}{deliveryTimeMs, deliveryTimeMs, deliveryID} } - -// convertCursorError converts cursor package errors to driver errors. -func convertCursorError(err error) error { - if errors.Is(err, cursor.ErrInvalidCursor) || errors.Is(err, cursor.ErrVersionMismatch) { - return driver.ErrInvalidCursor - } - return err -} diff --git a/internal/logstore/driver/driver.go b/internal/logstore/driver/driver.go index a56a2ebe8..dd86fc9d3 100644 --- a/internal/logstore/driver/driver.go +++ b/internal/logstore/driver/driver.go @@ -2,16 +2,11 @@ package driver import ( "context" - "errors" "time" "github.com/hookdeck/outpost/internal/models" ) -// ErrInvalidCursor is returned when the cursor is malformed or doesn't match -// the current query parameters (e.g., different sort order). -var ErrInvalidCursor = errors.New("invalid cursor") - type LogStore interface { ListEvent(context.Context, ListEventRequest) (ListEventResponse, error) ListDeliveryEvent(context.Context, ListDeliveryEventRequest) (ListDeliveryEventResponse, error) diff --git a/internal/logstore/drivertest/crud.go b/internal/logstore/drivertest/crud.go new file mode 100644 index 000000000..d4853bc04 --- /dev/null +++ b/internal/logstore/drivertest/crud.go @@ -0,0 +1,342 @@ +package drivertest + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/hookdeck/outpost/internal/idgen" + "github.com/hookdeck/outpost/internal/logstore/driver" + "github.com/hookdeck/outpost/internal/models" + "github.com/hookdeck/outpost/internal/util/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// testCRUD tests basic CRUD operations with a single shared harness and dataset. +func testCRUD(t *testing.T, newHarness HarnessMaker) { + t.Helper() + + ctx := context.Background() + h, err := newHarness(ctx, t) + require.NoError(t, err) + t.Cleanup(h.Close) + + logStore, err := h.MakeDriver(ctx) + require.NoError(t, err) + + // Shared test data + tenantID := idgen.String() + destinationIDs := []string{idgen.Destination(), idgen.Destination(), idgen.Destination()} + baseTime := time.Now().Truncate(time.Second) + startTime := baseTime.Add(-48 * time.Hour) + + // We'll populate these as we insert + var allDeliveryEvents []*models.DeliveryEvent + destinationEvents := map[string][]*models.Event{} + topicEvents := map[string][]*models.Event{} + statusDeliveryEvents := map[string][]*models.DeliveryEvent{} + + t.Run("insert and verify", func(t *testing.T) { + t.Run("single delivery event", func(t *testing.T) { + destID := destinationIDs[0] + topic := testutil.TestTopics[0] + event := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithID("single_evt"), + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(destID), + testutil.EventFactory.WithTopic(topic), + testutil.EventFactory.WithTime(baseTime.Add(-30*time.Minute)), + ) + delivery := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithID("single_del"), + testutil.DeliveryFactory.WithEventID(event.ID), + testutil.DeliveryFactory.WithDestinationID(destID), + testutil.DeliveryFactory.WithStatus("success"), + testutil.DeliveryFactory.WithTime(baseTime.Add(-30*time.Minute)), + ) + de := &models.DeliveryEvent{ + ID: "single_de", + DestinationID: destID, + Event: *event, + Delivery: delivery, + } + + err := logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{de}) + require.NoError(t, err) + require.NoError(t, h.FlushWrites(ctx)) + + // Track in maps for later filter tests + destinationEvents[destID] = append(destinationEvents[destID], event) + topicEvents[topic] = append(topicEvents[topic], event) + statusDeliveryEvents["success"] = append(statusDeliveryEvents["success"], de) + + // Verify via List + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + EventID: event.ID, + Limit: 10, + Start: &startTime, + }) + require.NoError(t, err) + require.Len(t, response.Data, 1) + assert.Equal(t, event.ID, response.Data[0].Event.ID) + assert.Equal(t, "success", response.Data[0].Delivery.Status) + + // Verify via Retrieve + retrieved, err := logStore.RetrieveEvent(ctx, driver.RetrieveEventRequest{ + TenantID: tenantID, + EventID: event.ID, + }) + require.NoError(t, err) + require.NotNil(t, retrieved) + assert.Equal(t, event.ID, retrieved.ID) + }) + + t.Run("batch delivery events", func(t *testing.T) { + // Create 15 events spread across destinations and topics for filter testing + for i := range 15 { + destID := destinationIDs[i%len(destinationIDs)] + topic := testutil.TestTopics[i%len(testutil.TestTopics)] + eventTime := baseTime.Add(-time.Duration(i+1) * time.Minute) + status := "success" + if i%2 == 1 { + status = "failed" + } + + event := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithID(fmt.Sprintf("batch_evt_%02d", i)), + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(destID), + testutil.EventFactory.WithTopic(topic), + testutil.EventFactory.WithTime(eventTime), + ) + delivery := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithID(fmt.Sprintf("batch_del_%02d", i)), + testutil.DeliveryFactory.WithEventID(event.ID), + testutil.DeliveryFactory.WithDestinationID(destID), + testutil.DeliveryFactory.WithStatus(status), + testutil.DeliveryFactory.WithTime(eventTime.Add(time.Millisecond)), + ) + de := &models.DeliveryEvent{ + ID: fmt.Sprintf("batch_de_%02d", i), + DestinationID: destID, + Event: *event, + Delivery: delivery, + } + + allDeliveryEvents = append(allDeliveryEvents, de) + destinationEvents[destID] = append(destinationEvents[destID], event) + topicEvents[topic] = append(topicEvents[topic], event) + statusDeliveryEvents[status] = append(statusDeliveryEvents[status], de) + } + + err := logStore.InsertManyDeliveryEvent(ctx, allDeliveryEvents) + require.NoError(t, err) + require.NoError(t, h.FlushWrites(ctx)) + + // Verify all inserted + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + Limit: 100, + Start: &startTime, + }) + require.NoError(t, err) + // 15 batch + 1 single = 16 + assert.GreaterOrEqual(t, len(response.Data), 15) + }) + + t.Run("empty batch is no-op", func(t *testing.T) { + err := logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{}) + require.NoError(t, err) + }) + }) + + t.Run("list filters", func(t *testing.T) { + t.Run("ListEvent by destination", func(t *testing.T) { + destID := destinationIDs[0] + response, err := logStore.ListEvent(ctx, driver.ListEventRequest{ + TenantID: tenantID, + DestinationIDs: []string{destID}, + Limit: 100, + EventStart: &startTime, + }) + require.NoError(t, err) + require.Len(t, response.Data, len(destinationEvents[destID])) + for _, event := range response.Data { + assert.Equal(t, destID, event.DestinationID) + } + }) + + t.Run("ListEvent by multiple destinations", func(t *testing.T) { + destIDs := []string{destinationIDs[0], destinationIDs[1]} + response, err := logStore.ListEvent(ctx, driver.ListEventRequest{ + TenantID: tenantID, + DestinationIDs: destIDs, + Limit: 100, + EventStart: &startTime, + }) + require.NoError(t, err) + expectedCount := len(destinationEvents[destIDs[0]]) + len(destinationEvents[destIDs[1]]) + require.Len(t, response.Data, expectedCount) + }) + + t.Run("ListEvent by topic", func(t *testing.T) { + topic := testutil.TestTopics[0] + response, err := logStore.ListEvent(ctx, driver.ListEventRequest{ + TenantID: tenantID, + Topics: []string{topic}, + Limit: 100, + EventStart: &startTime, + }) + require.NoError(t, err) + require.Len(t, response.Data, len(topicEvents[topic])) + }) + + t.Run("ListEvent by time range", func(t *testing.T) { + eventStart := baseTime.Add(-5 * time.Minute) + eventEnd := baseTime + response, err := logStore.ListEvent(ctx, driver.ListEventRequest{ + TenantID: tenantID, + EventStart: &eventStart, + EventEnd: &eventEnd, + Limit: 100, + }) + require.NoError(t, err) + // Should include events within the 5-minute window + assert.NotEmpty(t, response.Data) + for _, evt := range response.Data { + assert.True(t, !evt.Time.Before(eventStart) && !evt.Time.After(eventEnd)) + } + }) + + t.Run("ListDeliveryEvent by destination", func(t *testing.T) { + destID := destinationIDs[0] + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + DestinationIDs: []string{destID}, + Limit: 100, + Start: &startTime, + }) + require.NoError(t, err) + for _, de := range response.Data { + assert.Equal(t, destID, de.DestinationID) + } + }) + + t.Run("ListDeliveryEvent by status", func(t *testing.T) { + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + Status: "success", + Limit: 100, + Start: &startTime, + }) + require.NoError(t, err) + for _, de := range response.Data { + assert.Equal(t, "success", de.Delivery.Status) + } + }) + + t.Run("ListDeliveryEvent by topic", func(t *testing.T) { + topic := testutil.TestTopics[0] + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + Topics: []string{topic}, + Limit: 100, + Start: &startTime, + }) + require.NoError(t, err) + for _, de := range response.Data { + assert.Equal(t, topic, de.Event.Topic) + } + }) + + t.Run("ListDeliveryEvent by event ID", func(t *testing.T) { + eventID := "batch_evt_00" + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + EventID: eventID, + Limit: 100, + Start: &startTime, + }) + require.NoError(t, err) + require.Len(t, response.Data, 1) + assert.Equal(t, eventID, response.Data[0].Event.ID) + }) + }) + + t.Run("retrieve", func(t *testing.T) { + // Use one of our batch events for retrieve tests + knownEventID := "batch_evt_00" + knownDeliveryID := "batch_del_00" + + t.Run("RetrieveEvent existing", func(t *testing.T) { + retrieved, err := logStore.RetrieveEvent(ctx, driver.RetrieveEventRequest{ + TenantID: tenantID, + EventID: knownEventID, + }) + require.NoError(t, err) + require.NotNil(t, retrieved) + assert.Equal(t, knownEventID, retrieved.ID) + assert.Equal(t, tenantID, retrieved.TenantID) + }) + + t.Run("RetrieveEvent with destination filter", func(t *testing.T) { + retrieved, err := logStore.RetrieveEvent(ctx, driver.RetrieveEventRequest{ + TenantID: tenantID, + EventID: knownEventID, + DestinationID: destinationIDs[0], + }) + require.NoError(t, err) + require.NotNil(t, retrieved) + assert.Equal(t, destinationIDs[0], retrieved.DestinationID) + }) + + t.Run("RetrieveEvent non-existent returns nil", func(t *testing.T) { + retrieved, err := logStore.RetrieveEvent(ctx, driver.RetrieveEventRequest{ + TenantID: tenantID, + EventID: "non-existent-event", + }) + require.NoError(t, err) + assert.Nil(t, retrieved) + }) + + t.Run("RetrieveEvent wrong tenant returns nil", func(t *testing.T) { + retrieved, err := logStore.RetrieveEvent(ctx, driver.RetrieveEventRequest{ + TenantID: "wrong-tenant", + EventID: knownEventID, + }) + require.NoError(t, err) + assert.Nil(t, retrieved) + }) + + t.Run("RetrieveDeliveryEvent existing", func(t *testing.T) { + retrieved, err := logStore.RetrieveDeliveryEvent(ctx, driver.RetrieveDeliveryEventRequest{ + TenantID: tenantID, + DeliveryID: knownDeliveryID, + }) + require.NoError(t, err) + require.NotNil(t, retrieved) + assert.Equal(t, knownDeliveryID, retrieved.Delivery.ID) + }) + + t.Run("RetrieveDeliveryEvent non-existent returns nil", func(t *testing.T) { + retrieved, err := logStore.RetrieveDeliveryEvent(ctx, driver.RetrieveDeliveryEventRequest{ + TenantID: tenantID, + DeliveryID: "non-existent-delivery", + }) + require.NoError(t, err) + assert.Nil(t, retrieved) + }) + + t.Run("RetrieveDeliveryEvent wrong tenant returns nil", func(t *testing.T) { + retrieved, err := logStore.RetrieveDeliveryEvent(ctx, driver.RetrieveDeliveryEventRequest{ + TenantID: "wrong-tenant", + DeliveryID: knownDeliveryID, + }) + require.NoError(t, err) + assert.Nil(t, retrieved) + }) + }) +} diff --git a/internal/logstore/drivertest/drivertest.go b/internal/logstore/drivertest/drivertest.go index e2454a8ae..2c7d27727 100644 --- a/internal/logstore/drivertest/drivertest.go +++ b/internal/logstore/drivertest/drivertest.go @@ -1,23 +1,14 @@ +// Package drivertest provides a conformance test suite for logstore drivers. package drivertest import ( "context" - "errors" - "fmt" - "sort" - "strconv" "testing" - "time" - "github.com/hookdeck/outpost/internal/cursor" - "github.com/hookdeck/outpost/internal/idgen" "github.com/hookdeck/outpost/internal/logstore/driver" - "github.com/hookdeck/outpost/internal/models" - "github.com/hookdeck/outpost/internal/util/testutil" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) +// Harness provides the test infrastructure for a logstore driver implementation. type Harness interface { MakeDriver(ctx context.Context) (driver.LogStore, error) // FlushWrites ensures all writes are fully persisted and visible. @@ -28,2954 +19,24 @@ type Harness interface { Close() } +// HarnessMaker creates a new Harness for each test. type HarnessMaker func(ctx context.Context, t *testing.T) (Harness, error) +// RunConformanceTests executes the full conformance test suite for a logstore driver. +// The suite is organized into three parts: +// - CRUD: basic insert, list, and retrieve operations +// - Pagination: cursor-based pagination tests using paginationtest.Suite +// - Misc: isolation, edge cases, and cursor validation func RunConformanceTests(t *testing.T, newHarness HarnessMaker) { t.Helper() - t.Run("TestInsertManyDeliveryEvent", func(t *testing.T) { - testInsertManyDeliveryEvent(t, newHarness) + t.Run("CRUD", func(t *testing.T) { + testCRUD(t, newHarness) }) - t.Run("TestListEvent", func(t *testing.T) { - testListEvent(t, newHarness) + t.Run("Pagination", func(t *testing.T) { + testPagination(t, newHarness) }) - t.Run("TestListDeliveryEvent", func(t *testing.T) { - testListDeliveryEvent(t, newHarness) - }) - t.Run("TestRetrieveEvent", func(t *testing.T) { - testRetrieveEvent(t, newHarness) - }) - t.Run("TestRetrieveDeliveryEvent", func(t *testing.T) { - testRetrieveDeliveryEvent(t, newHarness) - }) - t.Run("TestTenantIsolation", func(t *testing.T) { - testTenantIsolation(t, newHarness) - }) - t.Run("TestPaginationSimple", func(t *testing.T) { - testPaginationSimple(t, newHarness) - }) - t.Run("TestPaginationSuite", func(t *testing.T) { - testPaginationSuite(t, newHarness) - }) - t.Run("TestEdgeCases", func(t *testing.T) { - testEdgeCases(t, newHarness) - }) - t.Run("TestCursorValidation", func(t *testing.T) { - testCursorValidation(t, newHarness) - }) - t.Run("TestCrossTenantQueries", func(t *testing.T) { - testCrossTenantQueries(t, newHarness) - }) -} - -// testInsertManyDeliveryEvent tests the InsertManyDeliveryEvent method -func testInsertManyDeliveryEvent(t *testing.T, newHarness HarnessMaker) { - t.Helper() - - ctx := context.Background() - h, err := newHarness(ctx, t) - require.NoError(t, err) - t.Cleanup(h.Close) - - logStore, err := h.MakeDriver(ctx) - require.NoError(t, err) - - tenantID := idgen.String() - destinationID := idgen.Destination() - startTime := time.Now().Add(-1 * time.Hour) - - t.Run("insert single delivery event", func(t *testing.T) { - event := testutil.EventFactory.AnyPointer( - testutil.EventFactory.WithTenantID(tenantID), - testutil.EventFactory.WithDestinationID(destinationID), - ) - delivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithEventID(event.ID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithStatus("success"), - ) - de := &models.DeliveryEvent{ - ID: idgen.String(), - DestinationID: destinationID, - Event: *event, - Delivery: delivery, - } - - err := logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{de}) - require.NoError(t, err) - - // Verify it was inserted - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenantID, - EventID: event.ID, - Limit: 10, - Start: &startTime, - }) - require.NoError(t, err) - require.Len(t, response.Data, 1) - assert.Equal(t, event.ID, response.Data[0].Event.ID) - assert.Equal(t, "success", response.Data[0].Delivery.Status) - }) - - t.Run("insert multiple delivery events", func(t *testing.T) { - eventID := idgen.Event() - baseDeliveryTime := time.Now().Truncate(time.Second) - event := testutil.EventFactory.AnyPointer( - testutil.EventFactory.WithID(eventID), - testutil.EventFactory.WithTenantID(tenantID), - testutil.EventFactory.WithDestinationID(destinationID), - ) - - // Insert multiple deliveries for the same event (simulating retries) - deliveryEvents := []*models.DeliveryEvent{} - for i := 0; i < 3; i++ { - status := "failed" - if i == 2 { - status = "success" - } - delivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID(fmt.Sprintf("del_%d", i)), - testutil.DeliveryFactory.WithEventID(eventID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithStatus(status), - testutil.DeliveryFactory.WithTime(baseDeliveryTime.Add(time.Duration(i)*time.Second)), - ) - deliveryEvents = append(deliveryEvents, &models.DeliveryEvent{ - ID: fmt.Sprintf("de_%d", i), - DestinationID: destinationID, - Event: *event, - Delivery: delivery, - }) - } - - err := logStore.InsertManyDeliveryEvent(ctx, deliveryEvents) - require.NoError(t, err) - - // Verify all were inserted - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenantID, - EventID: eventID, - Limit: 10, - Start: &startTime, - }) - require.NoError(t, err) - require.Len(t, response.Data, 3) - }) - - t.Run("insert empty slice", func(t *testing.T) { - err := logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{}) - require.NoError(t, err) - }) - - t.Run("duplicate insert is idempotent", func(t *testing.T) { - // Create unique tenant to isolate this test - idempotentTenantID := idgen.String() - idempotentDestID := idgen.Destination() - eventTime := time.Now().Add(-30 * time.Minute).Truncate(time.Second) - deliveryTime := eventTime.Add(1 * time.Second) - - event := testutil.EventFactory.AnyPointer( - testutil.EventFactory.WithTenantID(idempotentTenantID), - testutil.EventFactory.WithDestinationID(idempotentDestID), - testutil.EventFactory.WithTime(eventTime), - ) - delivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithEventID(event.ID), - testutil.DeliveryFactory.WithDestinationID(idempotentDestID), - testutil.DeliveryFactory.WithStatus("success"), - testutil.DeliveryFactory.WithTime(deliveryTime), - ) - de := &models.DeliveryEvent{ - ID: idgen.String(), - DestinationID: idempotentDestID, - Event: *event, - Delivery: delivery, - } - batch := []*models.DeliveryEvent{de} - - // First insert - err := logStore.InsertManyDeliveryEvent(ctx, batch) - require.NoError(t, err) - - // Second insert (duplicate) - should not error - err = logStore.InsertManyDeliveryEvent(ctx, batch) - require.NoError(t, err) - - // Third insert (duplicate) - should not error - err = logStore.InsertManyDeliveryEvent(ctx, batch) - require.NoError(t, err) - - // Flush writes to ensure deduplication is visible (for eventually consistent stores) - err = h.FlushWrites(ctx) - require.NoError(t, err) - - // Verify only 1 record exists (no duplicates) - queryStart := eventTime.Add(-1 * time.Hour) - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: idempotentTenantID, - Limit: 100, - Start: &queryStart, - }) - require.NoError(t, err) - require.Len(t, response.Data, 1, "duplicate inserts should not create multiple records") - assert.Equal(t, event.ID, response.Data[0].Event.ID) - assert.Equal(t, delivery.ID, response.Data[0].Delivery.ID) - }) -} - -// testListEvent tests the ListEvent method with various filters and pagination -func testListEvent(t *testing.T, newHarness HarnessMaker) { - t.Helper() - - ctx := context.Background() - h, err := newHarness(ctx, t) - require.NoError(t, err) - t.Cleanup(h.Close) - - logStore, err := h.MakeDriver(ctx) - require.NoError(t, err) - - tenantID := idgen.String() - destinationIDs := []string{ - idgen.Destination(), - idgen.Destination(), - idgen.Destination(), - } - - // Track events by various dimensions for assertions - destinationEvents := map[string][]*models.Event{} - topicEvents := map[string][]*models.Event{} - allEvents := []*models.Event{} - - // Use a fixed baseTime for deterministic tests - baseTime := time.Now().Truncate(time.Second) - startTime := baseTime.Add(-48 * time.Hour) - start := &startTime - - // Create 15 events spread across destinations and topics - for i := 0; i < 15; i++ { - destinationID := destinationIDs[i%len(destinationIDs)] - topic := testutil.TestTopics[i%len(testutil.TestTopics)] - - // Spread events across time - eventTime := baseTime.Add(-time.Duration(i) * time.Minute) - - event := testutil.EventFactory.AnyPointer( - testutil.EventFactory.WithID(fmt.Sprintf("evt_%02d", i)), - testutil.EventFactory.WithTenantID(tenantID), - testutil.EventFactory.WithTime(eventTime), - testutil.EventFactory.WithDestinationID(destinationID), - testutil.EventFactory.WithTopic(topic), - testutil.EventFactory.WithMetadata(map[string]string{ - "index": strconv.Itoa(i), - }), - ) - - delivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID(fmt.Sprintf("del_%02d", i)), - testutil.DeliveryFactory.WithEventID(event.ID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithStatus("success"), - testutil.DeliveryFactory.WithTime(eventTime.Add(time.Millisecond)), - ) - - de := &models.DeliveryEvent{ - ID: fmt.Sprintf("de_%02d", i), - DestinationID: destinationID, - Event: *event, - Delivery: delivery, - } - - err := logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{de}) - require.NoError(t, err) - - allEvents = append(allEvents, event) - destinationEvents[destinationID] = append(destinationEvents[destinationID], event) - topicEvents[topic] = append(topicEvents[topic], event) - } - - // Sort allEvents by event_time DESC for ordering assertions - sortedEvents := make([]*models.Event, len(allEvents)) - copy(sortedEvents, allEvents) - sort.Slice(sortedEvents, func(i, j int) bool { - return sortedEvents[i].Time.After(sortedEvents[j].Time) - }) - - t.Run("empty result for unknown tenant", func(t *testing.T) { - response, err := logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: "unknown", - Limit: 5, - EventStart: start, - }) - require.NoError(t, err) - assert.Empty(t, response.Data) - assert.Empty(t, response.Next) - assert.Empty(t, response.Prev) - }) - - t.Run("default ordering (event_time DESC)", func(t *testing.T) { - response, err := logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - Limit: 10, - EventStart: start, - }) - require.NoError(t, err) - require.Len(t, response.Data, 10) - - // Verify ordering: should be sorted by event_time DESC - for i, event := range response.Data { - assert.Equal(t, sortedEvents[i].ID, event.ID, - "event ID mismatch at position %d", i) - } - - // Verify first page has next cursor but no prev cursor - assert.NotEmpty(t, response.Next, "should have next cursor with more data") - assert.Empty(t, response.Prev, "first page should have no prev cursor") - }) - - t.Run("filter by destination", func(t *testing.T) { - destID := destinationIDs[0] - response, err := logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - DestinationIDs: []string{destID}, - Limit: 100, - EventStart: start, - }) - require.NoError(t, err) - require.Len(t, response.Data, len(destinationEvents[destID])) - for _, event := range response.Data { - assert.Equal(t, destID, event.DestinationID) - } - }) - - t.Run("filter by multiple destinations", func(t *testing.T) { - destIDs := []string{destinationIDs[0], destinationIDs[1]} - response, err := logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - DestinationIDs: destIDs, - Limit: 100, - EventStart: start, - }) - require.NoError(t, err) - expectedCount := len(destinationEvents[destIDs[0]]) + len(destinationEvents[destIDs[1]]) - require.Len(t, response.Data, expectedCount) - for _, event := range response.Data { - assert.Contains(t, destIDs, event.DestinationID) - } - }) - - t.Run("filter by single topic", func(t *testing.T) { - topic := testutil.TestTopics[0] - response, err := logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - Topics: []string{topic}, - Limit: 100, - EventStart: start, - }) - require.NoError(t, err) - require.Len(t, response.Data, len(topicEvents[topic])) - for _, event := range response.Data { - assert.Equal(t, topic, event.Topic) - } - }) - - t.Run("filter by multiple topics", func(t *testing.T) { - topics := []string{testutil.TestTopics[0], testutil.TestTopics[1]} - response, err := logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - Topics: topics, - Limit: 100, - EventStart: start, - }) - require.NoError(t, err) - expectedCount := len(topicEvents[topics[0]]) + len(topicEvents[topics[1]]) - require.Len(t, response.Data, expectedCount) - for _, event := range response.Data { - assert.Contains(t, topics, event.Topic) - } - }) - - t.Run("filter by event time range", func(t *testing.T) { - // Get events from the last 5 minutes - eventStart := baseTime.Add(-5 * time.Minute) - eventEnd := baseTime.Add(time.Minute) - response, err := logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - EventStart: &eventStart, - EventEnd: &eventEnd, - Limit: 100, - }) - require.NoError(t, err) - // Should return events 0-5 (6 events within 5 minutes) - require.Len(t, response.Data, 6) - for _, event := range response.Data { - assert.True(t, !event.Time.Before(eventStart), "event time should be >= eventStart") - assert.True(t, !event.Time.After(eventEnd), "event time should be <= eventEnd") - } - }) - - t.Run("pagination with limit", func(t *testing.T) { - // First page - response1, err := logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - Limit: 5, - EventStart: start, - }) - require.NoError(t, err) - require.Len(t, response1.Data, 5) - assert.NotEmpty(t, response1.Next) - assert.Empty(t, response1.Prev) - - // Second page using next cursor - response2, err := logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - Next: response1.Next, - Limit: 5, - EventStart: start, - }) - require.NoError(t, err) - require.Len(t, response2.Data, 5) - assert.NotEmpty(t, response2.Prev) - assert.NotEmpty(t, response2.Next) - - // Verify no overlap between pages - for _, e1 := range response1.Data { - for _, e2 := range response2.Data { - assert.NotEqual(t, e1.ID, e2.ID, "pages should not overlap") - } - } - - // Third page - response3, err := logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - Next: response2.Next, - Limit: 5, - EventStart: start, - }) - require.NoError(t, err) - require.Len(t, response3.Data, 5) - assert.NotEmpty(t, response3.Prev) - assert.Empty(t, response3.Next, "last page should have no next cursor") - }) - - t.Run("sort order ASC", func(t *testing.T) { - response, err := logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - Limit: 10, - SortOrder: "asc", - EventStart: start, - }) - require.NoError(t, err) - require.Len(t, response.Data, 10) - - // Verify ascending order - for i := 1; i < len(response.Data); i++ { - assert.True(t, !response.Data[i].Time.Before(response.Data[i-1].Time), - "events should be in ascending order") - } - }) - - t.Run("events are deduplicated", func(t *testing.T) { - // Insert another delivery for an existing event - existingEvent := allEvents[0] - newDelivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID(fmt.Sprintf("del_retry_%s", existingEvent.ID)), - testutil.DeliveryFactory.WithEventID(existingEvent.ID), - testutil.DeliveryFactory.WithDestinationID(existingEvent.DestinationID), - testutil.DeliveryFactory.WithStatus("success"), - testutil.DeliveryFactory.WithTime(existingEvent.Time.Add(time.Second)), - ) - - de := &models.DeliveryEvent{ - ID: fmt.Sprintf("de_retry_%s", existingEvent.ID), - DestinationID: existingEvent.DestinationID, - Event: *existingEvent, - Delivery: newDelivery, - } - - err := logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{de}) - require.NoError(t, err) - - // Flush writes to ensure deduplication has occurred (eventual consistency) - require.NoError(t, h.FlushWrites(ctx)) - - // ListEvent should still return the same number of events (no duplicates) - response, err := logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - Limit: 100, - EventStart: start, - }) - require.NoError(t, err) - require.Len(t, response.Data, 15, "ListEvent should return unique events only") - }) -} - -// testListDeliveryEvent tests the ListDeliveryEvent method with various filters and pagination -func testListDeliveryEvent(t *testing.T, newHarness HarnessMaker) { - t.Helper() - - ctx := context.Background() - h, err := newHarness(ctx, t) - require.NoError(t, err) - t.Cleanup(h.Close) - - logStore, err := h.MakeDriver(ctx) - require.NoError(t, err) - - tenantID := idgen.String() - destinationIDs := []string{ - idgen.Destination(), - idgen.Destination(), - idgen.Destination(), - } - - // Track events by various dimensions for assertions - destinationDeliveryEvents := map[string][]*models.DeliveryEvent{} - statusDeliveryEvents := map[string][]*models.DeliveryEvent{} - topicDeliveryEvents := map[string][]*models.DeliveryEvent{} - timeDeliveryEvents := map[string][]*models.DeliveryEvent{} // "1h", "3h", "6h", "24h" - allDeliveryEvents := []*models.DeliveryEvent{} - - // Use a fixed baseTime for deterministic tests - baseTime := time.Now().Truncate(time.Second) - startTime := baseTime.Add(-48 * time.Hour) // before ALL events - start := &startTime - - for i := 0; i < 20; i++ { - destinationID := destinationIDs[i%len(destinationIDs)] - topic := testutil.TestTopics[i%len(testutil.TestTopics)] - shouldSucceed := i%2 == 0 - shouldRetry := i%3 == 0 - - // Event times are distributed across time buckets: - // i=0-4: within last hour (1h bucket) - // i=5-9: 2-3 hours ago (3h bucket) - // i=10-14: 5-6 hours ago (6h bucket) - // i=15-19: 23-24 hours ago (24h bucket) - var eventTime time.Time - switch { - case i < 5: - eventTime = baseTime.Add(-time.Duration(i) * time.Minute) - case i < 10: - eventTime = baseTime.Add(-time.Duration(2*60+i) * time.Minute) - case i < 15: - eventTime = baseTime.Add(-time.Duration(5*60+i) * time.Minute) - default: - eventTime = baseTime.Add(-time.Duration(23*60+i) * time.Minute) - } - - event := testutil.EventFactory.AnyPointer( - testutil.EventFactory.WithID(fmt.Sprintf("evt_%02d", i)), - testutil.EventFactory.WithTenantID(tenantID), - testutil.EventFactory.WithTime(eventTime), - testutil.EventFactory.WithDestinationID(destinationID), - testutil.EventFactory.WithEligibleForRetry(shouldRetry), - testutil.EventFactory.WithTopic(topic), - testutil.EventFactory.WithMetadata(map[string]string{ - "index": strconv.Itoa(i), - }), - ) - - // Delivery times are based on eventTime for consistency - // Each delivery is slightly after the event, with retries having earlier deliveryTime than final - deliveryTime := eventTime.Add(time.Duration(i) * time.Millisecond) - - if shouldRetry { - initDelivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID(fmt.Sprintf("del_%02d_init", i)), - testutil.DeliveryFactory.WithEventID(event.ID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithStatus("failed"), - testutil.DeliveryFactory.WithTime(deliveryTime), - ) - de := &models.DeliveryEvent{ - ID: fmt.Sprintf("de_%02d_init", i), - DestinationID: destinationID, - Event: *event, - Delivery: initDelivery, - } - allDeliveryEvents = append(allDeliveryEvents, de) - destinationDeliveryEvents[destinationID] = append(destinationDeliveryEvents[destinationID], de) - statusDeliveryEvents["failed"] = append(statusDeliveryEvents["failed"], de) - topicDeliveryEvents[topic] = append(topicDeliveryEvents[topic], de) - categorizeByTime(i, de, timeDeliveryEvents) - - deliveryTime = deliveryTime.Add(time.Millisecond) // Final delivery is later - } - - var finalStatus string - if shouldSucceed { - finalStatus = "success" - } else { - finalStatus = "failed" - } - - finalDelivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID(fmt.Sprintf("del_%02d_final", i)), - testutil.DeliveryFactory.WithEventID(event.ID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithStatus(finalStatus), - testutil.DeliveryFactory.WithTime(deliveryTime), - ) - de := &models.DeliveryEvent{ - ID: fmt.Sprintf("de_%02d_final", i), - DestinationID: destinationID, - Event: *event, - Delivery: finalDelivery, - } - allDeliveryEvents = append(allDeliveryEvents, de) - destinationDeliveryEvents[destinationID] = append(destinationDeliveryEvents[destinationID], de) - statusDeliveryEvents[finalStatus] = append(statusDeliveryEvents[finalStatus], de) - topicDeliveryEvents[topic] = append(topicDeliveryEvents[topic], de) - categorizeByTime(i, de, timeDeliveryEvents) - } - - // Insert all delivery events - require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, allDeliveryEvents)) - - // Sort allDeliveryEvents by delivery_time DESC for ordering assertions - // This is the expected order when querying - sortedDeliveryEvents := make([]*models.DeliveryEvent, len(allDeliveryEvents)) - copy(sortedDeliveryEvents, allDeliveryEvents) - sort.Slice(sortedDeliveryEvents, func(i, j int) bool { - return sortedDeliveryEvents[i].Delivery.Time.After(sortedDeliveryEvents[j].Delivery.Time) - }) - - t.Run("empty result for unknown tenant", func(t *testing.T) { - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: "unknown", - Limit: 5, - Start: start, - }) - require.NoError(t, err) - assert.Empty(t, response.Data) - assert.Empty(t, response.Next) - assert.Empty(t, response.Prev) - }) - - t.Run("default ordering (delivery_time DESC)", func(t *testing.T) { - // Verify default ordering is by delivery_time DESC - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenantID, - Limit: 10, - Start: start, - }) - require.NoError(t, err) - require.Len(t, response.Data, 10) - - // Verify ordering: should be sorted by delivery_time DESC - for i, de := range response.Data { - assert.Equal(t, sortedDeliveryEvents[i].Delivery.ID, de.Delivery.ID, - "delivery ID mismatch at position %d", i) - } - - // Verify first page has next cursor but no prev cursor - assert.NotEmpty(t, response.Next, "should have next cursor with more data") - assert.Empty(t, response.Prev, "first page should have no prev cursor") - }) - - t.Run("filter by destination", func(t *testing.T) { - destID := destinationIDs[0] - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenantID, - DestinationIDs: []string{destID}, - Limit: 100, - Start: start, - }) - require.NoError(t, err) - require.Len(t, response.Data, len(destinationDeliveryEvents[destID])) - for _, de := range response.Data { - assert.Equal(t, destID, de.DestinationID) - } - }) - - t.Run("filter by multiple destinations", func(t *testing.T) { - destIDs := []string{destinationIDs[0], destinationIDs[1]} - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenantID, - DestinationIDs: destIDs, - Limit: 100, - Start: start, - }) - require.NoError(t, err) - expectedCount := len(destinationDeliveryEvents[destIDs[0]]) + len(destinationDeliveryEvents[destIDs[1]]) - require.Len(t, response.Data, expectedCount) - for _, de := range response.Data { - assert.Contains(t, destIDs, de.DestinationID) - } - }) - - t.Run("filter by status", func(t *testing.T) { - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenantID, - Status: "success", - Limit: 100, - Start: start, - }) - require.NoError(t, err) - require.Len(t, response.Data, len(statusDeliveryEvents["success"])) - for _, de := range response.Data { - assert.Equal(t, "success", de.Delivery.Status) - } - }) - - t.Run("filter by single topic", func(t *testing.T) { - topic := testutil.TestTopics[0] - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenantID, - Topics: []string{topic}, - Limit: 100, - Start: start, - }) - require.NoError(t, err) - require.Len(t, response.Data, len(topicDeliveryEvents[topic])) - for _, de := range response.Data { - assert.Equal(t, topic, de.Event.Topic) - } - }) - - t.Run("filter by multiple topics", func(t *testing.T) { - topics := []string{testutil.TestTopics[0], testutil.TestTopics[1]} - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenantID, - Topics: topics, - Limit: 100, - Start: start, - }) - require.NoError(t, err) - expectedCount := len(topicDeliveryEvents[topics[0]]) + len(topicDeliveryEvents[topics[1]]) - require.Len(t, response.Data, expectedCount) - for _, de := range response.Data { - assert.Contains(t, topics, de.Event.Topic) - } - }) - - t.Run("filter by event ID (replaces ListDelivery)", func(t *testing.T) { - t.Run("returns empty for unknown event", func(t *testing.T) { - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenantID, - EventID: "unknown-event", - Limit: 100, - Start: start, - }) - require.NoError(t, err) - assert.Empty(t, response.Data) - }) - - t.Run("returns all deliveries for event", func(t *testing.T) { - eventID := "evt_00" // This event has retry (i%3==0), so 2 deliveries - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenantID, - EventID: eventID, - Limit: 100, - Start: start, - }) - require.NoError(t, err) - require.Len(t, response.Data, 2, "evt_00 should have 2 deliveries (init failed + final)") - for _, de := range response.Data { - assert.Equal(t, eventID, de.Event.ID) - } - }) - - t.Run("filter by event ID and destination", func(t *testing.T) { - eventID := "evt_00" - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenantID, - EventID: eventID, - DestinationIDs: []string{destinationIDs[0]}, // evt_00 goes to destinationIDs[0] - Limit: 100, - Start: start, - }) - require.NoError(t, err) - require.Len(t, response.Data, 2) - for _, de := range response.Data { - assert.Equal(t, eventID, de.Event.ID) - assert.Equal(t, destinationIDs[0], de.DestinationID) - } - }) - }) - - t.Run("time range filtering", func(t *testing.T) { - sevenHoursAgo := baseTime.Add(-7 * time.Hour) - fiveHoursAgo := baseTime.Add(-5 * time.Hour) - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenantID, - Start: &sevenHoursAgo, - End: &fiveHoursAgo, - Limit: 100, - }) - require.NoError(t, err) - require.Len(t, response.Data, len(timeDeliveryEvents["6h"])) - }) - - t.Run("combined filters", func(t *testing.T) { - threeHoursAgo := baseTime.Add(-3 * time.Hour) - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenantID, - Start: &threeHoursAgo, - DestinationIDs: []string{destinationIDs[0]}, - Status: "success", - Topics: []string{testutil.TestTopics[0]}, - Limit: 100, - }) - require.NoError(t, err) - for _, de := range response.Data { - assert.Equal(t, destinationIDs[0], de.DestinationID) - assert.Equal(t, "success", de.Delivery.Status) - assert.Equal(t, testutil.TestTopics[0], de.Event.Topic) - assert.True(t, de.Delivery.Time.After(threeHoursAgo)) - } - }) - - t.Run("verify all fields returned correctly", func(t *testing.T) { - // Get first delivery event and verify all fields - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenantID, - EventID: "evt_00", // Known event with specific data - Limit: 1, - Start: start, - }) - require.NoError(t, err) - require.Len(t, response.Data, 1) - - de := response.Data[0] - - // Event fields - assert.Equal(t, "evt_00", de.Event.ID) - assert.Equal(t, tenantID, de.Event.TenantID) - assert.Equal(t, destinationIDs[0], de.Event.DestinationID) // i=0 -> destinationIDs[0%3] - assert.Equal(t, testutil.TestTopics[0], de.Event.Topic) // i=0 -> TestTopics[0%len] - assert.Equal(t, true, de.Event.EligibleForRetry) // i=0 -> 0%3==0 -> true - assert.NotNil(t, de.Event.Metadata) - assert.Equal(t, "0", de.Event.Metadata["index"]) - - // Delivery fields - assert.NotEmpty(t, de.Delivery.ID) - assert.Equal(t, "evt_00", de.Delivery.EventID) - assert.Equal(t, destinationIDs[0], de.Delivery.DestinationID) - assert.Contains(t, []string{"success", "failed"}, de.Delivery.Status) - assert.False(t, de.Delivery.Time.IsZero()) - }) - - t.Run("limit edge cases", func(t *testing.T) { - t.Run("limit 1 returns single item", func(t *testing.T) { - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenantID, - Limit: 1, - Start: start, - }) - require.NoError(t, err) - require.Len(t, response.Data, 1) - assert.NotEmpty(t, response.Next, "should have next cursor with more data") - }) - - t.Run("limit greater than total returns all", func(t *testing.T) { - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenantID, - Limit: 1000, - Start: start, - }) - require.NoError(t, err) - require.Len(t, response.Data, len(allDeliveryEvents)) - assert.Empty(t, response.Next, "should have no next cursor when all data returned") - }) - }) - -} - -// testRetrieveEvent tests the RetrieveEvent method -func testRetrieveEvent(t *testing.T, newHarness HarnessMaker) { - t.Helper() - - ctx := context.Background() - h, err := newHarness(ctx, t) - require.NoError(t, err) - t.Cleanup(h.Close) - - logStore, err := h.MakeDriver(ctx) - require.NoError(t, err) - - tenantID := idgen.String() - destinationID := idgen.Destination() - eventID := idgen.Event() - eventTime := time.Now().Truncate(time.Millisecond) - - event := testutil.EventFactory.AnyPointer( - testutil.EventFactory.WithID(eventID), - testutil.EventFactory.WithTenantID(tenantID), - testutil.EventFactory.WithDestinationID(destinationID), - testutil.EventFactory.WithTopic("user.created"), - testutil.EventFactory.WithTime(eventTime), - testutil.EventFactory.WithEligibleForRetry(true), - testutil.EventFactory.WithMetadata(map[string]string{ - "source": "test", - "env": "development", - }), - testutil.EventFactory.WithData(map[string]interface{}{ - "user_id": "usr_123", - "email": "test@example.com", - }), - ) - - delivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithEventID(eventID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithStatus("success"), - ) - - de := &models.DeliveryEvent{ - ID: idgen.String(), - DestinationID: destinationID, - Event: *event, - Delivery: delivery, - } - - require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{de})) - - t.Run("retrieve existing event with all fields", func(t *testing.T) { - retrieved, err := logStore.RetrieveEvent(ctx, driver.RetrieveEventRequest{ - TenantID: tenantID, - EventID: eventID, - }) - require.NoError(t, err) - require.NotNil(t, retrieved) - - // Verify all event fields - assert.Equal(t, eventID, retrieved.ID) - assert.Equal(t, tenantID, retrieved.TenantID) - assert.Equal(t, destinationID, retrieved.DestinationID) - assert.Equal(t, "user.created", retrieved.Topic) - assert.Equal(t, true, retrieved.EligibleForRetry) - assert.WithinDuration(t, eventTime, retrieved.Time, time.Second) - assert.Equal(t, "test", retrieved.Metadata["source"]) - assert.Equal(t, "development", retrieved.Metadata["env"]) - assert.Equal(t, "usr_123", retrieved.Data["user_id"]) - assert.Equal(t, "test@example.com", retrieved.Data["email"]) - }) - - t.Run("retrieve with destination filter", func(t *testing.T) { - retrieved, err := logStore.RetrieveEvent(ctx, driver.RetrieveEventRequest{ - TenantID: tenantID, - EventID: eventID, - DestinationID: destinationID, - }) - require.NoError(t, err) - require.NotNil(t, retrieved) - assert.Equal(t, eventID, retrieved.ID) - assert.Equal(t, destinationID, retrieved.DestinationID) - }) - - t.Run("retrieve non-existent event", func(t *testing.T) { - retrieved, err := logStore.RetrieveEvent(ctx, driver.RetrieveEventRequest{ - TenantID: tenantID, - EventID: "non-existent", - }) - require.NoError(t, err) - assert.Nil(t, retrieved) - }) - - t.Run("retrieve with wrong tenant", func(t *testing.T) { - retrieved, err := logStore.RetrieveEvent(ctx, driver.RetrieveEventRequest{ - TenantID: "wrong-tenant", - EventID: eventID, - }) - require.NoError(t, err) - assert.Nil(t, retrieved, "should not return event for wrong tenant") - }) - - t.Run("retrieve with wrong destination", func(t *testing.T) { - retrieved, err := logStore.RetrieveEvent(ctx, driver.RetrieveEventRequest{ - TenantID: tenantID, - EventID: eventID, - DestinationID: "wrong-destination", - }) - require.NoError(t, err) - assert.Nil(t, retrieved, "should not return event for wrong destination") - }) -} - -// testRetrieveDeliveryEvent tests the RetrieveDeliveryEvent method -func testRetrieveDeliveryEvent(t *testing.T, newHarness HarnessMaker) { - t.Helper() - - ctx := context.Background() - h, err := newHarness(ctx, t) - require.NoError(t, err) - t.Cleanup(h.Close) - - logStore, err := h.MakeDriver(ctx) - require.NoError(t, err) - - tenantID := idgen.String() - destinationID := idgen.Destination() - eventID := idgen.Event() - deliveryID := idgen.Delivery() - eventTime := time.Now().Truncate(time.Millisecond) - deliveryTime := eventTime.Add(100 * time.Millisecond) - - event := testutil.EventFactory.AnyPointer( - testutil.EventFactory.WithID(eventID), - testutil.EventFactory.WithTenantID(tenantID), - testutil.EventFactory.WithDestinationID(destinationID), - testutil.EventFactory.WithTopic("order.created"), - testutil.EventFactory.WithTime(eventTime), - testutil.EventFactory.WithEligibleForRetry(true), - testutil.EventFactory.WithMetadata(map[string]string{ - "source": "api", - }), - testutil.EventFactory.WithData(map[string]interface{}{ - "order_id": "ord_456", - "amount": 99.99, - }), - ) - - delivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID(deliveryID), - testutil.DeliveryFactory.WithEventID(eventID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithStatus("success"), - testutil.DeliveryFactory.WithTime(deliveryTime), - ) - delivery.Code = "200" - delivery.ResponseData = map[string]interface{}{ - "latency_ms": 42, - } - - de := &models.DeliveryEvent{ - ID: fmt.Sprintf("%s_%s", eventID, deliveryID), - DestinationID: destinationID, - Event: *event, - Delivery: delivery, - Manual: true, - Attempt: 3, - } - - require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{de})) - require.NoError(t, h.FlushWrites(ctx)) - - t.Run("retrieve existing delivery with all fields", func(t *testing.T) { - retrieved, err := logStore.RetrieveDeliveryEvent(ctx, driver.RetrieveDeliveryEventRequest{ - TenantID: tenantID, - DeliveryID: deliveryID, - }) - require.NoError(t, err) - require.NotNil(t, retrieved) - - // Verify delivery event ID - assert.Equal(t, de.ID, retrieved.ID) - assert.Equal(t, destinationID, retrieved.DestinationID) - - // Verify event fields - assert.Equal(t, eventID, retrieved.Event.ID) - assert.Equal(t, tenantID, retrieved.Event.TenantID) - assert.Equal(t, destinationID, retrieved.Event.DestinationID) - assert.Equal(t, "order.created", retrieved.Event.Topic) - assert.Equal(t, true, retrieved.Event.EligibleForRetry) - assert.WithinDuration(t, eventTime, retrieved.Event.Time, time.Second) - assert.Equal(t, "api", retrieved.Event.Metadata["source"]) - assert.Equal(t, "ord_456", retrieved.Event.Data["order_id"]) - - // Verify delivery fields - require.NotNil(t, retrieved.Delivery) - assert.Equal(t, deliveryID, retrieved.Delivery.ID) - assert.Equal(t, eventID, retrieved.Delivery.EventID) - assert.Equal(t, destinationID, retrieved.Delivery.DestinationID) - assert.Equal(t, "success", retrieved.Delivery.Status) - assert.WithinDuration(t, deliveryTime, retrieved.Delivery.Time, time.Second) - assert.Equal(t, "200", retrieved.Delivery.Code) - // Note: JSON unmarshaling converts integers to float64, but in-memory stores keep them as int - latencyMs := retrieved.Delivery.ResponseData["latency_ms"] - switch v := latencyMs.(type) { - case int: - assert.Equal(t, 42, v) - case float64: - assert.Equal(t, float64(42), v) - default: - t.Errorf("unexpected type for latency_ms: %T", latencyMs) - } - - // Verify delivery event fields - assert.True(t, retrieved.Manual, "Manual field should be true") - assert.Equal(t, 3, retrieved.Attempt, "Attempt field should be 3") - }) - - t.Run("retrieve non-existent delivery", func(t *testing.T) { - retrieved, err := logStore.RetrieveDeliveryEvent(ctx, driver.RetrieveDeliveryEventRequest{ - TenantID: tenantID, - DeliveryID: "non-existent", - }) - require.NoError(t, err) - assert.Nil(t, retrieved) - }) - - t.Run("retrieve with wrong tenant", func(t *testing.T) { - retrieved, err := logStore.RetrieveDeliveryEvent(ctx, driver.RetrieveDeliveryEventRequest{ - TenantID: "wrong-tenant", - DeliveryID: deliveryID, - }) - require.NoError(t, err) - assert.Nil(t, retrieved, "should not return delivery for wrong tenant") - }) - - t.Run("retrieve multiple deliveries for same event", func(t *testing.T) { - // Insert another delivery for the same event (simulating a retry) - secondDeliveryID := idgen.Delivery() - secondDeliveryTime := deliveryTime.Add(time.Second) - - secondDelivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID(secondDeliveryID), - testutil.DeliveryFactory.WithEventID(eventID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithStatus("failed"), - testutil.DeliveryFactory.WithTime(secondDeliveryTime), - ) - secondDelivery.Code = "500" - - secondDE := &models.DeliveryEvent{ - ID: fmt.Sprintf("%s_%s", eventID, secondDeliveryID), - DestinationID: destinationID, - Event: *event, - Delivery: secondDelivery, - } - - require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{secondDE})) - require.NoError(t, h.FlushWrites(ctx)) - - // Retrieve first delivery - should get first delivery - first, err := logStore.RetrieveDeliveryEvent(ctx, driver.RetrieveDeliveryEventRequest{ - TenantID: tenantID, - DeliveryID: deliveryID, - }) - require.NoError(t, err) - require.NotNil(t, first) - assert.Equal(t, deliveryID, first.Delivery.ID) - assert.Equal(t, "success", first.Delivery.Status) - - // Retrieve second delivery - should get second delivery - second, err := logStore.RetrieveDeliveryEvent(ctx, driver.RetrieveDeliveryEventRequest{ - TenantID: tenantID, - DeliveryID: secondDeliveryID, - }) - require.NoError(t, err) - require.NotNil(t, second) - assert.Equal(t, secondDeliveryID, second.Delivery.ID) - assert.Equal(t, "failed", second.Delivery.Status) - }) -} - -// testTenantIsolation ensures data from one tenant cannot be accessed by another -func testTenantIsolation(t *testing.T, newHarness HarnessMaker) { - t.Helper() - - ctx := context.Background() - h, err := newHarness(ctx, t) - require.NoError(t, err) - t.Cleanup(h.Close) - - logStore, err := h.MakeDriver(ctx) - require.NoError(t, err) - - tenant1ID := idgen.String() - tenant2ID := idgen.String() - destinationID := idgen.Destination() - - // Create events for tenant1 - event1 := testutil.EventFactory.AnyPointer( - testutil.EventFactory.WithID("tenant1-event"), - testutil.EventFactory.WithTenantID(tenant1ID), - testutil.EventFactory.WithDestinationID(destinationID), - ) - delivery1 := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithEventID(event1.ID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - ) - - // Create events for tenant2 - event2 := testutil.EventFactory.AnyPointer( - testutil.EventFactory.WithID("tenant2-event"), - testutil.EventFactory.WithTenantID(tenant2ID), - testutil.EventFactory.WithDestinationID(destinationID), - ) - delivery2 := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithEventID(event2.ID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - ) - - require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{ - {ID: idgen.DeliveryEvent(), DestinationID: destinationID, Event: *event1, Delivery: delivery1}, - {ID: idgen.DeliveryEvent(), DestinationID: destinationID, Event: *event2, Delivery: delivery2}, - })) - - startTime := time.Now().Add(-1 * time.Hour) - - t.Run("ListDeliveryEvent isolates by tenant", func(t *testing.T) { - // Tenant1 should only see their events - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenant1ID, - Limit: 100, - Start: &startTime, - }) - require.NoError(t, err) - require.Len(t, response.Data, 1) - assert.Equal(t, "tenant1-event", response.Data[0].Event.ID) - - // Tenant2 should only see their events - response, err = logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenant2ID, - Limit: 100, - Start: &startTime, - }) - require.NoError(t, err) - require.Len(t, response.Data, 1) - assert.Equal(t, "tenant2-event", response.Data[0].Event.ID) - }) - - t.Run("RetrieveEvent isolates by tenant", func(t *testing.T) { - // Tenant1 cannot access tenant2's event - retrieved, err := logStore.RetrieveEvent(ctx, driver.RetrieveEventRequest{ - TenantID: tenant1ID, - EventID: "tenant2-event", - }) - require.NoError(t, err) - assert.Nil(t, retrieved) - - // Tenant2 cannot access tenant1's event - retrieved, err = logStore.RetrieveEvent(ctx, driver.RetrieveEventRequest{ - TenantID: tenant2ID, - EventID: "tenant1-event", - }) - require.NoError(t, err) - assert.Nil(t, retrieved) - }) -} - -// Helper function to categorize delivery events by time bucket -func categorizeByTime(i int, de *models.DeliveryEvent, timeDeliveryEvents map[string][]*models.DeliveryEvent) { - switch { - case i < 5: - timeDeliveryEvents["1h"] = append(timeDeliveryEvents["1h"], de) - case i < 10: - timeDeliveryEvents["3h"] = append(timeDeliveryEvents["3h"], de) - case i < 15: - timeDeliveryEvents["6h"] = append(timeDeliveryEvents["6h"], de) - default: - timeDeliveryEvents["24h"] = append(timeDeliveryEvents["24h"], de) - } -} - -// ============================================================================= -// SIMPLE PAGINATION TEST -// ============================================================================= -// -// A quick sanity check for pagination during development. Tests core mechanics -// with minimal data. Run the full TestPaginationSuite for comprehensive testing. -// -// Usage: make test TEST=./internal/logstore/memlogstore TESTARGS="-run TestPaginationSimple" -// ============================================================================= - -func testPaginationSimple(t *testing.T, newHarness HarnessMaker) { - t.Helper() - - ctx := context.Background() - h, err := newHarness(ctx, t) - require.NoError(t, err) - t.Cleanup(h.Close) - - logStore, err := h.MakeDriver(ctx) - require.NoError(t, err) - - tenantID := idgen.String() - destinationID := idgen.Destination() - baseTime := time.Now().Truncate(time.Second) - - // Create 5 delivery events with distinct times - var allEvents []*models.DeliveryEvent - for i := 0; i < 5; i++ { - event := &models.Event{ - ID: fmt.Sprintf("evt_%d", i), - TenantID: tenantID, - DestinationID: destinationID, - Topic: "test.topic", - Time: baseTime.Add(-time.Duration(i) * time.Hour), - } - delivery := &models.Delivery{ - ID: fmt.Sprintf("del_%d", i), - EventID: event.ID, - DestinationID: destinationID, - Status: "success", - Time: baseTime.Add(-time.Duration(i) * time.Hour), - } - de := &models.DeliveryEvent{ - ID: fmt.Sprintf("de_%d", i), - DestinationID: destinationID, - Event: *event, - Delivery: delivery, - } - allEvents = append(allEvents, de) - } - - require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, allEvents)) - - startTime := baseTime.Add(-48 * time.Hour) - - t.Run("forward pagination collects all items", func(t *testing.T) { - var collected []string - req := driver.ListDeliveryEventRequest{ - TenantID: tenantID, - Start: &startTime, - Limit: 2, - } - - response, err := logStore.ListDeliveryEvent(ctx, req) - require.NoError(t, err) - for _, de := range response.Data { - collected = append(collected, de.Delivery.ID) - } - - for response.Next != "" { - req.Next = response.Next - response, err = logStore.ListDeliveryEvent(ctx, req) - require.NoError(t, err) - for _, de := range response.Data { - collected = append(collected, de.Delivery.ID) - } - } - - assert.Len(t, collected, 5, "should collect all 5 items") - // Default sort is delivery_time DESC, so del_0 (most recent) comes first - assert.Equal(t, "del_0", collected[0], "first item should be most recent") - assert.Equal(t, "del_4", collected[4], "last item should be oldest") - }) - - t.Run("backward pagination returns to start", func(t *testing.T) { - req := driver.ListDeliveryEventRequest{ - TenantID: tenantID, - Start: &startTime, - Limit: 2, - } - - // Get first page - firstPage, err := logStore.ListDeliveryEvent(ctx, req) - require.NoError(t, err) - require.NotEmpty(t, firstPage.Next, "should have next cursor") - assert.Empty(t, firstPage.Prev, "first page should have no prev") - - // Go to second page - req.Next = firstPage.Next - secondPage, err := logStore.ListDeliveryEvent(ctx, req) - require.NoError(t, err) - require.NotEmpty(t, secondPage.Prev, "second page should have prev") - - // Go back to first page - req.Next = "" - req.Prev = secondPage.Prev - backToFirst, err := logStore.ListDeliveryEvent(ctx, req) - require.NoError(t, err) - - assert.Equal(t, firstPage.Data[0].Delivery.ID, backToFirst.Data[0].Delivery.ID, - "returning to first page should show same data") - assert.Empty(t, backToFirst.Prev, "first page should have no prev") - }) - - t.Run("sorting changes order", func(t *testing.T) { - req := driver.ListDeliveryEventRequest{ - TenantID: tenantID, - Start: &startTime, - SortOrder: "asc", - Limit: 5, - } - - response, err := logStore.ListDeliveryEvent(ctx, req) - require.NoError(t, err) - require.Len(t, response.Data, 5) - - // ASC order: oldest first - assert.Equal(t, "del_4", response.Data[0].Delivery.ID, "ASC: oldest should be first") - assert.Equal(t, "del_0", response.Data[4].Delivery.ID, "ASC: newest should be last") - }) - - t.Run("cursor stability", func(t *testing.T) { - req := driver.ListDeliveryEventRequest{ - TenantID: tenantID, - Start: &startTime, - Limit: 2, - } - - // Same request twice should return identical results - resp1, err := logStore.ListDeliveryEvent(ctx, req) - require.NoError(t, err) - - resp2, err := logStore.ListDeliveryEvent(ctx, req) - require.NoError(t, err) - - assert.Equal(t, resp1.Next, resp2.Next, "cursors should be stable") - assert.Equal(t, len(resp1.Data), len(resp2.Data)) - for i := range resp1.Data { - assert.Equal(t, resp1.Data[i].Delivery.ID, resp2.Data[i].Delivery.ID) - } - }) -} - -// ============================================================================= -// PAGINATION TEST SUITE -// ============================================================================= -// -// This suite tests pagination behavior with various filters and sort options. -// It uses dedicated test data with realistic timestamps to verify: -// - Forward and backward traversal -// - Cursor stability -// - Different sort orders (event_time vs delivery_time) -// - Time-based filtering (EventStart/End, DeliveryStart/End) -// -// SORTING LOGIC -// ============= -// -// Sorting uses multi-column ordering to ensure deterministic pagination. -// This is critical because: -// 1. Multiple deliveries can have the same event_time (same event, multiple attempts) -// 2. In rare cases, deliveries could have identical timestamps -// 3. Cursor-based pagination requires stable, repeatable ordering -// -// The sorting columns are: -// -// | SortBy | SortOrder | SQL Equivalent | -// |---------------|-----------|-----------------------------------------------------| -// | delivery_time | desc | ORDER BY delivery_time DESC, delivery_id DESC | -// | delivery_time | asc | ORDER BY delivery_time ASC, delivery_id ASC | -// | event_time | desc | ORDER BY event_time DESC, event_id DESC, delivery_time DESC | -// | event_time | asc | ORDER BY event_time ASC, event_id ASC, delivery_time ASC | -// -// Why these columns? -// -// For delivery_time sorting: -// - Primary: delivery_time - the user's requested sort -// - Secondary: delivery_id - tiebreaker for identical timestamps (rare but possible) -// -// For event_time sorting: -// - Primary: event_time - the user's requested sort -// - Secondary: event_id - groups all deliveries for the same event together -// - Tertiary: delivery_time - orders retries within an event chronologically -// -// The secondary/tertiary columns always use the same direction (ASC/DESC) as the -// primary column to maintain consistent ordering semantics. -// -// TEST DATA STRUCTURE -// =================== -// -// We create 10 events spread over 24 hours, each with 1-5 delivery attempts. -// This models realistic webhook delivery with retries. -// -// Timeline (times relative to baseTime, which is "now"): -// -// Event 0: event_time = -1h -// └── del_0_0: delivery_time = -55m (success) -// └── 1 delivery, immediate success -// -// Event 1: event_time = -2h -// ├── del_1_0: delivery_time = -1h55m (failed) -// └── del_1_1: delivery_time = -1h50m (success) -// └── 2 deliveries, 1 retry -// -// Event 2: event_time = -3h -// ├── del_2_0: delivery_time = -2h55m (failed) -// ├── del_2_1: delivery_time = -2h50m (failed) -// └── del_2_2: delivery_time = -2h45m (success) -// └── 3 deliveries, 2 retries -// -// Event 3: event_time = -5h -// ├── del_3_0: delivery_time = -4h55m (failed) -// ├── del_3_1: delivery_time = -4h50m (failed) -// ├── del_3_2: delivery_time = -4h45m (failed) -// └── del_3_3: delivery_time = -4h40m (success) -// └── 4 deliveries, 3 retries -// -// Event 4: event_time = -6h -// ├── del_4_0: delivery_time = -5h55m (failed) -// ├── del_4_1: delivery_time = -5h50m (failed) -// ├── del_4_2: delivery_time = -5h45m (failed) -// ├── del_4_3: delivery_time = -5h40m (failed) -// └── del_4_4: delivery_time = -5h35m (success) -// └── 5 deliveries, 4 retries -// -// Event 5: event_time = -8h -// ├── del_5_0: delivery_time = -7h55m (failed) -// └── del_5_1: delivery_time = -7h50m (success) -// └── 2 deliveries, 1 retry -// -// Event 6: event_time = -12h -// ├── del_6_0: delivery_time = -11h55m (failed) -// ├── del_6_1: delivery_time = -11h50m (failed) -// └── del_6_2: delivery_time = -11h45m (success) -// └── 3 deliveries, 2 retries -// -// Event 7: event_time = -18h -// └── del_7_0: delivery_time = -17h55m (success) -// └── 1 delivery, immediate success -// -// Event 8: event_time = -20h -// ├── del_8_0: delivery_time = -19h55m (failed) -// └── del_8_1: delivery_time = -19h50m (success) -// └── 2 deliveries, 1 retry -// -// Event 9: event_time = -23h -// ├── del_9_0: delivery_time = -22h55m (failed) -// ├── del_9_1: delivery_time = -22h50m (failed) -// ├── del_9_2: delivery_time = -22h45m (failed) -// └── del_9_3: delivery_time = -22h40m (success) -// └── 4 deliveries, 3 retries -// -// NAMING CONVENTION -// ================= -// -// - Event ID: evt_{event_index} -// Example: evt_3 is the 4th event (0-indexed) -// -// - Delivery ID: del_{event_index}_{delivery_index} -// Example: del_3_2 is event 3's 3rd delivery attempt (0-indexed) -// -// This makes it easy to understand relationships when debugging: -// - del_3_2 → belongs to evt_3, is the 3rd attempt -// -// TOTALS -// ====== -// -// - 10 events -// - 27 delivery events total (1+2+3+4+5+2+3+1+2+4) -// - Event times span: -1h to -23h -// - Delivery times span: -55m to -22h40m -// -// ============================================================================= - -// ============================================================================= -// PAGINATION SUITE TYPES AND HELPERS -// ============================================================================= - -// PaginationTestCase defines a single filter/sort combination to test -type PaginationTestCase struct { - Name string - Request driver.ListDeliveryEventRequest // Base request (TenantID will be set by suite) - Expected []*models.DeliveryEvent // Expected results in exact order -} - -// PaginationSuiteData contains everything needed to run the pagination suite -type PaginationSuiteData struct { - // Name describes this test data set (e.g., "realistic_timestamps", "identical_timestamps") - Name string - - // TenantID for all test data - TenantID string - - // TestCases are the filter/sort combinations to test - // Each test case runs through all pagination behaviors (forward, backward, zigzag, etc.) - TestCases []PaginationTestCase -} - -// PaginationDataGenerator creates test data and returns the suite configuration -// It receives the logStore to insert data and returns the suite data for verification -type PaginationDataGenerator func(t *testing.T, logStore driver.LogStore) *PaginationSuiteData - -// Sorting helper functions for multi-column sorting (see SORTING LOGIC above) -// These are used by generators to compute expected results. - -func compareByDeliveryTime(a, b *models.DeliveryEvent, desc bool) bool { - // Primary: delivery_time - if !a.Delivery.Time.Equal(b.Delivery.Time) { - if desc { - return a.Delivery.Time.After(b.Delivery.Time) - } - return a.Delivery.Time.Before(b.Delivery.Time) - } - // Secondary: delivery_id - if desc { - return a.Delivery.ID > b.Delivery.ID - } - return a.Delivery.ID < b.Delivery.ID -} - -func compareByEventTime(a, b *models.DeliveryEvent, desc bool) bool { - // Primary: event_time - if !a.Event.Time.Equal(b.Event.Time) { - if desc { - return a.Event.Time.After(b.Event.Time) - } - return a.Event.Time.Before(b.Event.Time) - } - // Secondary: event_id - if a.Event.ID != b.Event.ID { - if desc { - return a.Event.ID > b.Event.ID - } - return a.Event.ID < b.Event.ID - } - // Tertiary: delivery_time - if !a.Delivery.Time.Equal(b.Delivery.Time) { - if desc { - return a.Delivery.Time.After(b.Delivery.Time) - } - return a.Delivery.Time.Before(b.Delivery.Time) - } - // Quaternary: delivery_id (for deterministic ordering when all above are equal) - if desc { - return a.Delivery.ID > b.Delivery.ID - } - return a.Delivery.ID < b.Delivery.ID -} - -func sortDeliveryEvents(events []*models.DeliveryEvent, sortBy string, desc bool) []*models.DeliveryEvent { - result := make([]*models.DeliveryEvent, len(events)) - copy(result, events) - sort.Slice(result, func(i, j int) bool { - if sortBy == "event_time" { - return compareByEventTime(result[i], result[j], desc) - } - return compareByDeliveryTime(result[i], result[j], desc) - }) - return result -} - -// ============================================================================= -// DATA GENERATORS -// ============================================================================= - -// generateRealisticTimestampData creates test data with varied, realistic timestamps. -// This is the primary test data set that exercises all filters and sort options. -// -// See TEST DATA STRUCTURE documentation above for details on the data layout. -func generateRealisticTimestampData(t *testing.T, logStore driver.LogStore) *PaginationSuiteData { - t.Helper() - - ctx := context.Background() - tenantID := idgen.String() - destinationIDs := []string{ - idgen.Destination(), - idgen.Destination(), - } - baseTime := time.Now().Truncate(time.Second) - - // Event configuration: [event_index] = {hours_ago, num_deliveries} - eventConfigs := []struct { - hoursAgo int - numDeliveries int - }{ - {1, 1}, // evt_0: 1 delivery - {2, 2}, // evt_1: 2 deliveries - {3, 3}, // evt_2: 3 deliveries - {5, 4}, // evt_3: 4 deliveries - {6, 5}, // evt_4: 5 deliveries - {8, 2}, // evt_5: 2 deliveries - {12, 3}, // evt_6: 3 deliveries - {18, 1}, // evt_7: 1 delivery - {20, 2}, // evt_8: 2 deliveries - {23, 4}, // evt_9: 4 deliveries - } - - var allDeliveryEvents []*models.DeliveryEvent - byDestination := make(map[string][]*models.DeliveryEvent) - byStatus := make(map[string][]*models.DeliveryEvent) - - for eventIdx, cfg := range eventConfigs { - eventTime := baseTime.Add(-time.Duration(cfg.hoursAgo) * time.Hour) - destinationID := destinationIDs[eventIdx%len(destinationIDs)] - topic := testutil.TestTopics[eventIdx%len(testutil.TestTopics)] - - event := &models.Event{ - ID: fmt.Sprintf("evt_%d", eventIdx), - TenantID: tenantID, - DestinationID: destinationID, - Topic: topic, - EligibleForRetry: cfg.numDeliveries > 1, - Time: eventTime, - Metadata: map[string]string{"event_index": strconv.Itoa(eventIdx)}, - Data: map[string]interface{}{"test": true}, - } - - // Create deliveries: first delivery is 5 minutes after event, - // subsequent deliveries are 5 minutes apart - for delIdx := 0; delIdx < cfg.numDeliveries; delIdx++ { - deliveryTime := eventTime.Add(5*time.Minute + time.Duration(delIdx)*5*time.Minute) - - // Last delivery succeeds, others fail - status := "failed" - if delIdx == cfg.numDeliveries-1 { - status = "success" - } - - delivery := &models.Delivery{ - ID: fmt.Sprintf("del_%d_%d", eventIdx, delIdx), - EventID: event.ID, - DestinationID: destinationID, - Status: status, - Time: deliveryTime, - Code: "200", - } - - de := &models.DeliveryEvent{ - ID: fmt.Sprintf("de_%d_%d", eventIdx, delIdx), - DestinationID: destinationID, - Event: *event, - Delivery: delivery, - } - - allDeliveryEvents = append(allDeliveryEvents, de) - byDestination[destinationID] = append(byDestination[destinationID], de) - byStatus[status] = append(byStatus[status], de) - } - } - - // Insert all delivery events - require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, allDeliveryEvents)) - - // Pre-compute sorted lists - sortedByDeliveryTimeDesc := sortDeliveryEvents(allDeliveryEvents, "delivery_time", true) - sortedByDeliveryTimeAsc := sortDeliveryEvents(allDeliveryEvents, "delivery_time", false) - - // Pre-compute filtered subsets - sixHoursAgo := baseTime.Add(-6 * time.Hour) - twelveHoursAgo := baseTime.Add(-12 * time.Hour) - farPast := baseTime.Add(-48 * time.Hour) - - var eventsInLast6Hours []*models.DeliveryEvent - var eventsFrom6hTo12h []*models.DeliveryEvent - var deliveriesInLast6Hours []*models.DeliveryEvent - var deliveriesFrom6hTo12h []*models.DeliveryEvent - - for _, de := range sortedByDeliveryTimeDesc { - // Filter by event time (inclusive semantics) - if !de.Event.Time.Before(sixHoursAgo) { - eventsInLast6Hours = append(eventsInLast6Hours, de) - } - if !de.Event.Time.Before(twelveHoursAgo) && !de.Event.Time.After(sixHoursAgo) { - eventsFrom6hTo12h = append(eventsFrom6hTo12h, de) - } - - // Filter by delivery time (inclusive semantics) - if !de.Delivery.Time.Before(sixHoursAgo) { - deliveriesInLast6Hours = append(deliveriesInLast6Hours, de) - } - if !de.Delivery.Time.Before(twelveHoursAgo) && !de.Delivery.Time.After(sixHoursAgo) { - deliveriesFrom6hTo12h = append(deliveriesFrom6hTo12h, de) - } - } - - // Sort byDestination and byStatus - for destID := range byDestination { - byDestination[destID] = sortDeliveryEvents(byDestination[destID], "delivery_time", true) - } - for status := range byStatus { - byStatus[status] = sortDeliveryEvents(byStatus[status], "delivery_time", true) - } - - // Build test cases - return &PaginationSuiteData{ - Name: "realistic_timestamps", - TenantID: tenantID, - TestCases: []PaginationTestCase{ - { - Name: "default sort (delivery_time DESC)", - Request: driver.ListDeliveryEventRequest{Start: &farPast}, - Expected: sortedByDeliveryTimeDesc, - }, - { - Name: "sort by delivery_time ASC", - Request: driver.ListDeliveryEventRequest{Start: &farPast, SortOrder: "asc"}, - Expected: sortedByDeliveryTimeAsc, - }, - { - Name: "filter by delivery time (last 6 hours)", - Request: driver.ListDeliveryEventRequest{Start: &sixHoursAgo}, - Expected: deliveriesInLast6Hours, - }, - { - Name: "filter by delivery time range (6h to 12h ago)", - Request: driver.ListDeliveryEventRequest{Start: &twelveHoursAgo, End: &sixHoursAgo}, - Expected: deliveriesFrom6hTo12h, - }, - { - Name: "filter by destination", - Request: driver.ListDeliveryEventRequest{DestinationIDs: []string{destinationIDs[0]}, Start: &farPast}, - Expected: byDestination[destinationIDs[0]], - }, - { - Name: "filter by status (success)", - Request: driver.ListDeliveryEventRequest{Status: "success", Start: &farPast}, - Expected: byStatus["success"], - }, - }, - } -} - -// generateIdenticalTimestampData creates test data where all events and deliveries -// have the SAME timestamp. This forces sorting to rely entirely on secondary/tertiary -// columns (event_id, delivery_id) and verifies deterministic ordering. -// -// This is a critical edge case for cursor-based pagination stability. -func generateIdenticalTimestampData(t *testing.T, logStore driver.LogStore) *PaginationSuiteData { - t.Helper() - - ctx := context.Background() - tenantID := idgen.String() - destinationID := idgen.Destination() - - // All events and deliveries share the SAME timestamp - sameTime := time.Now().Truncate(time.Second) - farPast := sameTime.Add(-1 * time.Hour) - - // Create 10 events, each with 2 deliveries, all at the same timestamp - var allDeliveryEvents []*models.DeliveryEvent - - for eventIdx := 0; eventIdx < 10; eventIdx++ { - event := &models.Event{ - ID: fmt.Sprintf("evt_%02d", eventIdx), - TenantID: tenantID, - DestinationID: destinationID, - Topic: "test.topic", - EligibleForRetry: true, - Time: sameTime, - Metadata: map[string]string{}, - Data: map[string]interface{}{}, - } - - for delIdx := 0; delIdx < 2; delIdx++ { - status := "failed" - if delIdx == 1 { - status = "success" - } - - delivery := &models.Delivery{ - ID: fmt.Sprintf("del_%02d_%d", eventIdx, delIdx), - EventID: event.ID, - DestinationID: destinationID, - Status: status, - Time: sameTime, - Code: "200", - } - - de := &models.DeliveryEvent{ - ID: fmt.Sprintf("de_%02d_%d", eventIdx, delIdx), - DestinationID: destinationID, - Event: *event, - Delivery: delivery, - } - - allDeliveryEvents = append(allDeliveryEvents, de) - } - } - - require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, allDeliveryEvents)) - - // With identical timestamps, sorting falls back to ID columns - sortedByDeliveryTimeDesc := sortDeliveryEvents(allDeliveryEvents, "delivery_time", true) - sortedByDeliveryTimeAsc := sortDeliveryEvents(allDeliveryEvents, "delivery_time", false) - - return &PaginationSuiteData{ - Name: "identical_timestamps", - TenantID: tenantID, - TestCases: []PaginationTestCase{ - { - Name: "delivery_time DESC (falls back to delivery_id)", - Request: driver.ListDeliveryEventRequest{Start: &farPast}, - Expected: sortedByDeliveryTimeDesc, - }, - { - Name: "delivery_time ASC (falls back to delivery_id)", - Request: driver.ListDeliveryEventRequest{Start: &farPast, SortOrder: "asc"}, - Expected: sortedByDeliveryTimeAsc, - }, - }, - } -} - -// testPaginationSuite runs the pagination test suite with multiple data generators. -// Each generator creates a different test data set (e.g., realistic timestamps, -// identical timestamps) and the suite runs all pagination behaviors against each. -func testPaginationSuite(t *testing.T, newHarness HarnessMaker) { - t.Helper() - - // List of data generators to run - generators := []PaginationDataGenerator{ - generateRealisticTimestampData, - generateIdenticalTimestampData, - } - - for _, generator := range generators { - generator := generator // capture range variable - - // Each generator gets its own harness/logstore for isolation - ctx := context.Background() - h, err := newHarness(ctx, t) - require.NoError(t, err) - - logStore, err := h.MakeDriver(ctx) - require.NoError(t, err) - - // Generate test data - suiteData := generator(t, logStore) - - t.Run(suiteData.Name, func(t *testing.T) { - for _, tc := range suiteData.TestCases { - tc := tc // capture range variable - t.Run(tc.Name, func(t *testing.T) { - // Set tenant ID - tc.Request.TenantID = suiteData.TenantID - - // Skip if no expected data - if len(tc.Expected) == 0 { - t.Skip("No expected data for this test case") - } - - runPaginationTests(t, logStore, tc) - }) - } - }) - - h.Close() - } -} - -// runPaginationTests runs all pagination behavior tests for a single test case -func runPaginationTests(t *testing.T, logStore driver.LogStore, tc PaginationTestCase) { - t.Helper() - ctx := context.Background() - - // Use a page size that creates multiple pages but not too many - pageSize := 5 - if len(tc.Expected) <= pageSize { - pageSize = 2 // Ensure at least 2 pages for small datasets - } - if len(tc.Expected) <= 2 { - pageSize = 1 - } - - t.Run("forward traversal", func(t *testing.T) { - var collected []*models.DeliveryEvent - req := tc.Request - req.Limit = pageSize - - response, err := logStore.ListDeliveryEvent(ctx, req) - require.NoError(t, err) - collected = append(collected, response.Data...) - - for response.Next != "" { - req.Next = response.Next - req.Prev = "" - response, err = logStore.ListDeliveryEvent(ctx, req) - require.NoError(t, err) - collected = append(collected, response.Data...) - } - - require.Len(t, collected, len(tc.Expected), "forward traversal should collect all items") - for i, de := range collected { - assert.Equal(t, tc.Expected[i].Delivery.ID, de.Delivery.ID, - "forward traversal: mismatch at position %d", i) - } - }) - - t.Run("backward traversal", func(t *testing.T) { - // First go to the last page - req := tc.Request - req.Limit = pageSize - - response, err := logStore.ListDeliveryEvent(ctx, req) - require.NoError(t, err) - - for response.Next != "" { - req.Next = response.Next - req.Prev = "" - response, err = logStore.ListDeliveryEvent(ctx, req) - require.NoError(t, err) - } - - // Now traverse backward - var collected []*models.DeliveryEvent - collected = append(collected, response.Data...) - - for response.Prev != "" { - req.Prev = response.Prev - req.Next = "" - response, err = logStore.ListDeliveryEvent(ctx, req) - require.NoError(t, err) - collected = append(response.Data, collected...) // Prepend - } - - require.Len(t, collected, len(tc.Expected), "backward traversal should collect all items") - for i, de := range collected { - assert.Equal(t, tc.Expected[i].Delivery.ID, de.Delivery.ID, - "backward traversal: mismatch at position %d", i) - } - }) - - t.Run("forward then backward", func(t *testing.T) { - req := tc.Request - req.Limit = pageSize - - // First page - firstPage, err := logStore.ListDeliveryEvent(ctx, req) - require.NoError(t, err) - - if firstPage.Next == "" { - t.Skip("Only one page, cannot test forward then backward") - } - - // Second page - req.Next = firstPage.Next - secondPage, err := logStore.ListDeliveryEvent(ctx, req) - require.NoError(t, err) - - // Back to first page - req.Next = "" - req.Prev = secondPage.Prev - backToFirst, err := logStore.ListDeliveryEvent(ctx, req) - require.NoError(t, err) - - // Verify we got the same first page - require.Len(t, backToFirst.Data, len(firstPage.Data)) - for i, de := range backToFirst.Data { - assert.Equal(t, firstPage.Data[i].Delivery.ID, de.Delivery.ID, - "back to first page: mismatch at position %d", i) - } - assert.Empty(t, backToFirst.Prev, "first page should have no prev cursor") - }) - - t.Run("zigzag navigation", func(t *testing.T) { - req := tc.Request - req.Limit = pageSize - - // First page - page1, err := logStore.ListDeliveryEvent(ctx, req) - require.NoError(t, err) - - if page1.Next == "" { - t.Skip("Only one page, cannot test zigzag") - } - - // Second page - req.Next = page1.Next - page2, err := logStore.ListDeliveryEvent(ctx, req) - require.NoError(t, err) - - // Back to first - req.Next = "" - req.Prev = page2.Prev - page1Again, err := logStore.ListDeliveryEvent(ctx, req) - require.NoError(t, err) - - // Forward to second again - req.Prev = "" - req.Next = page1Again.Next - page2Again, err := logStore.ListDeliveryEvent(ctx, req) - require.NoError(t, err) - - // Verify page 2 is consistent - require.Len(t, page2Again.Data, len(page2.Data)) - for i, de := range page2Again.Data { - assert.Equal(t, page2.Data[i].Delivery.ID, de.Delivery.ID, - "zigzag page 2: mismatch at position %d", i) - } - - // If there's a third page, go there - if page2Again.Next != "" { - req.Prev = "" - req.Next = page2Again.Next - page3, err := logStore.ListDeliveryEvent(ctx, req) - require.NoError(t, err) - assert.NotEmpty(t, page3.Prev, "page 3 should have prev cursor") - } - }) - - t.Run("cursor stability", func(t *testing.T) { - req := tc.Request - req.Limit = pageSize - - // Get first page twice - first1, err := logStore.ListDeliveryEvent(ctx, req) - require.NoError(t, err) - - first2, err := logStore.ListDeliveryEvent(ctx, req) - require.NoError(t, err) - - // Same data - require.Len(t, first1.Data, len(first2.Data)) - for i := range first1.Data { - assert.Equal(t, first1.Data[i].Delivery.ID, first2.Data[i].Delivery.ID) - } - - // Same cursors - assert.Equal(t, first1.Next, first2.Next) - assert.Equal(t, first1.Prev, first2.Prev) - }) - - t.Run("boundary conditions", func(t *testing.T) { - req := tc.Request - req.Limit = pageSize - - // First page - firstPage, err := logStore.ListDeliveryEvent(ctx, req) - require.NoError(t, err) - assert.Empty(t, firstPage.Prev, "first page should have no prev cursor") - - if len(tc.Expected) > pageSize { - assert.NotEmpty(t, firstPage.Next, "first page should have next cursor when more data exists") - } - - // Go to last page - response := firstPage - for response.Next != "" { - req.Next = response.Next - req.Prev = "" - response, err = logStore.ListDeliveryEvent(ctx, req) - require.NoError(t, err) - } - - assert.Empty(t, response.Next, "last page should have no next cursor") - if len(tc.Expected) > pageSize { - assert.NotEmpty(t, response.Prev, "last page should have prev cursor when more data exists") - } - }) - - t.Run("single item pages", func(t *testing.T) { - req := tc.Request - req.Limit = 1 - - var collected []*models.DeliveryEvent - response, err := logStore.ListDeliveryEvent(ctx, req) - require.NoError(t, err) - require.Len(t, response.Data, 1, "limit 1 should return exactly 1 item") - collected = append(collected, response.Data...) - - for response.Next != "" { - req.Next = response.Next - req.Prev = "" - response, err = logStore.ListDeliveryEvent(ctx, req) - require.NoError(t, err) - require.LessOrEqual(t, len(response.Data), 1, "limit 1 should return at most 1 item") - collected = append(collected, response.Data...) - } - - require.Len(t, collected, len(tc.Expected), "single item pagination should collect all items") - }) -} - -// ============================================================================= -// EDGE CASES TEST SUITE -// ============================================================================= -// -// This suite tests edge cases and boundary conditions that implementations -// must handle correctly: -// - Invalid/unknown sort values (should use defaults) -// - Empty vs nil filter semantics (should be equivalent) -// - Time boundary precision (inclusive semantics) -// - EventID filtering with pagination -// - Data immutability (returned data shouldn't affect stored data) -// ============================================================================= - -func testEdgeCases(t *testing.T, newHarness HarnessMaker) { - t.Helper() - - t.Run("invalid sort values use defaults", func(t *testing.T) { - testInvalidSortValues(t, newHarness) - }) - t.Run("empty vs nil filter semantics", func(t *testing.T) { - testEmptyVsNilFilters(t, newHarness) - }) - t.Run("time boundary precision", func(t *testing.T) { - testTimeBoundaryPrecision(t, newHarness) - }) - t.Run("eventID filtering with pagination", func(t *testing.T) { - testEventIDPagination(t, newHarness) - }) - t.Run("data immutability", func(t *testing.T) { - testDataImmutability(t, newHarness) - }) -} - -// testInvalidSortValues verifies that invalid sort values fall back to defaults -// rather than causing errors. This is important for API robustness. -func testInvalidSortValues(t *testing.T, newHarness HarnessMaker) { - t.Helper() - - ctx := context.Background() - h, err := newHarness(ctx, t) - require.NoError(t, err) - t.Cleanup(h.Close) - - logStore, err := h.MakeDriver(ctx) - require.NoError(t, err) - - tenantID := idgen.String() - destinationID := idgen.Destination() - baseTime := time.Now().Truncate(time.Second) - - // Insert test data with distinct delivery times - var deliveryEvents []*models.DeliveryEvent - for i := 0; i < 3; i++ { - event := testutil.EventFactory.AnyPointer( - testutil.EventFactory.WithID(fmt.Sprintf("evt_%d", i)), - testutil.EventFactory.WithTenantID(tenantID), - testutil.EventFactory.WithDestinationID(destinationID), - testutil.EventFactory.WithTime(baseTime.Add(-time.Duration(i)*time.Hour)), - ) - delivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID(fmt.Sprintf("del_%d", i)), - testutil.DeliveryFactory.WithEventID(event.ID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithTime(baseTime.Add(-time.Duration(i)*time.Hour)), - ) - deliveryEvents = append(deliveryEvents, &models.DeliveryEvent{ - ID: fmt.Sprintf("de_%d", i), - DestinationID: destinationID, - Event: *event, - Delivery: delivery, - }) - } - require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, deliveryEvents)) - - startTime := baseTime.Add(-48 * time.Hour) - - t.Run("invalid SortOrder uses default (desc)", func(t *testing.T) { - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenantID, - SortOrder: "sideways", - Start: &startTime, - Limit: 10, - }) - require.NoError(t, err) - require.Len(t, response.Data, 3) - - // Should be sorted DESC (default) - assert.Equal(t, "del_0", response.Data[0].Delivery.ID) - assert.Equal(t, "del_2", response.Data[2].Delivery.ID) - }) -} - -// testEmptyVsNilFilters verifies that empty slices and nil slices are treated -// equivalently (both mean "no filter"). -func testEmptyVsNilFilters(t *testing.T, newHarness HarnessMaker) { - t.Helper() - - ctx := context.Background() - h, err := newHarness(ctx, t) - require.NoError(t, err) - t.Cleanup(h.Close) - - logStore, err := h.MakeDriver(ctx) - require.NoError(t, err) - - tenantID := idgen.String() - destinationID := idgen.Destination() - startTime := time.Now().Add(-1 * time.Hour) - - // Insert test data - event := testutil.EventFactory.AnyPointer( - testutil.EventFactory.WithTenantID(tenantID), - testutil.EventFactory.WithDestinationID(destinationID), - testutil.EventFactory.WithTopic("test.topic"), - ) - delivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithEventID(event.ID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - ) - require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{ - {ID: idgen.DeliveryEvent(), DestinationID: destinationID, Event: *event, Delivery: delivery}, - })) - - t.Run("nil DestinationIDs equals empty DestinationIDs", func(t *testing.T) { - responseNil, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenantID, - DestinationIDs: nil, - Start: &startTime, - Limit: 10, - }) - require.NoError(t, err) - - responseEmpty, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenantID, - DestinationIDs: []string{}, - Start: &startTime, - Limit: 10, - }) - require.NoError(t, err) - - assert.Equal(t, len(responseNil.Data), len(responseEmpty.Data), - "nil and empty DestinationIDs should return same count") - assert.Equal(t, 1, len(responseNil.Data), "should return all data when no filter") - }) - - t.Run("nil Topics equals empty Topics", func(t *testing.T) { - responseNil, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenantID, - Topics: nil, - Start: &startTime, - Limit: 10, - }) - require.NoError(t, err) - - responseEmpty, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenantID, - Topics: []string{}, - Start: &startTime, - Limit: 10, - }) - require.NoError(t, err) - - assert.Equal(t, len(responseNil.Data), len(responseEmpty.Data), - "nil and empty Topics should return same count") - }) - - t.Run("empty Status equals no status filter", func(t *testing.T) { - responseEmpty, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenantID, - Status: "", - Start: &startTime, - Limit: 10, - }) - require.NoError(t, err) - - responseNoFilter, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenantID, - Start: &startTime, - Limit: 10, - }) - require.NoError(t, err) - - assert.Equal(t, len(responseEmpty.Data), len(responseNoFilter.Data), - "empty Status should return same as no Status filter") - }) -} - -// testTimeBoundaryPrecision verifies that time filters use inclusive semantics: -// - Start: >= (deliveries at exactly start time are included) -// - End: <= (deliveries at exactly end time are included) -func testTimeBoundaryPrecision(t *testing.T, newHarness HarnessMaker) { - t.Helper() - - ctx := context.Background() - h, err := newHarness(ctx, t) - require.NoError(t, err) - t.Cleanup(h.Close) - - logStore, err := h.MakeDriver(ctx) - require.NoError(t, err) - - tenantID := idgen.String() - destinationID := idgen.Destination() - - // Create events at precise times - boundaryTime := time.Now().Truncate(time.Second) - beforeBoundary := boundaryTime.Add(-1 * time.Second) - afterBoundary := boundaryTime.Add(1 * time.Second) - - // Event exactly at boundary - eventAtBoundary := testutil.EventFactory.AnyPointer( - testutil.EventFactory.WithID("evt_at_boundary"), - testutil.EventFactory.WithTenantID(tenantID), - testutil.EventFactory.WithDestinationID(destinationID), - testutil.EventFactory.WithTime(boundaryTime), - ) - deliveryAtBoundary := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID("del_at_boundary"), - testutil.DeliveryFactory.WithEventID(eventAtBoundary.ID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithTime(boundaryTime), - ) - - // Event before boundary - eventBefore := testutil.EventFactory.AnyPointer( - testutil.EventFactory.WithID("evt_before"), - testutil.EventFactory.WithTenantID(tenantID), - testutil.EventFactory.WithDestinationID(destinationID), - testutil.EventFactory.WithTime(beforeBoundary), - ) - deliveryBefore := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID("del_before"), - testutil.DeliveryFactory.WithEventID(eventBefore.ID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithTime(beforeBoundary), - ) - - // Event after boundary - eventAfter := testutil.EventFactory.AnyPointer( - testutil.EventFactory.WithID("evt_after"), - testutil.EventFactory.WithTenantID(tenantID), - testutil.EventFactory.WithDestinationID(destinationID), - testutil.EventFactory.WithTime(afterBoundary), - ) - deliveryAfter := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID("del_after"), - testutil.DeliveryFactory.WithEventID(eventAfter.ID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithTime(afterBoundary), - ) - - require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{ - {ID: "de_at", DestinationID: destinationID, Event: *eventAtBoundary, Delivery: deliveryAtBoundary}, - {ID: "de_before", DestinationID: destinationID, Event: *eventBefore, Delivery: deliveryBefore}, - {ID: "de_after", DestinationID: destinationID, Event: *eventAfter, Delivery: deliveryAfter}, - })) - - t.Run("Start is inclusive (>=)", func(t *testing.T) { - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenantID, - Start: &boundaryTime, - Limit: 10, - }) - require.NoError(t, err) - - // Should include delivery at boundary and after, but not before - ids := extractDeliveryIDs(response.Data) - assert.Contains(t, ids, "del_at_boundary", "Start should include deliveries at exact boundary") - assert.Contains(t, ids, "del_after", "Start should include deliveries after boundary") - assert.NotContains(t, ids, "del_before", "Start should exclude deliveries before boundary") - }) - - t.Run("End is inclusive (<=)", func(t *testing.T) { - farPast := boundaryTime.Add(-1 * time.Hour) - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenantID, - Start: &farPast, - End: &boundaryTime, - Limit: 10, - }) - require.NoError(t, err) - - // Should include delivery at boundary and before, but not after - ids := extractDeliveryIDs(response.Data) - assert.Contains(t, ids, "del_at_boundary", "End should include deliveries at exact boundary") - assert.Contains(t, ids, "del_before", "End should include deliveries before boundary") - assert.NotContains(t, ids, "del_after", "End should exclude deliveries after boundary") - }) - - t.Run("exact range includes boundary items", func(t *testing.T) { - // Range from exactly beforeBoundary to exactly afterBoundary - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenantID, - Start: &beforeBoundary, - End: &afterBoundary, - Limit: 10, - }) - require.NoError(t, err) - - // Should include all three - assert.Len(t, response.Data, 3, "range should include all items including boundaries") - }) -} - -// testEventIDPagination verifies pagination works correctly when filtering by EventID. -// This is a common use case: viewing all delivery attempts for a specific event. -func testEventIDPagination(t *testing.T, newHarness HarnessMaker) { - t.Helper() - - ctx := context.Background() - h, err := newHarness(ctx, t) - require.NoError(t, err) - t.Cleanup(h.Close) - - logStore, err := h.MakeDriver(ctx) - require.NoError(t, err) - - tenantID := idgen.String() - destinationID := idgen.Destination() - eventID := "evt_with_many_deliveries" - baseTime := time.Now().Truncate(time.Second) - - // Create one event with many delivery attempts (simulating retries) - event := testutil.EventFactory.AnyPointer( - testutil.EventFactory.WithID(eventID), - testutil.EventFactory.WithTenantID(tenantID), - testutil.EventFactory.WithDestinationID(destinationID), - testutil.EventFactory.WithTime(baseTime.Add(-1*time.Hour)), - ) - - // Create 10 delivery attempts - var deliveryEvents []*models.DeliveryEvent - for i := 0; i < 10; i++ { - status := "failed" - if i == 9 { - status = "success" - } - delivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID(fmt.Sprintf("del_%02d", i)), - testutil.DeliveryFactory.WithEventID(eventID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithStatus(status), - testutil.DeliveryFactory.WithTime(baseTime.Add(-time.Duration(60-i)*time.Minute)), - ) - deliveryEvents = append(deliveryEvents, &models.DeliveryEvent{ - ID: fmt.Sprintf("de_%02d", i), - DestinationID: destinationID, - Event: *event, - Delivery: delivery, - }) - } - - // Also insert a different event to ensure EventID filter works - otherEvent := testutil.EventFactory.AnyPointer( - testutil.EventFactory.WithID("other_event"), - testutil.EventFactory.WithTenantID(tenantID), - testutil.EventFactory.WithDestinationID(destinationID), - ) - otherDelivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithEventID(otherEvent.ID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - ) - deliveryEvents = append(deliveryEvents, &models.DeliveryEvent{ - ID: "de_other", - DestinationID: destinationID, - Event: *otherEvent, - Delivery: otherDelivery, - }) - - require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, deliveryEvents)) - - startTime := baseTime.Add(-2 * time.Hour) - - t.Run("forward pagination with EventID filter", func(t *testing.T) { - var collected []*models.DeliveryEvent - req := driver.ListDeliveryEventRequest{ - TenantID: tenantID, - EventID: eventID, - Start: &startTime, - Limit: 3, // Small page size to test pagination - } - - response, err := logStore.ListDeliveryEvent(ctx, req) - require.NoError(t, err) - collected = append(collected, response.Data...) - - for response.Next != "" { - req.Next = response.Next - response, err = logStore.ListDeliveryEvent(ctx, req) - require.NoError(t, err) - collected = append(collected, response.Data...) - } - - assert.Len(t, collected, 10, "should collect exactly 10 deliveries for the event") - - // All should belong to the same event - for _, de := range collected { - assert.Equal(t, eventID, de.Event.ID, "all deliveries should be for the filtered event") - } - }) - - t.Run("backward pagination with EventID filter", func(t *testing.T) { - req := driver.ListDeliveryEventRequest{ - TenantID: tenantID, - EventID: eventID, - Start: &startTime, - Limit: 3, - } - - // Go to last page - response, err := logStore.ListDeliveryEvent(ctx, req) - require.NoError(t, err) - - for response.Next != "" { - req.Next = response.Next - req.Prev = "" - response, err = logStore.ListDeliveryEvent(ctx, req) - require.NoError(t, err) - } - - // Now go backward - var collected []*models.DeliveryEvent - collected = append(collected, response.Data...) - - for response.Prev != "" { - req.Prev = response.Prev - req.Next = "" - response, err = logStore.ListDeliveryEvent(ctx, req) - require.NoError(t, err) - collected = append(response.Data, collected...) - } - - assert.Len(t, collected, 10, "backward traversal should collect all deliveries") - }) -} - -// testDataImmutability verifies that modifying returned data doesn't affect -// subsequent queries. This is important for data integrity. -func testDataImmutability(t *testing.T, newHarness HarnessMaker) { - t.Helper() - - ctx := context.Background() - h, err := newHarness(ctx, t) - require.NoError(t, err) - t.Cleanup(h.Close) - - logStore, err := h.MakeDriver(ctx) - require.NoError(t, err) - - tenantID := idgen.String() - destinationID := idgen.Destination() - eventID := "immutable_event" - startTime := time.Now().Add(-1 * time.Hour) - - event := testutil.EventFactory.AnyPointer( - testutil.EventFactory.WithID(eventID), - testutil.EventFactory.WithTenantID(tenantID), - testutil.EventFactory.WithDestinationID(destinationID), - testutil.EventFactory.WithTopic("original.topic"), - testutil.EventFactory.WithMetadata(map[string]string{"key": "original"}), - ) - delivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID("original_delivery"), - testutil.DeliveryFactory.WithEventID(eventID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithStatus("success"), - ) - - require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{ - {ID: "de_immutable", DestinationID: destinationID, Event: *event, Delivery: delivery}, - })) - - t.Run("modifying ListDeliveryEvent result doesn't affect subsequent queries", func(t *testing.T) { - // First query - response1, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenantID, - Start: &startTime, - Limit: 10, - }) - require.NoError(t, err) - require.Len(t, response1.Data, 1) - - // Mutate the returned data - response1.Data[0].Event.Topic = "MUTATED" - response1.Data[0].Event.Metadata["key"] = "MUTATED" - response1.Data[0].Delivery.Status = "MUTATED" - - // Second query should return original values - response2, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenantID, - Start: &startTime, - Limit: 10, - }) - require.NoError(t, err) - require.Len(t, response2.Data, 1) - - assert.Equal(t, "original.topic", response2.Data[0].Event.Topic, - "Event.Topic should not be affected by mutation") - assert.Equal(t, "original", response2.Data[0].Event.Metadata["key"], - "Event.Metadata should not be affected by mutation") - assert.Equal(t, "success", response2.Data[0].Delivery.Status, - "Delivery.Status should not be affected by mutation") - }) - - t.Run("modifying RetrieveEvent result doesn't affect subsequent queries", func(t *testing.T) { - // First retrieval - event1, err := logStore.RetrieveEvent(ctx, driver.RetrieveEventRequest{ - TenantID: tenantID, - EventID: eventID, - }) - require.NoError(t, err) - require.NotNil(t, event1) - - // Mutate the returned event - event1.Topic = "MUTATED" - event1.Metadata["key"] = "MUTATED" - - // Second retrieval should return original values - event2, err := logStore.RetrieveEvent(ctx, driver.RetrieveEventRequest{ - TenantID: tenantID, - EventID: eventID, - }) - require.NoError(t, err) - require.NotNil(t, event2) - - assert.Equal(t, "original.topic", event2.Topic, - "RetrieveEvent should return fresh copy") - assert.Equal(t, "original", event2.Metadata["key"], - "RetrieveEvent metadata should not be affected by mutation") - }) -} - -// Helper function to extract delivery IDs from delivery events -func extractDeliveryIDs(des []*models.DeliveryEvent) []string { - ids := make([]string, len(des)) - for i, de := range des { - ids[i] = de.Delivery.ID - } - return ids -} - -// ============================================================================= -// Cursor Validation Tests -// ============================================================================= -// These tests verify that cursors encode sort parameters and that using a cursor -// with different sort parameters returns an error. This prevents confusing -// behavior when changing query params mid-pagination. -// ============================================================================= - -func testCursorValidation(t *testing.T, newHarness HarnessMaker) { - t.Helper() - - t.Run("malformed cursor returns error", func(t *testing.T) { - testMalformedCursor(t, newHarness) - }) - t.Run("cursor works with matching sort params", func(t *testing.T) { - testCursorMatchingSortParams(t, newHarness) - }) -} - -// testMalformedCursor verifies that a malformed cursor string returns ErrInvalidCursor. -func testMalformedCursor(t *testing.T, newHarness HarnessMaker) { - t.Helper() - - ctx := context.Background() - h, err := newHarness(ctx, t) - require.NoError(t, err) - t.Cleanup(h.Close) - - logStore, err := h.MakeDriver(ctx) - require.NoError(t, err) - - tenantID := idgen.String() - startTime := time.Now().Add(-1 * time.Hour) - - testCases := []struct { - name string - cursor string - }{ - {"completely invalid base62", "!!!invalid!!!"}, - {"random string", "abcdef123456"}, - {"wrong resource", cursor.Encode("evt", 1, "test")}, // evt is for events, not deliveries - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - _, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenantID, - SortOrder: "desc", - Start: &startTime, - Next: tc.cursor, - Limit: 10, - }) - // All invalid cursor cases should error - require.Error(t, err) - assert.True(t, errors.Is(err, driver.ErrInvalidCursor), - "expected ErrInvalidCursor for %s, got: %v", tc.name, err) - }) - } -} - -// testCursorMatchingSortParams verifies that cursors work correctly when -// sort parameters match between cursor generation and usage. -func testCursorMatchingSortParams(t *testing.T, newHarness HarnessMaker) { - t.Helper() - - ctx := context.Background() - h, err := newHarness(ctx, t) - require.NoError(t, err) - t.Cleanup(h.Close) - - logStore, err := h.MakeDriver(ctx) - require.NoError(t, err) - - tenantID := idgen.String() - destinationID := idgen.Destination() - baseTime := time.Now().Truncate(time.Second) - startTime := baseTime.Add(-48 * time.Hour) - - // Insert data - var deliveryEvents []*models.DeliveryEvent - for i := 0; i < 5; i++ { - event := testutil.EventFactory.AnyPointer( - testutil.EventFactory.WithID(fmt.Sprintf("evt_match_%d", i)), - testutil.EventFactory.WithTenantID(tenantID), - testutil.EventFactory.WithDestinationID(destinationID), - testutil.EventFactory.WithTime(baseTime.Add(-time.Duration(i)*time.Hour)), - ) - delivery := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID(fmt.Sprintf("del_match_%d", i)), - testutil.DeliveryFactory.WithEventID(event.ID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithTime(baseTime.Add(-time.Duration(i)*time.Hour)), - ) - deliveryEvents = append(deliveryEvents, &models.DeliveryEvent{ - ID: fmt.Sprintf("de_match_%d", i), - DestinationID: destinationID, - Event: *event, - Delivery: delivery, - }) - } - require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, deliveryEvents)) - - sortOrders := []string{"desc", "asc"} - - for _, sortOrder := range sortOrders { - t.Run(fmt.Sprintf("delivery_time_%s", sortOrder), func(t *testing.T) { - // Get first page with cursor - response1, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenantID, - SortOrder: sortOrder, - Start: &startTime, - Limit: 2, - }) - require.NoError(t, err) - require.NotEmpty(t, response1.Next, "expected next cursor for delivery_time %s", sortOrder) - - // Use cursor with same sort params - should succeed - response2, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenantID, - SortOrder: sortOrder, - Start: &startTime, - Next: response1.Next, - Limit: 2, - }) - require.NoError(t, err, "cursor should work with matching sort params for delivery_time %s", sortOrder) - require.NotEmpty(t, response2.Data, "should return data for second page") - - // Verify we got different data (not the same page) - if len(response1.Data) > 0 && len(response2.Data) > 0 { - assert.NotEqual(t, response1.Data[0].ID, response2.Data[0].ID, - "second page should have different data") - } - }) - } -} - -// ============================================================================= -// CROSS-TENANT QUERY TESTS -// ============================================================================= -// -// Tests that queries with empty TenantID return records from all tenants. -// This supports admin-level queries that need to view data across the system. -// ============================================================================= - -func testCrossTenantQueries(t *testing.T, newHarness HarnessMaker) { - t.Helper() - - ctx := context.Background() - h, err := newHarness(ctx, t) - require.NoError(t, err) - t.Cleanup(h.Close) - - logStore, err := h.MakeDriver(ctx) - require.NoError(t, err) - - // Create data for two different tenants - tenant1ID := idgen.String() - tenant2ID := idgen.String() - destinationID := idgen.Destination() - baseTime := time.Now().Truncate(time.Second) - startTime := baseTime.Add(-1 * time.Hour) - - // Create event and delivery for tenant1 - event1 := testutil.EventFactory.AnyPointer( - testutil.EventFactory.WithID("cross-tenant-event-1"), - testutil.EventFactory.WithTenantID(tenant1ID), - testutil.EventFactory.WithDestinationID(destinationID), - testutil.EventFactory.WithTopic("test.topic"), - testutil.EventFactory.WithTime(baseTime.Add(-10*time.Minute)), - ) - delivery1 := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID("cross-tenant-delivery-1"), - testutil.DeliveryFactory.WithEventID(event1.ID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithStatus("success"), - testutil.DeliveryFactory.WithTime(baseTime.Add(-10*time.Minute)), - ) - - // Create event and delivery for tenant2 - event2 := testutil.EventFactory.AnyPointer( - testutil.EventFactory.WithID("cross-tenant-event-2"), - testutil.EventFactory.WithTenantID(tenant2ID), - testutil.EventFactory.WithDestinationID(destinationID), - testutil.EventFactory.WithTopic("test.topic"), - testutil.EventFactory.WithTime(baseTime.Add(-5*time.Minute)), - ) - delivery2 := testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID("cross-tenant-delivery-2"), - testutil.DeliveryFactory.WithEventID(event2.ID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithStatus("failed"), - testutil.DeliveryFactory.WithTime(baseTime.Add(-5*time.Minute)), - ) - - // Insert both delivery events - require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{ - {ID: idgen.DeliveryEvent(), DestinationID: destinationID, Event: *event1, Delivery: delivery1}, - {ID: idgen.DeliveryEvent(), DestinationID: destinationID, Event: *event2, Delivery: delivery2}, - })) - require.NoError(t, h.FlushWrites(ctx)) - - t.Run("ListEvent returns all tenants when TenantID empty", func(t *testing.T) { - response, err := logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: "", // Empty TenantID for cross-tenant query - DestinationIDs: []string{destinationID}, - Limit: 100, - EventStart: &startTime, - }) - require.NoError(t, err) - require.Len(t, response.Data, 2) - - // Verify we got events from both tenants - tenantsSeen := map[string]bool{} - for _, event := range response.Data { - tenantsSeen[event.TenantID] = true - } - assert.True(t, tenantsSeen[tenant1ID], "should include tenant1 events") - assert.True(t, tenantsSeen[tenant2ID], "should include tenant2 events") - }) - - t.Run("ListDeliveryEvent returns all tenants when TenantID empty", func(t *testing.T) { - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: "", // Empty TenantID for cross-tenant query - DestinationIDs: []string{destinationID}, - Limit: 100, - Start: &startTime, - }) - require.NoError(t, err) - require.Len(t, response.Data, 2) - - // Verify we got deliveries from both tenants - tenantsSeen := map[string]bool{} - for _, de := range response.Data { - tenantsSeen[de.Event.TenantID] = true - } - assert.True(t, tenantsSeen[tenant1ID], "should include tenant1 deliveries") - assert.True(t, tenantsSeen[tenant2ID], "should include tenant2 deliveries") - }) - - t.Run("RetrieveEvent finds event across tenants when TenantID empty", func(t *testing.T) { - // Retrieve tenant1's event without specifying tenant - retrieved1, err := logStore.RetrieveEvent(ctx, driver.RetrieveEventRequest{ - TenantID: "", // Empty TenantID for cross-tenant query - EventID: "cross-tenant-event-1", - }) - require.NoError(t, err) - require.NotNil(t, retrieved1) - assert.Equal(t, "cross-tenant-event-1", retrieved1.ID) - assert.Equal(t, tenant1ID, retrieved1.TenantID) - - // Retrieve tenant2's event without specifying tenant - retrieved2, err := logStore.RetrieveEvent(ctx, driver.RetrieveEventRequest{ - TenantID: "", // Empty TenantID for cross-tenant query - EventID: "cross-tenant-event-2", - }) - require.NoError(t, err) - require.NotNil(t, retrieved2) - assert.Equal(t, "cross-tenant-event-2", retrieved2.ID) - assert.Equal(t, tenant2ID, retrieved2.TenantID) - }) - - t.Run("RetrieveDeliveryEvent finds delivery across tenants when TenantID empty", func(t *testing.T) { - // Retrieve tenant1's delivery without specifying tenant - retrieved1, err := logStore.RetrieveDeliveryEvent(ctx, driver.RetrieveDeliveryEventRequest{ - TenantID: "", // Empty TenantID for cross-tenant query - DeliveryID: "cross-tenant-delivery-1", - }) - require.NoError(t, err) - require.NotNil(t, retrieved1) - assert.Equal(t, "cross-tenant-delivery-1", retrieved1.Delivery.ID) - assert.Equal(t, tenant1ID, retrieved1.Event.TenantID) - - // Retrieve tenant2's delivery without specifying tenant - retrieved2, err := logStore.RetrieveDeliveryEvent(ctx, driver.RetrieveDeliveryEventRequest{ - TenantID: "", // Empty TenantID for cross-tenant query - DeliveryID: "cross-tenant-delivery-2", - }) - require.NoError(t, err) - require.NotNil(t, retrieved2) - assert.Equal(t, "cross-tenant-delivery-2", retrieved2.Delivery.ID) - assert.Equal(t, tenant2ID, retrieved2.Event.TenantID) + t.Run("Misc", func(t *testing.T) { + testMisc(t, newHarness) }) } diff --git a/internal/logstore/drivertest/misc.go b/internal/logstore/drivertest/misc.go new file mode 100644 index 000000000..4070ac48e --- /dev/null +++ b/internal/logstore/drivertest/misc.go @@ -0,0 +1,523 @@ +package drivertest + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/hookdeck/outpost/internal/cursor" + "github.com/hookdeck/outpost/internal/idgen" + "github.com/hookdeck/outpost/internal/logstore/driver" + "github.com/hookdeck/outpost/internal/models" + "github.com/hookdeck/outpost/internal/util/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// testMisc tests isolation, edge cases, and cursor validation with a single shared harness. +func testMisc(t *testing.T, newHarness HarnessMaker) { + t.Helper() + + ctx := context.Background() + h, err := newHarness(ctx, t) + require.NoError(t, err) + t.Cleanup(h.Close) + + logStore, err := h.MakeDriver(ctx) + require.NoError(t, err) + + t.Run("Isolation", func(t *testing.T) { + testIsolation(t, ctx, logStore, h) + }) + t.Run("EdgeCases", func(t *testing.T) { + testEdgeCases(t, ctx, logStore, h) + }) + t.Run("CursorValidation", func(t *testing.T) { + testCursorValidation(t, ctx, logStore, h) + }) +} + +func testIsolation(t *testing.T, ctx context.Context, logStore driver.LogStore, h Harness) { + tenant1ID := idgen.String() + tenant2ID := idgen.String() + destinationID := idgen.Destination() + baseTime := time.Now().Truncate(time.Second) + startTime := baseTime.Add(-1 * time.Hour) + + event1 := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithID("tenant1-event"), + testutil.EventFactory.WithTenantID(tenant1ID), + testutil.EventFactory.WithDestinationID(destinationID), + testutil.EventFactory.WithTopic("test.topic"), + testutil.EventFactory.WithTime(baseTime.Add(-10*time.Minute)), + ) + delivery1 := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithID("tenant1-delivery"), + testutil.DeliveryFactory.WithEventID(event1.ID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + testutil.DeliveryFactory.WithStatus("success"), + testutil.DeliveryFactory.WithTime(baseTime.Add(-10*time.Minute)), + ) + + event2 := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithID("tenant2-event"), + testutil.EventFactory.WithTenantID(tenant2ID), + testutil.EventFactory.WithDestinationID(destinationID), + testutil.EventFactory.WithTopic("test.topic"), + testutil.EventFactory.WithTime(baseTime.Add(-5*time.Minute)), + ) + delivery2 := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithID("tenant2-delivery"), + testutil.DeliveryFactory.WithEventID(event2.ID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + testutil.DeliveryFactory.WithStatus("failed"), + testutil.DeliveryFactory.WithTime(baseTime.Add(-5*time.Minute)), + ) + + require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{ + {ID: idgen.DeliveryEvent(), DestinationID: destinationID, Event: *event1, Delivery: delivery1}, + {ID: idgen.DeliveryEvent(), DestinationID: destinationID, Event: *event2, Delivery: delivery2}, + })) + require.NoError(t, h.FlushWrites(ctx)) + + t.Run("TenantIsolation", func(t *testing.T) { + t.Run("ListDeliveryEvent isolates by tenant", func(t *testing.T) { + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenant1ID, + Limit: 100, + Start: &startTime, + }) + require.NoError(t, err) + require.Len(t, response.Data, 1) + assert.Equal(t, "tenant1-event", response.Data[0].Event.ID) + + response, err = logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenant2ID, + Limit: 100, + Start: &startTime, + }) + require.NoError(t, err) + require.Len(t, response.Data, 1) + assert.Equal(t, "tenant2-event", response.Data[0].Event.ID) + }) + + t.Run("RetrieveEvent isolates by tenant", func(t *testing.T) { + retrieved, err := logStore.RetrieveEvent(ctx, driver.RetrieveEventRequest{ + TenantID: tenant1ID, + EventID: "tenant2-event", + }) + require.NoError(t, err) + assert.Nil(t, retrieved) + + retrieved, err = logStore.RetrieveEvent(ctx, driver.RetrieveEventRequest{ + TenantID: tenant2ID, + EventID: "tenant1-event", + }) + require.NoError(t, err) + assert.Nil(t, retrieved) + }) + }) + + t.Run("CrossTenantQueries", func(t *testing.T) { + t.Run("ListEvent returns all tenants when TenantID empty", func(t *testing.T) { + response, err := logStore.ListEvent(ctx, driver.ListEventRequest{ + TenantID: "", + DestinationIDs: []string{destinationID}, + Limit: 100, + EventStart: &startTime, + }) + require.NoError(t, err) + require.Len(t, response.Data, 2) + + tenantsSeen := map[string]bool{} + for _, event := range response.Data { + tenantsSeen[event.TenantID] = true + } + assert.True(t, tenantsSeen[tenant1ID]) + assert.True(t, tenantsSeen[tenant2ID]) + }) + + t.Run("ListDeliveryEvent returns all tenants when TenantID empty", func(t *testing.T) { + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: "", + DestinationIDs: []string{destinationID}, + Limit: 100, + Start: &startTime, + }) + require.NoError(t, err) + require.Len(t, response.Data, 2) + }) + + t.Run("RetrieveEvent finds event across tenants when TenantID empty", func(t *testing.T) { + retrieved1, err := logStore.RetrieveEvent(ctx, driver.RetrieveEventRequest{ + TenantID: "", + EventID: "tenant1-event", + }) + require.NoError(t, err) + require.NotNil(t, retrieved1) + assert.Equal(t, tenant1ID, retrieved1.TenantID) + + retrieved2, err := logStore.RetrieveEvent(ctx, driver.RetrieveEventRequest{ + TenantID: "", + EventID: "tenant2-event", + }) + require.NoError(t, err) + require.NotNil(t, retrieved2) + assert.Equal(t, tenant2ID, retrieved2.TenantID) + }) + + t.Run("RetrieveDeliveryEvent finds delivery across tenants when TenantID empty", func(t *testing.T) { + retrieved1, err := logStore.RetrieveDeliveryEvent(ctx, driver.RetrieveDeliveryEventRequest{ + TenantID: "", + DeliveryID: "tenant1-delivery", + }) + require.NoError(t, err) + require.NotNil(t, retrieved1) + assert.Equal(t, tenant1ID, retrieved1.Event.TenantID) + + retrieved2, err := logStore.RetrieveDeliveryEvent(ctx, driver.RetrieveDeliveryEventRequest{ + TenantID: "", + DeliveryID: "tenant2-delivery", + }) + require.NoError(t, err) + require.NotNil(t, retrieved2) + assert.Equal(t, tenant2ID, retrieved2.Event.TenantID) + }) + }) +} + +func testEdgeCases(t *testing.T, ctx context.Context, logStore driver.LogStore, h Harness) { + t.Run("invalid sort values use defaults", func(t *testing.T) { + tenantID := idgen.String() + destinationID := idgen.Destination() + baseTime := time.Now().Truncate(time.Second) + + var deliveryEvents []*models.DeliveryEvent + for i := range 3 { + event := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithID(fmt.Sprintf("sort_evt_%d", i)), + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(destinationID), + testutil.EventFactory.WithTime(baseTime.Add(-time.Duration(i)*time.Hour)), + ) + delivery := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithID(fmt.Sprintf("sort_del_%d", i)), + testutil.DeliveryFactory.WithEventID(event.ID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + testutil.DeliveryFactory.WithTime(baseTime.Add(-time.Duration(i)*time.Hour)), + ) + deliveryEvents = append(deliveryEvents, &models.DeliveryEvent{ + ID: fmt.Sprintf("sort_de_%d", i), + DestinationID: destinationID, + Event: *event, + Delivery: delivery, + }) + } + require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, deliveryEvents)) + + startTime := baseTime.Add(-48 * time.Hour) + + t.Run("invalid SortOrder uses default (desc)", func(t *testing.T) { + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + SortOrder: "sideways", + Start: &startTime, + Limit: 10, + }) + require.NoError(t, err) + require.Len(t, response.Data, 3) + assert.Equal(t, "sort_del_0", response.Data[0].Delivery.ID) + assert.Equal(t, "sort_del_2", response.Data[2].Delivery.ID) + }) + }) + + t.Run("empty vs nil filter semantics", func(t *testing.T) { + tenantID := idgen.String() + destinationID := idgen.Destination() + startTime := time.Now().Add(-1 * time.Hour) + + event := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(destinationID), + testutil.EventFactory.WithTopic("test.topic"), + ) + delivery := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithEventID(event.ID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + ) + require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{ + {ID: idgen.DeliveryEvent(), DestinationID: destinationID, Event: *event, Delivery: delivery}, + })) + + t.Run("nil DestinationIDs equals empty DestinationIDs", func(t *testing.T) { + responseNil, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + DestinationIDs: nil, + Start: &startTime, + Limit: 10, + }) + require.NoError(t, err) + + responseEmpty, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + DestinationIDs: []string{}, + Start: &startTime, + Limit: 10, + }) + require.NoError(t, err) + + assert.Equal(t, len(responseNil.Data), len(responseEmpty.Data)) + assert.Equal(t, 1, len(responseNil.Data)) + }) + }) + + t.Run("time boundary precision", func(t *testing.T) { + tenantID := idgen.String() + destinationID := idgen.Destination() + boundaryTime := time.Now().Truncate(time.Second) + beforeBoundary := boundaryTime.Add(-1 * time.Second) + afterBoundary := boundaryTime.Add(1 * time.Second) + + eventBefore := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithID("time_evt_before"), + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(destinationID), + testutil.EventFactory.WithTime(beforeBoundary), + ) + eventAt := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithID("time_evt_at"), + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(destinationID), + testutil.EventFactory.WithTime(boundaryTime), + ) + eventAfter := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithID("time_evt_after"), + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(destinationID), + testutil.EventFactory.WithTime(afterBoundary), + ) + + for _, evt := range []*models.Event{eventBefore, eventAt, eventAfter} { + delivery := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithID(fmt.Sprintf("del_%s", evt.ID)), + testutil.DeliveryFactory.WithEventID(evt.ID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + testutil.DeliveryFactory.WithTime(evt.Time), + ) + require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{ + {ID: idgen.DeliveryEvent(), DestinationID: destinationID, Event: *evt, Delivery: delivery}, + })) + } + + t.Run("Start is inclusive (>=)", func(t *testing.T) { + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + Start: &boundaryTime, + Limit: 10, + }) + require.NoError(t, err) + require.Len(t, response.Data, 2) + }) + + t.Run("End is inclusive (<=)", func(t *testing.T) { + farPast := boundaryTime.Add(-1 * time.Hour) + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + Start: &farPast, + End: &boundaryTime, + Limit: 10, + }) + require.NoError(t, err) + require.Len(t, response.Data, 2) + }) + }) + + t.Run("data immutability", func(t *testing.T) { + tenantID := idgen.String() + destinationID := idgen.Destination() + startTime := time.Now().Add(-1 * time.Hour) + + event := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(destinationID), + ) + delivery := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithEventID(event.ID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + ) + require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{ + {ID: idgen.DeliveryEvent(), DestinationID: destinationID, Event: *event, Delivery: delivery}, + })) + + t.Run("modifying ListDeliveryEvent result doesn't affect subsequent queries", func(t *testing.T) { + response1, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + Limit: 10, + Start: &startTime, + }) + require.NoError(t, err) + require.Len(t, response1.Data, 1) + + originalID := response1.Data[0].Event.ID + response1.Data[0].Event.ID = "MODIFIED" + + response2, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + Limit: 10, + Start: &startTime, + }) + require.NoError(t, err) + require.Len(t, response2.Data, 1) + assert.Equal(t, originalID, response2.Data[0].Event.ID) + }) + }) + + t.Run("concurrent duplicate inserts are idempotent", func(t *testing.T) { + tenantID := idgen.String() + destinationID := idgen.Destination() + eventTime := time.Now().Add(-30 * time.Minute).Truncate(time.Second) + deliveryTime := eventTime.Add(1 * time.Second) + startTime := eventTime.Add(-1 * time.Hour) + + event := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(destinationID), + testutil.EventFactory.WithTime(eventTime), + ) + delivery := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithEventID(event.ID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + testutil.DeliveryFactory.WithStatus("success"), + testutil.DeliveryFactory.WithTime(deliveryTime), + ) + de := &models.DeliveryEvent{ + ID: idgen.DeliveryEvent(), + DestinationID: destinationID, + Event: *event, + Delivery: delivery, + } + batch := []*models.DeliveryEvent{de} + + // Race N goroutines all inserting the same record + const numGoroutines = 10 + var wg sync.WaitGroup + for range numGoroutines { + wg.Add(1) + go func() { + defer wg.Done() + _ = logStore.InsertManyDeliveryEvent(ctx, batch) + }() + } + wg.Wait() + require.NoError(t, h.FlushWrites(ctx)) + + // Assert: still exactly 1 record + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + Limit: 100, + Start: &startTime, + }) + require.NoError(t, err) + require.Len(t, response.Data, 1, "concurrent duplicate inserts should result in exactly 1 record") + }) +} + +func testCursorValidation(t *testing.T, ctx context.Context, logStore driver.LogStore, h Harness) { + t.Run("malformed cursor returns error", func(t *testing.T) { + tenantID := idgen.String() + startTime := time.Now().Add(-1 * time.Hour) + + testCases := []struct { + name string + cursor string + }{ + {"completely invalid base62", "!!!invalid!!!"}, + {"random string", "abcdef123456"}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + _, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + SortOrder: "desc", + Next: tc.cursor, + Start: &startTime, + Limit: 10, + }) + require.Error(t, err) + assert.True(t, errors.Is(err, cursor.ErrInvalidCursor), "expected cursor.ErrInvalidCursor, got: %v", err) + }) + } + }) + + t.Run("cursor works with matching sort params", func(t *testing.T) { + tenantID := idgen.String() + destinationID := idgen.Destination() + baseTime := time.Now().Truncate(time.Second) + startTime := baseTime.Add(-48 * time.Hour) + + for i := range 5 { + event := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithID(fmt.Sprintf("cursor_evt_%d", i)), + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(destinationID), + testutil.EventFactory.WithTime(baseTime.Add(time.Duration(i)*time.Second)), + ) + delivery := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithID(fmt.Sprintf("cursor_del_%d", i)), + testutil.DeliveryFactory.WithEventID(event.ID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + testutil.DeliveryFactory.WithTime(baseTime.Add(time.Duration(i)*time.Second)), + ) + require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{ + {ID: fmt.Sprintf("cursor_de_%d", i), DestinationID: destinationID, Event: *event, Delivery: delivery}, + })) + } + require.NoError(t, h.FlushWrites(ctx)) + + t.Run("delivery_time desc", func(t *testing.T) { + page1, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + SortOrder: "desc", + Start: &startTime, + Limit: 2, + }) + require.NoError(t, err) + require.NotEmpty(t, page1.Next) + + page2, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + SortOrder: "desc", + Next: page1.Next, + Start: &startTime, + Limit: 2, + }) + require.NoError(t, err) + require.NotEmpty(t, page2.Data) + }) + + t.Run("delivery_time asc", func(t *testing.T) { + page1, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + SortOrder: "asc", + Start: &startTime, + Limit: 2, + }) + require.NoError(t, err) + require.NotEmpty(t, page1.Next) + + page2, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + SortOrder: "asc", + Next: page1.Next, + Start: &startTime, + Limit: 2, + }) + require.NoError(t, err) + require.NotEmpty(t, page2.Data) + }) + }) +} diff --git a/internal/logstore/drivertest/pagination.go b/internal/logstore/drivertest/pagination.go new file mode 100644 index 000000000..0ba297224 --- /dev/null +++ b/internal/logstore/drivertest/pagination.go @@ -0,0 +1,372 @@ +package drivertest + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/hookdeck/outpost/internal/idgen" + "github.com/hookdeck/outpost/internal/logstore/driver" + "github.com/hookdeck/outpost/internal/models" + "github.com/hookdeck/outpost/internal/pagination/paginationtest" + "github.com/stretchr/testify/require" +) + +// testPagination tests cursor-based pagination using paginationtest.Suite. +func testPagination(t *testing.T, newHarness HarnessMaker) { + t.Helper() + + ctx := context.Background() + h, err := newHarness(ctx, t) + require.NoError(t, err) + t.Cleanup(h.Close) + + logStore, err := h.MakeDriver(ctx) + require.NoError(t, err) + + baseTime := time.Now().Truncate(time.Second) + farPast := baseTime.Add(-48 * time.Hour) + + t.Run("ListDeliveryEvent", func(t *testing.T) { + var tenantID, destinationID, idPrefix string + + suite := paginationtest.Suite[*models.DeliveryEvent]{ + Name: "ListDeliveryEvent", + + Cleanup: func(ctx context.Context) error { + tenantID = idgen.String() + destinationID = idgen.Destination() + idPrefix = idgen.String()[:8] + return nil + }, + + NewItem: func(i int) *models.DeliveryEvent { + eventTime := baseTime.Add(time.Duration(i) * time.Second) + deliveryTime := eventTime.Add(100 * time.Millisecond) + + event := &models.Event{ + ID: fmt.Sprintf("%s_evt_%03d", idPrefix, i), + TenantID: tenantID, + DestinationID: destinationID, + Topic: "test.topic", + EligibleForRetry: true, + Time: eventTime, + Metadata: map[string]string{}, + Data: map[string]any{}, + } + + delivery := &models.Delivery{ + ID: fmt.Sprintf("%s_del_%03d", idPrefix, i), + EventID: event.ID, + DestinationID: destinationID, + Status: "success", + Time: deliveryTime, + Code: "200", + } + + return &models.DeliveryEvent{ + ID: fmt.Sprintf("%s_de_%03d", idPrefix, i), + DestinationID: destinationID, + Event: *event, + Delivery: delivery, + } + }, + + InsertMany: func(ctx context.Context, items []*models.DeliveryEvent) error { + return logStore.InsertManyDeliveryEvent(ctx, items) + }, + + List: func(ctx context.Context, opts paginationtest.ListOpts) (paginationtest.ListResult[*models.DeliveryEvent], error) { + res, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + Limit: opts.Limit, + SortOrder: opts.Order, + Next: opts.Next, + Prev: opts.Prev, + Start: &farPast, + }) + if err != nil { + return paginationtest.ListResult[*models.DeliveryEvent]{}, err + } + return paginationtest.ListResult[*models.DeliveryEvent]{ + Items: res.Data, + Next: res.Next, + Prev: res.Prev, + }, nil + }, + + GetID: func(de *models.DeliveryEvent) string { + return de.Delivery.ID + }, + + AfterInsert: func(ctx context.Context) error { + return h.FlushWrites(ctx) + }, + } + + suite.Run(t) + }) + + t.Run("ListDeliveryEvent_WithDestinationFilter", func(t *testing.T) { + var tenantID, targetDestID, otherDestID, idPrefix string + + suite := paginationtest.Suite[*models.DeliveryEvent]{ + Name: "ListDeliveryEvent_WithDestinationFilter", + + Cleanup: func(ctx context.Context) error { + tenantID = idgen.String() + targetDestID = idgen.Destination() + otherDestID = idgen.Destination() + idPrefix = idgen.String()[:8] + return nil + }, + + NewItem: func(i int) *models.DeliveryEvent { + eventTime := baseTime.Add(time.Duration(i) * time.Second) + deliveryTime := eventTime.Add(100 * time.Millisecond) + + destID := targetDestID + if i%2 == 1 { + destID = otherDestID + } + + event := &models.Event{ + ID: fmt.Sprintf("%s_evt_%03d", idPrefix, i), + TenantID: tenantID, + DestinationID: destID, + Topic: "test.topic", + EligibleForRetry: true, + Time: eventTime, + Metadata: map[string]string{}, + Data: map[string]any{}, + } + + delivery := &models.Delivery{ + ID: fmt.Sprintf("%s_del_%03d", idPrefix, i), + EventID: event.ID, + DestinationID: destID, + Status: "success", + Time: deliveryTime, + Code: "200", + } + + return &models.DeliveryEvent{ + ID: fmt.Sprintf("%s_de_%03d", idPrefix, i), + DestinationID: destID, + Event: *event, + Delivery: delivery, + } + }, + + InsertMany: func(ctx context.Context, items []*models.DeliveryEvent) error { + return logStore.InsertManyDeliveryEvent(ctx, items) + }, + + List: func(ctx context.Context, opts paginationtest.ListOpts) (paginationtest.ListResult[*models.DeliveryEvent], error) { + res, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + DestinationIDs: []string{targetDestID}, + Limit: opts.Limit, + SortOrder: opts.Order, + Next: opts.Next, + Prev: opts.Prev, + Start: &farPast, + }) + if err != nil { + return paginationtest.ListResult[*models.DeliveryEvent]{}, err + } + return paginationtest.ListResult[*models.DeliveryEvent]{ + Items: res.Data, + Next: res.Next, + Prev: res.Prev, + }, nil + }, + + GetID: func(de *models.DeliveryEvent) string { + return de.Delivery.ID + }, + + Matches: func(de *models.DeliveryEvent) bool { + return de.DestinationID == targetDestID + }, + + AfterInsert: func(ctx context.Context) error { + return h.FlushWrites(ctx) + }, + } + + suite.Run(t) + }) + + t.Run("ListEvent", func(t *testing.T) { + var eventTenantID, eventDestID, idPrefix string + + suite := paginationtest.Suite[*models.Event]{ + Name: "ListEvent", + + Cleanup: func(ctx context.Context) error { + eventTenantID = idgen.String() + eventDestID = idgen.Destination() + idPrefix = idgen.String()[:8] + return nil + }, + + NewItem: func(i int) *models.Event { + eventTime := baseTime.Add(time.Duration(i) * time.Second) + + return &models.Event{ + ID: fmt.Sprintf("%s_evt_%03d", idPrefix, i), + TenantID: eventTenantID, + DestinationID: eventDestID, + Topic: "test.topic", + EligibleForRetry: true, + Time: eventTime, + Metadata: map[string]string{}, + Data: map[string]any{}, + } + }, + + InsertMany: func(ctx context.Context, items []*models.Event) error { + des := make([]*models.DeliveryEvent, len(items)) + for i, evt := range items { + deliveryTime := evt.Time.Add(100 * time.Millisecond) + des[i] = &models.DeliveryEvent{ + ID: fmt.Sprintf("%s_de_%03d", idPrefix, i), + DestinationID: evt.DestinationID, + Event: *evt, + Delivery: &models.Delivery{ + ID: fmt.Sprintf("%s_del_%03d", idPrefix, i), + EventID: evt.ID, + DestinationID: evt.DestinationID, + Status: "success", + Time: deliveryTime, + Code: "200", + }, + } + } + return logStore.InsertManyDeliveryEvent(ctx, des) + }, + + List: func(ctx context.Context, opts paginationtest.ListOpts) (paginationtest.ListResult[*models.Event], error) { + res, err := logStore.ListEvent(ctx, driver.ListEventRequest{ + TenantID: eventTenantID, + Limit: opts.Limit, + SortOrder: opts.Order, + Next: opts.Next, + Prev: opts.Prev, + EventStart: &farPast, + }) + if err != nil { + return paginationtest.ListResult[*models.Event]{}, err + } + return paginationtest.ListResult[*models.Event]{ + Items: res.Data, + Next: res.Next, + Prev: res.Prev, + }, nil + }, + + GetID: func(e *models.Event) string { + return e.ID + }, + + AfterInsert: func(ctx context.Context) error { + return h.FlushWrites(ctx) + }, + } + + suite.Run(t) + }) + + t.Run("ListEvent_WithDestinationFilter", func(t *testing.T) { + var tenantID, targetDestID, otherDestID, idPrefix string + + suite := paginationtest.Suite[*models.Event]{ + Name: "ListEvent_WithDestinationFilter", + + Cleanup: func(ctx context.Context) error { + tenantID = idgen.String() + targetDestID = idgen.Destination() + otherDestID = idgen.Destination() + idPrefix = idgen.String()[:8] + return nil + }, + + NewItem: func(i int) *models.Event { + eventTime := baseTime.Add(time.Duration(i) * time.Second) + + destID := targetDestID + if i%2 == 1 { + destID = otherDestID + } + + return &models.Event{ + ID: fmt.Sprintf("%s_evt_%03d", idPrefix, i), + TenantID: tenantID, + DestinationID: destID, + Topic: "test.topic", + EligibleForRetry: true, + Time: eventTime, + Metadata: map[string]string{}, + Data: map[string]any{}, + } + }, + + InsertMany: func(ctx context.Context, items []*models.Event) error { + des := make([]*models.DeliveryEvent, len(items)) + for i, evt := range items { + deliveryTime := evt.Time.Add(100 * time.Millisecond) + des[i] = &models.DeliveryEvent{ + ID: fmt.Sprintf("%s_de_%03d", idPrefix, i), + DestinationID: evt.DestinationID, + Event: *evt, + Delivery: &models.Delivery{ + ID: fmt.Sprintf("%s_del_%03d", idPrefix, i), + EventID: evt.ID, + DestinationID: evt.DestinationID, + Status: "success", + Time: deliveryTime, + Code: "200", + }, + } + } + return logStore.InsertManyDeliveryEvent(ctx, des) + }, + + List: func(ctx context.Context, opts paginationtest.ListOpts) (paginationtest.ListResult[*models.Event], error) { + res, err := logStore.ListEvent(ctx, driver.ListEventRequest{ + TenantID: tenantID, + DestinationIDs: []string{targetDestID}, + Limit: opts.Limit, + SortOrder: opts.Order, + Next: opts.Next, + Prev: opts.Prev, + EventStart: &farPast, + }) + if err != nil { + return paginationtest.ListResult[*models.Event]{}, err + } + return paginationtest.ListResult[*models.Event]{ + Items: res.Data, + Next: res.Next, + Prev: res.Prev, + }, nil + }, + + GetID: func(e *models.Event) string { + return e.ID + }, + + Matches: func(e *models.Event) bool { + return e.DestinationID == targetDestID + }, + + AfterInsert: func(ctx context.Context) error { + return h.FlushWrites(ctx) + }, + } + + suite.Run(t) + }) +} diff --git a/internal/logstore/memlogstore/memlogstore.go b/internal/logstore/memlogstore/memlogstore.go index 4c4edd83c..a68abcda6 100644 --- a/internal/logstore/memlogstore/memlogstore.go +++ b/internal/logstore/memlogstore/memlogstore.go @@ -2,13 +2,15 @@ package memlogstore import ( "context" - "errors" + "fmt" "sort" "sync" + "time" "github.com/hookdeck/outpost/internal/cursor" "github.com/hookdeck/outpost/internal/logstore/driver" "github.com/hookdeck/outpost/internal/models" + "github.com/hookdeck/outpost/internal/pagination" ) const ( @@ -41,16 +43,12 @@ func (s *memLogStore) ListEvent(ctx context.Context, req driver.ListEventRequest sortOrder = "desc" } - nextPosition, err := cursor.Decode(req.Next, cursorResourceEvent, cursorVersion) - if err != nil { - return driver.ListEventResponse{}, convertCursorError(err) - } - prevPosition, err := cursor.Decode(req.Prev, cursorResourceEvent, cursorVersion) - if err != nil { - return driver.ListEventResponse{}, convertCursorError(err) + limit := req.Limit + if limit <= 0 { + limit = 100 } - // Dedupe by event ID + // Dedupe by event ID and filter eventMap := make(map[string]*models.Event) for _, de := range s.deliveryEvents { if !s.matchesEventFilter(&de.Event, req) { @@ -61,72 +59,88 @@ func (s *memLogStore) ListEvent(ctx context.Context, req driver.ListEventRequest } } - var filtered []*models.Event + var allEvents []*models.Event for _, event := range eventMap { - filtered = append(filtered, event) + allEvents = append(allEvents, event) } - isDesc := sortOrder == "desc" - sort.Slice(filtered, func(i, j int) bool { - if !filtered[i].Time.Equal(filtered[j].Time) { - if isDesc { - return filtered[i].Time.After(filtered[j].Time) - } - return filtered[i].Time.Before(filtered[j].Time) - } - if isDesc { - return filtered[i].ID > filtered[j].ID - } - return filtered[i].ID < filtered[j].ID - }) - - limit := req.Limit - if limit <= 0 { - limit = 100 + // eventWithTimeID pairs an event with its sortable time ID for cursor operations. + type eventWithTimeID struct { + event *models.Event + timeID string } - startIdx := 0 - if nextPosition != "" { - for i, event := range filtered { - if event.ID == nextPosition { - startIdx = i - break - } + // Build list with time IDs + eventsWithTimeID := make([]eventWithTimeID, len(allEvents)) + for i, e := range allEvents { + eventsWithTimeID[i] = eventWithTimeID{ + event: e, + timeID: makeTimeID(e.Time, e.ID), } - } else if prevPosition != "" { - for i, event := range filtered { - if event.ID == prevPosition { - startIdx = i - limit - if startIdx < 0 { - startIdx = 0 + } + + res, err := pagination.Run(ctx, pagination.Config[eventWithTimeID]{ + Limit: limit, + Order: sortOrder, + Next: req.Next, + Prev: req.Prev, + Fetch: func(_ context.Context, q pagination.QueryInput) ([]eventWithTimeID, error) { + // Sort based on query direction + isDesc := q.SortDir == "desc" + sort.Slice(eventsWithTimeID, func(i, j int) bool { + if isDesc { + return eventsWithTimeID[i].timeID > eventsWithTimeID[j].timeID + } + return eventsWithTimeID[i].timeID < eventsWithTimeID[j].timeID + }) + + // Filter using q.Compare (like SQL WHERE clause) + var filtered []eventWithTimeID + for _, e := range eventsWithTimeID { + // If no cursor, include all items + // If cursor exists, filter using Compare operator + if q.CursorPos == "" || compareTimeID(e.timeID, q.Compare, q.CursorPos) { + filtered = append(filtered, e) } - break } - } - } - endIdx := startIdx + limit - if endIdx > len(filtered) { - endIdx = len(filtered) - } + // Return up to limit items + if len(filtered) > q.Limit { + filtered = filtered[:q.Limit] + } - data := make([]*models.Event, endIdx-startIdx) - for i, event := range filtered[startIdx:endIdx] { - data[i] = copyEvent(event) + result := make([]eventWithTimeID, len(filtered)) + for i, e := range filtered { + result[i] = eventWithTimeID{ + event: copyEvent(e.event), + timeID: e.timeID, + } + } + return result, nil + }, + Cursor: pagination.Cursor[eventWithTimeID]{ + Encode: func(e eventWithTimeID) string { + return cursor.Encode(cursorResourceEvent, cursorVersion, e.timeID) + }, + Decode: func(c string) (string, error) { + return cursor.Decode(c, cursorResourceEvent, cursorVersion) + }, + }, + }) + if err != nil { + return driver.ListEventResponse{}, err } - var nextEncoded, prevEncoded string - if endIdx < len(filtered) { - nextEncoded = cursor.Encode(cursorResourceEvent, cursorVersion, filtered[endIdx].ID) - } - if startIdx > 0 { - prevEncoded = cursor.Encode(cursorResourceEvent, cursorVersion, filtered[startIdx].ID) + // Extract events from results + data := make([]*models.Event, len(res.Items)) + for i, item := range res.Items { + data[i] = item.event } return driver.ListEventResponse{ Data: data, - Next: nextEncoded, - Prev: prevEncoded, + Next: res.Next, + Prev: res.Prev, }, nil } @@ -211,84 +225,97 @@ func (s *memLogStore) ListDeliveryEvent(ctx context.Context, req driver.ListDeli sortOrder = "desc" } - nextPosition, err := cursor.Decode(req.Next, cursorResourceDelivery, cursorVersion) - if err != nil { - return driver.ListDeliveryEventResponse{}, convertCursorError(err) - } - prevPosition, err := cursor.Decode(req.Prev, cursorResourceDelivery, cursorVersion) - if err != nil { - return driver.ListDeliveryEventResponse{}, convertCursorError(err) + limit := req.Limit + if limit <= 0 { + limit = 100 } - var filtered []*models.DeliveryEvent + // Filter delivery events + var allDeliveryEvents []*models.DeliveryEvent for _, de := range s.deliveryEvents { if !s.matchesFilter(de, req) { continue } - filtered = append(filtered, de) + allDeliveryEvents = append(allDeliveryEvents, de) } - isDesc := sortOrder == "desc" - sort.Slice(filtered, func(i, j int) bool { - if !filtered[i].Delivery.Time.Equal(filtered[j].Delivery.Time) { - if isDesc { - return filtered[i].Delivery.Time.After(filtered[j].Delivery.Time) - } - return filtered[i].Delivery.Time.Before(filtered[j].Delivery.Time) - } - if isDesc { - return filtered[i].Delivery.ID > filtered[j].Delivery.ID - } - return filtered[i].Delivery.ID < filtered[j].Delivery.ID - }) - - limit := req.Limit - if limit <= 0 { - limit = 100 + // deliveryEventWithTimeID pairs a delivery event with its sortable time ID. + type deliveryEventWithTimeID struct { + de *models.DeliveryEvent + timeID string } - startIdx := 0 - if nextPosition != "" { - for i, de := range filtered { - if de.ID == nextPosition { - startIdx = i - break - } + // Build list with time IDs (using delivery time) + deliveryEventsWithTimeID := make([]deliveryEventWithTimeID, len(allDeliveryEvents)) + for i, de := range allDeliveryEvents { + deliveryEventsWithTimeID[i] = deliveryEventWithTimeID{ + de: de, + timeID: makeTimeID(de.Delivery.Time, de.Delivery.ID), } - } else if prevPosition != "" { - for i, de := range filtered { - if de.ID == prevPosition { - startIdx = i - limit - if startIdx < 0 { - startIdx = 0 + } + + res, err := pagination.Run(ctx, pagination.Config[deliveryEventWithTimeID]{ + Limit: limit, + Order: sortOrder, + Next: req.Next, + Prev: req.Prev, + Fetch: func(_ context.Context, q pagination.QueryInput) ([]deliveryEventWithTimeID, error) { + // Sort based on query direction + isDesc := q.SortDir == "desc" + sort.Slice(deliveryEventsWithTimeID, func(i, j int) bool { + if isDesc { + return deliveryEventsWithTimeID[i].timeID > deliveryEventsWithTimeID[j].timeID + } + return deliveryEventsWithTimeID[i].timeID < deliveryEventsWithTimeID[j].timeID + }) + + // Filter using q.Compare (like SQL WHERE clause) + var filtered []deliveryEventWithTimeID + for _, de := range deliveryEventsWithTimeID { + // If no cursor, include all items + // If cursor exists, filter using Compare operator + if q.CursorPos == "" || compareTimeID(de.timeID, q.Compare, q.CursorPos) { + filtered = append(filtered, de) } - break } - } - } - endIdx := startIdx + limit - if endIdx > len(filtered) { - endIdx = len(filtered) - } + // Return up to limit items + if len(filtered) > q.Limit { + filtered = filtered[:q.Limit] + } - data := make([]*models.DeliveryEvent, endIdx-startIdx) - for i, de := range filtered[startIdx:endIdx] { - data[i] = copyDeliveryEvent(de) + result := make([]deliveryEventWithTimeID, len(filtered)) + for i, de := range filtered { + result[i] = deliveryEventWithTimeID{ + de: copyDeliveryEvent(de.de), + timeID: de.timeID, + } + } + return result, nil + }, + Cursor: pagination.Cursor[deliveryEventWithTimeID]{ + Encode: func(de deliveryEventWithTimeID) string { + return cursor.Encode(cursorResourceDelivery, cursorVersion, de.timeID) + }, + Decode: func(c string) (string, error) { + return cursor.Decode(c, cursorResourceDelivery, cursorVersion) + }, + }, + }) + if err != nil { + return driver.ListDeliveryEventResponse{}, err } - var nextEncoded, prevEncoded string - if endIdx < len(filtered) { - nextEncoded = cursor.Encode(cursorResourceDelivery, cursorVersion, filtered[endIdx].ID) - } - if startIdx > 0 { - prevEncoded = cursor.Encode(cursorResourceDelivery, cursorVersion, filtered[startIdx].ID) + // Extract delivery events from results + data := make([]*models.DeliveryEvent, len(res.Items)) + for i, item := range res.Items { + data[i] = item.de } return driver.ListDeliveryEventResponse{ Data: data, - Next: nextEncoded, - Prev: prevEncoded, + Next: res.Next, + Prev: res.Prev, }, nil } @@ -434,10 +461,21 @@ func copyDelivery(d *models.Delivery) *models.Delivery { return copied } -// convertCursorError converts cursor package errors to driver errors. -func convertCursorError(err error) error { - if errors.Is(err, cursor.ErrInvalidCursor) || errors.Is(err, cursor.ErrVersionMismatch) { - return driver.ErrInvalidCursor +// makeTimeID creates a sortable string from time and ID, similar to pglogstore's time_id. +// Uses fixed-width nanoseconds to ensure correct string sorting (RFC3339Nano has variable width). +// Format: "2006-01-02T15:04:05.000000000Z_id" +func makeTimeID(t time.Time, id string) string { + return fmt.Sprintf("%s_%s", t.UTC().Format("2006-01-02T15:04:05.000000000Z07:00"), id) +} + +// compareTimeID compares two time IDs using the given operator. +func compareTimeID(a, op, b string) bool { + switch op { + case "<": + return a < b + case ">": + return a > b + default: + return false } - return err } diff --git a/internal/logstore/pglogstore/pglogstore.go b/internal/logstore/pglogstore/pglogstore.go index 82d0e0b9e..9e026f3f3 100644 --- a/internal/logstore/pglogstore/pglogstore.go +++ b/internal/logstore/pglogstore/pglogstore.go @@ -2,13 +2,14 @@ package pglogstore import ( "context" - "errors" "fmt" + "strings" "time" "github.com/hookdeck/outpost/internal/cursor" "github.com/hookdeck/outpost/internal/logstore/driver" "github.com/hookdeck/outpost/internal/models" + "github.com/hookdeck/outpost/internal/pagination" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" ) @@ -29,6 +30,12 @@ func NewLogStore(db *pgxpool.Pool) driver.LogStore { } } +// eventWithTimeID wraps an event with its time_id for cursor encoding. +type eventWithTimeID struct { + *models.Event + TimeID string +} + func (s *logStore) ListEvent(ctx context.Context, req driver.ListEventRequest) (driver.ListEventResponse, error) { sortOrder := req.SortOrder if sortOrder != "asc" && sortOrder != "desc" { @@ -40,64 +47,51 @@ func (s *logStore) ListEvent(ctx context.Context, req driver.ListEventRequest) ( limit = 100 } - nextPosition, err := cursor.Decode(req.Next, cursorResourceEvent, cursorVersion) - if err != nil { - return driver.ListEventResponse{}, convertCursorError(err) - } - prevPosition, err := cursor.Decode(req.Prev, cursorResourceEvent, cursorVersion) + res, err := pagination.Run(ctx, pagination.Config[eventWithTimeID]{ + Limit: limit, + Order: sortOrder, + Next: req.Next, + Prev: req.Prev, + Fetch: func(ctx context.Context, q pagination.QueryInput) ([]eventWithTimeID, error) { + query, args := buildEventQuery(req, q) + rows, err := s.db.Query(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("query failed: %w", err) + } + defer rows.Close() + return scanEvents(rows) + }, + Cursor: pagination.Cursor[eventWithTimeID]{ + Encode: func(e eventWithTimeID) string { + return cursor.Encode(cursorResourceEvent, cursorVersion, e.TimeID) + }, + Decode: func(c string) (string, error) { + return cursor.Decode(c, cursorResourceEvent, cursorVersion) + }, + }, + }) if err != nil { - return driver.ListEventResponse{}, convertCursorError(err) + return driver.ListEventResponse{}, err } - goingBackward := prevPosition != "" - - var orderByClause, finalOrderByClause string - if sortOrder == "desc" { - if goingBackward { - orderByClause = "time ASC, id ASC" - } else { - orderByClause = "time DESC, id DESC" - } - finalOrderByClause = "time DESC, id DESC" - } else { - if goingBackward { - orderByClause = "time DESC, id DESC" - } else { - orderByClause = "time ASC, id ASC" - } - finalOrderByClause = "time ASC, id ASC" + // Extract events from results + data := make([]*models.Event, len(res.Items)) + for i, item := range res.Items { + data[i] = item.Event } - // Build cursor conditions using time_id column (generated column in events table) - var cursorCondition string - if sortOrder == "desc" { - cursorCondition = "AND ($6::text = '' OR time_id < $6::text) AND ($7::text = '' OR time_id > $7::text)" - } else { - cursorCondition = "AND ($6::text = '' OR time_id > $6::text) AND ($7::text = '' OR time_id < $7::text)" - } + return driver.ListEventResponse{ + Data: data, + Next: res.Next, + Prev: res.Prev, + }, nil +} + +func buildEventQuery(req driver.ListEventRequest, q pagination.QueryInput) (string, []any) { + cursorCondition := fmt.Sprintf("AND ($6::text = '' OR time_id %s $6::text)", q.Compare) + orderByClause := fmt.Sprintf("time %s, id %s", strings.ToUpper(q.SortDir), strings.ToUpper(q.SortDir)) query := fmt.Sprintf(` - WITH filtered AS ( - SELECT - id, - tenant_id, - destination_id, - time, - topic, - eligible_for_retry, - data, - metadata, - time_id - FROM events - WHERE ($1::text = '' OR tenant_id = $1) - AND (array_length($2::text[], 1) IS NULL OR destination_id = ANY($2)) - AND (array_length($3::text[], 1) IS NULL OR topic = ANY($3)) - AND ($4::timestamptz IS NULL OR time >= $4) - AND ($5::timestamptz IS NULL OR time <= $5) - %s - ORDER BY %s - LIMIT $8 - ) SELECT id, tenant_id, @@ -108,31 +102,32 @@ func (s *logStore) ListEvent(ctx context.Context, req driver.ListEventRequest) ( data, metadata, time_id - FROM filtered + FROM events + WHERE ($1::text = '' OR tenant_id = $1) + AND (array_length($2::text[], 1) IS NULL OR destination_id = ANY($2)) + AND (array_length($3::text[], 1) IS NULL OR topic = ANY($3)) + AND ($4::timestamptz IS NULL OR time >= $4) + AND ($5::timestamptz IS NULL OR time <= $5) + %s ORDER BY %s - `, cursorCondition, orderByClause, finalOrderByClause) + LIMIT $7 + `, cursorCondition, orderByClause) - rows, err := s.db.Query(ctx, query, + args := []any{ req.TenantID, // $1 req.DestinationIDs, // $2 req.Topics, // $3 req.EventStart, // $4 req.EventEnd, // $5 - nextPosition, // $6 - prevPosition, // $7 - limit+1, // $8 - fetch one extra to detect if there's more - ) - if err != nil { - return driver.ListEventResponse{}, fmt.Errorf("query failed: %w", err) + q.CursorPos, // $6 + q.Limit, // $7 } - defer rows.Close() - type rowData struct { - event *models.Event - timeID string - } - var results []rowData + return query, args +} +func scanEvents(rows pgx.Rows) ([]eventWithTimeID, error) { + var results []eventWithTimeID for rows.Next() { var ( id string @@ -146,7 +141,7 @@ func (s *logStore) ListEvent(ctx context.Context, req driver.ListEventRequest) ( timeID string ) - err := rows.Scan( + if err := rows.Scan( &id, &tenantID, &destinationID, @@ -156,76 +151,36 @@ func (s *logStore) ListEvent(ctx context.Context, req driver.ListEventRequest) ( &data, &metadata, &timeID, - ) - if err != nil { - return driver.ListEventResponse{}, fmt.Errorf("scan failed: %w", err) - } - - event := &models.Event{ - ID: id, - TenantID: tenantID, - DestinationID: destinationID, - Topic: topic, - EligibleForRetry: eligibleForRetry, - Time: eventTime, - Data: data, - Metadata: metadata, + ); err != nil { + return nil, fmt.Errorf("scan failed: %w", err) } - results = append(results, rowData{event: event, timeID: timeID}) + results = append(results, eventWithTimeID{ + Event: &models.Event{ + ID: id, + TenantID: tenantID, + DestinationID: destinationID, + Topic: topic, + EligibleForRetry: eligibleForRetry, + Time: eventTime, + Data: data, + Metadata: metadata, + }, + TimeID: timeID, + }) } if err := rows.Err(); err != nil { - return driver.ListEventResponse{}, fmt.Errorf("rows error: %w", err) + return nil, fmt.Errorf("rows error: %w", err) } - var hasMore bool - if len(results) > limit { - hasMore = true - if goingBackward { - results = results[1:] - } else { - results = results[:limit] - } - } - - data := make([]*models.Event, len(results)) - for i, r := range results { - data[i] = r.event - } - - var nextEncoded, prevEncoded string - if len(results) > 0 { - getPosition := func(r rowData) string { - return r.timeID - } - - encodeCursor := func(position string) string { - return cursor.Encode(cursorResourceEvent, cursorVersion, position) - } - - if prevPosition != "" { - nextEncoded = encodeCursor(getPosition(results[len(results)-1])) - if hasMore { - prevEncoded = encodeCursor(getPosition(results[0])) - } - } else if nextPosition != "" { - prevEncoded = encodeCursor(getPosition(results[0])) - if hasMore { - nextEncoded = encodeCursor(getPosition(results[len(results)-1])) - } - } else { - if hasMore { - nextEncoded = encodeCursor(getPosition(results[len(results)-1])) - } - } - } + return results, nil +} - return driver.ListEventResponse{ - Data: data, - Next: nextEncoded, - Prev: prevEncoded, - }, nil +// deliveryEventWithTimeID wraps a delivery event with its time_delivery_id for cursor encoding. +type deliveryEventWithTimeID struct { + *models.DeliveryEvent + TimeDeliveryID string } func (s *logStore) ListDeliveryEvent(ctx context.Context, req driver.ListDeliveryEventRequest) (driver.ListDeliveryEventResponse, error) { @@ -239,94 +194,84 @@ func (s *logStore) ListDeliveryEvent(ctx context.Context, req driver.ListDeliver limit = 100 } - nextPosition, err := cursor.Decode(req.Next, cursorResourceDelivery, cursorVersion) - if err != nil { - return driver.ListDeliveryEventResponse{}, convertCursorError(err) - } - prevPosition, err := cursor.Decode(req.Prev, cursorResourceDelivery, cursorVersion) + res, err := pagination.Run(ctx, pagination.Config[deliveryEventWithTimeID]{ + Limit: limit, + Order: sortOrder, + Next: req.Next, + Prev: req.Prev, + Fetch: func(ctx context.Context, q pagination.QueryInput) ([]deliveryEventWithTimeID, error) { + query, args := buildDeliveryQuery(req, q) + rows, err := s.db.Query(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("query failed: %w", err) + } + defer rows.Close() + return scanDeliveryEvents(rows) + }, + Cursor: pagination.Cursor[deliveryEventWithTimeID]{ + Encode: func(de deliveryEventWithTimeID) string { + return cursor.Encode(cursorResourceDelivery, cursorVersion, de.TimeDeliveryID) + }, + Decode: func(c string) (string, error) { + return cursor.Decode(c, cursorResourceDelivery, cursorVersion) + }, + }, + }) if err != nil { - return driver.ListDeliveryEventResponse{}, convertCursorError(err) + return driver.ListDeliveryEventResponse{}, err } - cursorCol := "time_delivery_id" - goingBackward := prevPosition != "" - - var orderByClause, finalOrderByClause string - if sortOrder == "desc" { - if goingBackward { - orderByClause = "delivery_time ASC, delivery_id ASC" - } else { - orderByClause = "delivery_time DESC, delivery_id DESC" - } - finalOrderByClause = "delivery_time DESC, delivery_id DESC" - } else { - if goingBackward { - orderByClause = "delivery_time DESC, delivery_id DESC" - } else { - orderByClause = "delivery_time ASC, delivery_id ASC" - } - finalOrderByClause = "delivery_time ASC, delivery_id ASC" + // Extract delivery events from results + data := make([]*models.DeliveryEvent, len(res.Items)) + for i, item := range res.Items { + data[i] = item.DeliveryEvent } - var cursorCondition string - if sortOrder == "desc" { - cursorCondition = fmt.Sprintf("AND ($8::text = '' OR %s < $8::text) AND ($9::text = '' OR %s > $9::text)", cursorCol, cursorCol) - } else { - cursorCondition = fmt.Sprintf("AND ($8::text = '' OR %s > $8::text) AND ($9::text = '' OR %s < $9::text)", cursorCol, cursorCol) - } + return driver.ListDeliveryEventResponse{ + Data: data, + Next: res.Next, + Prev: res.Prev, + }, nil +} + +func buildDeliveryQuery(req driver.ListDeliveryEventRequest, q pagination.QueryInput) (string, []any) { + cursorCondition := fmt.Sprintf("AND ($8::text = '' OR idx.time_delivery_id %s $8::text)", q.Compare) + orderByClause := fmt.Sprintf("idx.delivery_time %s, idx.delivery_id %s", strings.ToUpper(q.SortDir), strings.ToUpper(q.SortDir)) query := fmt.Sprintf(` - WITH filtered AS ( - SELECT - idx.event_id, - idx.delivery_id, - idx.tenant_id, - idx.destination_id, - idx.event_time, - idx.delivery_time, - idx.topic, - idx.status, - idx.time_event_id, - idx.time_delivery_id, - idx.manual, - idx.attempt - FROM event_delivery_index idx - WHERE ($1::text = '' OR idx.tenant_id = $1) - AND ($2::text = '' OR idx.event_id = $2) - AND (array_length($3::text[], 1) IS NULL OR idx.destination_id = ANY($3)) - AND ($4::text = '' OR idx.status = $4) - AND (array_length($5::text[], 1) IS NULL OR idx.topic = ANY($5)) - AND ($6::timestamptz IS NULL OR idx.delivery_time >= $6) - AND ($7::timestamptz IS NULL OR idx.delivery_time <= $7) - %s - ORDER BY %s - LIMIT $10 - ) SELECT - f.event_id, - f.delivery_id, - f.destination_id, - f.event_time, - f.delivery_time, - f.topic, - f.status, - f.time_event_id, - f.time_delivery_id, + idx.event_id, + idx.delivery_id, + idx.destination_id, + idx.event_time, + idx.delivery_time, + idx.topic, + idx.status, + idx.time_delivery_id, e.tenant_id, e.eligible_for_retry, e.data, e.metadata, d.code, d.response_data, - f.manual, - f.attempt - FROM filtered f - JOIN events e ON e.id = f.event_id AND e.time = f.event_time - JOIN deliveries d ON d.id = f.delivery_id AND d.time = f.delivery_time + idx.manual, + idx.attempt + FROM event_delivery_index idx + JOIN events e ON e.id = idx.event_id AND e.time = idx.event_time + JOIN deliveries d ON d.id = idx.delivery_id AND d.time = idx.delivery_time + WHERE ($1::text = '' OR idx.tenant_id = $1) + AND ($2::text = '' OR idx.event_id = $2) + AND (array_length($3::text[], 1) IS NULL OR idx.destination_id = ANY($3)) + AND ($4::text = '' OR idx.status = $4) + AND (array_length($5::text[], 1) IS NULL OR idx.topic = ANY($5)) + AND ($6::timestamptz IS NULL OR idx.delivery_time >= $6) + AND ($7::timestamptz IS NULL OR idx.delivery_time <= $7) + %s ORDER BY %s - `, cursorCondition, orderByClause, finalOrderByClause) + LIMIT $9 + `, cursorCondition, orderByClause) - rows, err := s.db.Query(ctx, query, + args := []any{ req.TenantID, // $1 req.EventID, // $2 req.DestinationIDs, // $3 @@ -334,22 +279,15 @@ func (s *logStore) ListDeliveryEvent(ctx context.Context, req driver.ListDeliver req.Topics, // $5 req.Start, // $6 req.End, // $7 - nextPosition, // $8 - prevPosition, // $9 - limit+1, // $10 - fetch one extra to detect if there's more - ) - if err != nil { - return driver.ListDeliveryEventResponse{}, fmt.Errorf("query failed: %w", err) + q.CursorPos, // $8 + q.Limit, // $9 } - defer rows.Close() - type rowData struct { - de *models.DeliveryEvent - timeEventID string - timeDeliveryID string - } - var results []rowData + return query, args +} +func scanDeliveryEvents(rows pgx.Rows) ([]deliveryEventWithTimeID, error) { + var results []deliveryEventWithTimeID for rows.Next() { var ( eventID string @@ -359,7 +297,6 @@ func (s *logStore) ListDeliveryEvent(ctx context.Context, req driver.ListDeliver deliveryTime time.Time topic string status string - timeEventID string timeDeliveryID string tenantID string eligibleForRetry bool @@ -371,7 +308,7 @@ func (s *logStore) ListDeliveryEvent(ctx context.Context, req driver.ListDeliver attempt int ) - err := rows.Scan( + if err := rows.Scan( &eventID, &deliveryID, &destinationID, @@ -379,7 +316,6 @@ func (s *logStore) ListDeliveryEvent(ctx context.Context, req driver.ListDeliver &deliveryTime, &topic, &status, - &timeEventID, &timeDeliveryID, &tenantID, &eligibleForRetry, @@ -389,91 +325,45 @@ func (s *logStore) ListDeliveryEvent(ctx context.Context, req driver.ListDeliver &responseData, &manual, &attempt, - ) - if err != nil { - return driver.ListDeliveryEventResponse{}, fmt.Errorf("scan failed: %w", err) + ); err != nil { + return nil, fmt.Errorf("scan failed: %w", err) } - de := &models.DeliveryEvent{ - ID: fmt.Sprintf("%s_%s", eventID, deliveryID), - DestinationID: destinationID, - Manual: manual, - Attempt: attempt, - Event: models.Event{ - ID: eventID, - TenantID: tenantID, - DestinationID: destinationID, - Topic: topic, - EligibleForRetry: eligibleForRetry, - Time: eventTime, - Data: data, - Metadata: metadata, - }, - Delivery: &models.Delivery{ - ID: deliveryID, - EventID: eventID, + results = append(results, deliveryEventWithTimeID{ + DeliveryEvent: &models.DeliveryEvent{ + ID: fmt.Sprintf("%s_%s", eventID, deliveryID), DestinationID: destinationID, - Status: status, - Time: deliveryTime, - Code: code, - ResponseData: responseData, + Manual: manual, + Attempt: attempt, + Event: models.Event{ + ID: eventID, + TenantID: tenantID, + DestinationID: destinationID, + Topic: topic, + EligibleForRetry: eligibleForRetry, + Time: eventTime, + Data: data, + Metadata: metadata, + }, + Delivery: &models.Delivery{ + ID: deliveryID, + EventID: eventID, + DestinationID: destinationID, + Status: status, + Time: deliveryTime, + Code: code, + ResponseData: responseData, + }, }, - } - - results = append(results, rowData{de: de, timeEventID: timeEventID, timeDeliveryID: timeDeliveryID}) + TimeDeliveryID: timeDeliveryID, + }) } if err := rows.Err(); err != nil { - return driver.ListDeliveryEventResponse{}, fmt.Errorf("rows error: %w", err) - } - - var hasMore bool - if len(results) > limit { - hasMore = true - if goingBackward { - results = results[1:] - } else { - results = results[:limit] - } - } - - data := make([]*models.DeliveryEvent, len(results)) - for i, r := range results { - data[i] = r.de + return nil, fmt.Errorf("rows error: %w", err) } - var nextEncoded, prevEncoded string - if len(results) > 0 { - getPosition := func(r rowData) string { - return r.timeDeliveryID - } - - encodeCursor := func(position string) string { - return cursor.Encode(cursorResourceDelivery, cursorVersion, position) - } - - if prevPosition != "" { - nextEncoded = encodeCursor(getPosition(results[len(results)-1])) - if hasMore { - prevEncoded = encodeCursor(getPosition(results[0])) - } - } else if nextPosition != "" { - prevEncoded = encodeCursor(getPosition(results[0])) - if hasMore { - nextEncoded = encodeCursor(getPosition(results[len(results)-1])) - } - } else { - if hasMore { - nextEncoded = encodeCursor(getPosition(results[len(results)-1])) - } - } - } - - return driver.ListDeliveryEventResponse{ - Data: data, - Next: nextEncoded, - Prev: prevEncoded, - }, nil + return results, nil } func (s *logStore) RetrieveEvent(ctx context.Context, req driver.RetrieveEventRequest) (*models.Event, error) { @@ -819,11 +709,3 @@ func deliveryArrays(deliveries []*models.Delivery, deliveryEvents []*models.Deli attempts, } } - -// convertCursorError converts cursor package errors to driver errors. -func convertCursorError(err error) error { - if errors.Is(err, cursor.ErrInvalidCursor) || errors.Is(err, cursor.ErrVersionMismatch) { - return driver.ErrInvalidCursor - } - return err -} diff --git a/internal/pagination/pagination.go b/internal/pagination/pagination.go new file mode 100644 index 000000000..394288a90 --- /dev/null +++ b/internal/pagination/pagination.go @@ -0,0 +1,155 @@ +// Package pagination provides a generic cursor-based pagination implementation. +// It handles direction detection, n+1 pattern processing, and cursor building +// for stores that need paginated list operations. +package pagination + +import ( + "context" + "slices" +) + +// Direction indicates whether pagination is moving forward or backward. +type Direction int + +const ( + Forward Direction = iota + Backward +) + +// Cursor groups encode/decode functions for cursor serialization. +// These are paired together to support versioning and backward compatibility. +type Cursor[T any] struct { + Encode func(T) string // item → cursor string + Decode func(string) (string, error) // cursor string → position for query +} + +// Config contains all parameters for a paginated query. +type Config[T any] struct { + Limit int + Order string // "asc" or "desc" - user's requested order + Next string // next cursor (empty if none) + Prev string // prev cursor (empty if none) + + Fetch func(context.Context, QueryInput) ([]T, error) + Cursor Cursor[T] +} + +// QueryInput is passed to the Fetch function with computed query parameters. +type QueryInput struct { + Limit int + Compare string // "<" or ">" - cursor comparison operator + SortDir string // "asc" or "desc" - query sort direction (may differ from user's Order) + CursorPos string // decoded cursor position (empty if first page) +} + +// Result contains the paginated results and cursors. +type Result[T any] struct { + Items []T + Next string // empty if no more pages ahead + Prev string // empty if no more pages behind (or first page) +} + +// Run executes a paginated query. +func Run[T any](ctx context.Context, cfg Config[T]) (*Result[T], error) { + // 1. Determine direction + direction := determineDirection(cfg.Prev) + isFirstPage := cfg.Next == "" && cfg.Prev == "" + + // 2. Decode cursor position + var cursorPos string + if cfg.Next != "" { + pos, err := cfg.Cursor.Decode(cfg.Next) + if err != nil { + return nil, err + } + cursorPos = pos + } else if cfg.Prev != "" { + pos, err := cfg.Cursor.Decode(cfg.Prev) + if err != nil { + return nil, err + } + cursorPos = pos + } + + // 3. Compute query parameters (XOR logic) + isDesc := cfg.Order == "desc" + isBackward := direction == Backward + + // Determine compare operator based on order and direction + // desc + forward: "<" (want smaller/older items) + // desc + backward: ">" (want larger/newer items) + // asc + forward: ">" (want larger/newer items) + // asc + backward: "<" (want smaller/older items) + compare := "<" + if isDesc == isBackward { + compare = ">" + } + sortDir := cfg.Order + if isBackward { + sortDir = flip(cfg.Order) + } + + // 4. Fetch with n+1 + items, err := cfg.Fetch(ctx, QueryInput{ + Limit: cfg.Limit + 1, + Compare: compare, + SortDir: sortDir, + CursorPos: cursorPos, + }) + if err != nil { + return nil, err + } + + // 5. Process n+1 results + hasMore := len(items) > cfg.Limit + if hasMore { + items = items[:cfg.Limit] + } + + // 6. Reverse if backward + if isBackward { + slices.Reverse(items) + } + + // 7. Build cursors + var next, prev string + if len(items) > 0 { + firstEncoded := cfg.Cursor.Encode(items[0]) + lastEncoded := cfg.Cursor.Encode(items[len(items)-1]) + + switch { + case isFirstPage: + if hasMore { + next = lastEncoded + } + case direction == Forward: + prev = firstEncoded + if hasMore { + next = lastEncoded + } + case direction == Backward: + next = lastEncoded + if hasMore { + prev = firstEncoded + } + } + } + + return &Result[T]{Items: items, Next: next, Prev: prev}, nil +} + +// determineDirection returns the pagination direction based on cursors. +func determineDirection(prev string) Direction { + if prev != "" { + return Backward + } + return Forward +} + +// flip reverses the sort direction. +func flip(order string) string { + if order == "asc" { + return "desc" + } + return "asc" +} diff --git a/internal/pagination/paginationtest/suite.go b/internal/pagination/paginationtest/suite.go new file mode 100644 index 000000000..24fabfb7c --- /dev/null +++ b/internal/pagination/paginationtest/suite.go @@ -0,0 +1,420 @@ +// Package paginationtest provides a reusable test suite for cursor-based pagination. +package paginationtest + +import ( + "context" + "slices" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// ListOpts contains options for a paginated list request. +type ListOpts struct { + Limit int + Order string // "asc" or "desc" + Next string + Prev string +} + +// ListResult contains the result of a paginated list request. +type ListResult[T any] struct { + Items []T + Next string + Prev string +} + +// Suite is a reusable test suite for cursor-based pagination. +// +// # Basic Usage +// +// For testing pagination without filters, provide NewItem and InsertMany: +// +// suite := paginationtest.Suite[*Event]{ +// Name: "pglogstore", +// NewItem: func(i int) *Event { ... }, +// InsertMany: func(ctx context.Context, items []*Event) error { ... }, +// List: func(ctx context.Context, opts ListOpts) (ListResult[*Event], error) { ... }, +// GetID: func(e *Event) string { return e.ID }, +// } +// suite.Run(t) +// +// # Testing with Filters +// +// When testing pagination with filters, use Matches to tell the suite which +// created items should appear in results: +// +// suite := paginationtest.Suite[*Event]{ +// NewItem: func(i int) *Event { +// typ := "order.created" +// if i%2 == 0 { typ = "order.updated" } +// return &Event{Type: typ, ...} +// }, +// List: func(ctx context.Context, opts ListOpts) (ListResult[*Event], error) { +// return store.ListEvents(ctx, ListEventsRequest{EventType: "order.created", ...}) +// }, +// Matches: func(e *Event) bool { +// return e.Type == "order.created" // mirrors filter in List +// }, +// } +// +// # Async/Eventually Consistent Stores +// +// For stores with async writes, use AfterInsert: +// +// suite := paginationtest.Suite[*Event]{ +// AfterInsert: func(ctx context.Context) error { +// time.Sleep(100 * time.Millisecond) +// return nil +// }, +// } +type Suite[T any] struct { + Name string // Store name for test output + NewItem func(index int) T // Create item at index (0=oldest) + InsertMany func(ctx context.Context, items []T) error // Batch insert + List func(ctx context.Context, opts ListOpts) (ListResult[T], error) + GetID func(T) string // Unique ID for comparison + Matches func(T) bool // Filter predicate (optional) + AfterInsert func(ctx context.Context) error // Flush hook (optional) + Cleanup func(ctx context.Context) error // Reset state (optional) +} + +// Run executes all pagination test cases. +func (s Suite[T]) Run(t *testing.T) { + t.Helper() + + t.Run("ForwardTraversal", s.testForwardTraversal) + t.Run("BackwardTraversal", s.testBackwardTraversal) + t.Run("RoundTrip", s.testRoundTrip) + t.Run("FirstPageNoPrev", s.testFirstPageNoPrev) + t.Run("LastPageNoNext", s.testLastPageNoNext) + t.Run("EmptyResults", s.testEmptyResults) + t.Run("PartialLastPage", s.testPartialLastPage) + t.Run("ExactPageBoundary", s.testExactPageBoundary) + t.Run("SingleItem", s.testSingleItem) + t.Run("OrderAsc", s.testOrderAsc) + t.Run("OrderDesc", s.testOrderDesc) +} + +// setup creates items and returns the expected items after filtering. +func (s Suite[T]) setup(t *testing.T, ctx context.Context, count int) []T { + t.Helper() + + if s.Cleanup != nil { + require.NoError(t, s.Cleanup(ctx)) + } + + items := make([]T, count) + for i := range count { + items[i] = s.NewItem(i) + } + + if count > 0 { + require.NoError(t, s.InsertMany(ctx, items)) + } + + if s.AfterInsert != nil { + require.NoError(t, s.AfterInsert(ctx)) + } + + // Filter to expected items + if s.Matches != nil { + var expected []T + for _, item := range items { + if s.Matches(item) { + expected = append(expected, item) + } + } + return expected + } + return items +} + +// testForwardTraversal verifies forward pagination traverses all items without overlap. +func (s Suite[T]) testForwardTraversal(t *testing.T) { + ctx := context.Background() + expected := s.setup(t, ctx, 10) + + var collected []T + var seenIDs = make(map[string]bool) + pageSize := 3 + + // Page 1 + res, err := s.List(ctx, ListOpts{Limit: pageSize, Order: "desc"}) + require.NoError(t, err) + assert.Empty(t, res.Prev, "first page should have no prev cursor") + collected = append(collected, res.Items...) + for _, item := range res.Items { + id := s.GetID(item) + assert.False(t, seenIDs[id], "duplicate item ID: %s", id) + seenIDs[id] = true + } + + // Continue until no more pages + for res.Next != "" { + res, err = s.List(ctx, ListOpts{Limit: pageSize, Order: "desc", Next: res.Next}) + require.NoError(t, err) + collected = append(collected, res.Items...) + for _, item := range res.Items { + id := s.GetID(item) + assert.False(t, seenIDs[id], "duplicate item ID: %s", id) + seenIDs[id] = true + } + } + + assert.Len(t, collected, len(expected), "forward traversal should collect all items") +} + +// testBackwardTraversal verifies backward pagination returns to previous pages correctly. +func (s Suite[T]) testBackwardTraversal(t *testing.T) { + ctx := context.Background() + expected := s.setup(t, ctx, 9) + if len(expected) < 3 { + t.Skip("need at least 3 items for backward traversal test") + } + + pageSize := 3 + + // Navigate forward to collect pages + var pages []ListResult[T] + res, err := s.List(ctx, ListOpts{Limit: pageSize, Order: "desc"}) + require.NoError(t, err) + pages = append(pages, res) + + for res.Next != "" { + res, err = s.List(ctx, ListOpts{Limit: pageSize, Order: "desc", Next: res.Next}) + require.NoError(t, err) + pages = append(pages, res) + } + + if len(pages) < 2 { + t.Skip("need at least 2 pages for backward traversal test") + } + + // Navigate backward from the last page + lastPage := pages[len(pages)-1] + var backPages []ListResult[T] + + res = lastPage + for res.Prev != "" { + res, err = s.List(ctx, ListOpts{Limit: pageSize, Order: "desc", Prev: res.Prev}) + require.NoError(t, err) + backPages = append(backPages, res) + } + + // Verify backward pages match forward pages (in reverse order) + for i, backPage := range backPages { + forwardIdx := len(pages) - 2 - i // -2 because we started from last page + if forwardIdx < 0 { + break + } + forwardPage := pages[forwardIdx] + require.Len(t, backPage.Items, len(forwardPage.Items), "page %d item count mismatch", i) + for j, item := range backPage.Items { + assert.Equal(t, s.GetID(forwardPage.Items[j]), s.GetID(item), + "backward page %d item %d mismatch", i, j) + } + } + + // Verify we're back at first page (no prev cursor) + if len(backPages) > 0 { + assert.Empty(t, backPages[len(backPages)-1].Prev, "back at first page should have no prev") + } +} + +// testRoundTrip verifies forward then backward returns to same data. +func (s Suite[T]) testRoundTrip(t *testing.T) { + ctx := context.Background() + expected := s.setup(t, ctx, 9) + if len(expected) < 6 { + t.Skip("need at least 6 items for round trip test") + } + + pageSize := 3 + + // Page 1 + page1, err := s.List(ctx, ListOpts{Limit: pageSize, Order: "desc"}) + require.NoError(t, err) + + if page1.Next == "" { + t.Skip("only one page, cannot test round trip") + } + + // Page 2 + page2, err := s.List(ctx, ListOpts{Limit: pageSize, Order: "desc", Next: page1.Next}) + require.NoError(t, err) + + // Back to page 1 + back1, err := s.List(ctx, ListOpts{Limit: pageSize, Order: "desc", Prev: page2.Prev}) + require.NoError(t, err) + + // Verify back1 matches page1 + require.Len(t, back1.Items, len(page1.Items)) + for i, item := range back1.Items { + assert.Equal(t, s.GetID(page1.Items[i]), s.GetID(item), + "round trip item %d mismatch", i) + } +} + +// testFirstPageNoPrev verifies first page must never have a prev cursor. +func (s Suite[T]) testFirstPageNoPrev(t *testing.T) { + ctx := context.Background() + s.setup(t, ctx, 5) + + res, err := s.List(ctx, ListOpts{Limit: 3, Order: "desc"}) + require.NoError(t, err) + assert.Empty(t, res.Prev, "first page should have no prev cursor") +} + +// testLastPageNoNext verifies last page must never have a next cursor. +func (s Suite[T]) testLastPageNoNext(t *testing.T) { + ctx := context.Background() + expected := s.setup(t, ctx, 5) + if len(expected) == 0 { + t.Skip("no items to test") + } + + pageSize := 3 + + // Navigate to last page + res, err := s.List(ctx, ListOpts{Limit: pageSize, Order: "desc"}) + require.NoError(t, err) + + for res.Next != "" { + res, err = s.List(ctx, ListOpts{Limit: pageSize, Order: "desc", Next: res.Next}) + require.NoError(t, err) + } + + assert.Empty(t, res.Next, "last page should have no next cursor") + if len(expected) > pageSize { + assert.NotEmpty(t, res.Prev, "last page should have prev cursor when more data exists") + } +} + +// testEmptyResults verifies empty result set has no cursors. +func (s Suite[T]) testEmptyResults(t *testing.T) { + ctx := context.Background() + s.setup(t, ctx, 0) + + res, err := s.List(ctx, ListOpts{Limit: 10, Order: "desc"}) + require.NoError(t, err) + assert.Empty(t, res.Items, "should have no items") + assert.Empty(t, res.Next, "should have no next cursor") + assert.Empty(t, res.Prev, "should have no prev cursor") +} + +// testPartialLastPage verifies last page may have fewer items than limit. +func (s Suite[T]) testPartialLastPage(t *testing.T) { + ctx := context.Background() + expected := s.setup(t, ctx, 7) + if len(expected) == 0 { + t.Skip("no items to test") + } + + pageSize := 3 // 7 items / 3 = 2 full pages + 1 partial page with 1 item + + // Navigate to last page + res, err := s.List(ctx, ListOpts{Limit: pageSize, Order: "desc"}) + require.NoError(t, err) + + for res.Next != "" { + res, err = s.List(ctx, ListOpts{Limit: pageSize, Order: "desc", Next: res.Next}) + require.NoError(t, err) + } + + expectedLastPageSize := len(expected) % pageSize + if expectedLastPageSize == 0 { + expectedLastPageSize = pageSize + } + assert.Len(t, res.Items, expectedLastPageSize, "last page should have %d items", expectedLastPageSize) + assert.Empty(t, res.Next, "last page should have no next cursor") +} + +// testExactPageBoundary verifies when items divide evenly by limit, last page is full but has no next. +func (s Suite[T]) testExactPageBoundary(t *testing.T) { + ctx := context.Background() + expected := s.setup(t, ctx, 6) + if len(expected) != 6 { + t.Skip("need exactly 6 items for exact page boundary test") + } + + pageSize := 3 // exactly 2 pages + + // Page 1 + page1, err := s.List(ctx, ListOpts{Limit: pageSize, Order: "desc"}) + require.NoError(t, err) + assert.Len(t, page1.Items, pageSize) + assert.NotEmpty(t, page1.Next, "page 1 should have next cursor") + + // Page 2 + page2, err := s.List(ctx, ListOpts{Limit: pageSize, Order: "desc", Next: page1.Next}) + require.NoError(t, err) + assert.Len(t, page2.Items, pageSize) + assert.Empty(t, page2.Next, "page 2 should have no next cursor (last page)") +} + +// testSingleItem verifies single item has no pagination cursors. +func (s Suite[T]) testSingleItem(t *testing.T) { + ctx := context.Background() + expected := s.setup(t, ctx, 1) + if len(expected) != 1 { + t.Skip("need exactly 1 item for single item test") + } + + res, err := s.List(ctx, ListOpts{Limit: 10, Order: "desc"}) + require.NoError(t, err) + assert.Len(t, res.Items, 1, "should have 1 item") + assert.Empty(t, res.Next, "should have no next cursor") + assert.Empty(t, res.Prev, "should have no prev cursor") +} + +// testOrderAsc verifies ascending order returns oldest items first. +func (s Suite[T]) testOrderAsc(t *testing.T) { + ctx := context.Background() + expected := s.setup(t, ctx, 5) + if len(expected) == 0 { + t.Skip("no items to test") + } + + res, err := s.List(ctx, ListOpts{Limit: 10, Order: "asc"}) + require.NoError(t, err) + require.NotEmpty(t, res.Items) + + // First item should be the oldest (index 0) + assert.Equal(t, s.GetID(expected[0]), s.GetID(res.Items[0]), + "first item in asc order should be oldest") + + // Verify ascending order + for i := 1; i < len(res.Items); i++ { + // Items should be in same order as expected (which is created in index order) + assert.Equal(t, s.GetID(expected[i]), s.GetID(res.Items[i]), + "item %d should match expected order", i) + } +} + +// testOrderDesc verifies descending order returns newest items first. +func (s Suite[T]) testOrderDesc(t *testing.T) { + ctx := context.Background() + expected := s.setup(t, ctx, 5) + if len(expected) == 0 { + t.Skip("no items to test") + } + + res, err := s.List(ctx, ListOpts{Limit: 10, Order: "desc"}) + require.NoError(t, err) + require.NotEmpty(t, res.Items) + + // First item should be the newest (last index) + assert.Equal(t, s.GetID(expected[len(expected)-1]), s.GetID(res.Items[0]), + "first item in desc order should be newest") + + // Verify descending order (reverse of expected) + reversed := make([]T, len(expected)) + copy(reversed, expected) + slices.Reverse(reversed) + for i := 0; i < len(res.Items); i++ { + assert.Equal(t, s.GetID(reversed[i]), s.GetID(res.Items[i]), + "item %d should match expected desc order", i) + } +}