LogPoller support for MaxLogsKept

core/chains/evm/logpoller/log_poller.go

@@ -657,12 +657,14 @@ func (lp *logPoller) backgroundWorkerRun() {
 				blockPruneTick = time.After(utils.WithJitter(lp.pollPeriod * 100))
 			}
 		case <-logPruneTick:
+			lp.lggr.Infof("Pruning LogPoller logs...")
 			logPruneTick = time.After(utils.WithJitter(lp.pollPeriod * 2401)) // = 7^5 avoids common factors with 1000
-			if allRemoved, err := lp.PruneExpiredLogs(ctx); err != nil {
-				lp.lggr.Errorw("Unable to prune expired logs", "err", err)
-			} else if !allRemoved {
+			if lp.PruneExpiredLogs(ctx) {
+				lp.lggr.Infof("Finished pruning LogPoller logs. Next pruning in ~ %s", lp.pollPeriod*2401)
+			} else {
 				// Tick faster when cleanup can't keep up with the pace of new logs
 				logPruneTick = time.After(utils.WithJitter(lp.pollPeriod * 241))
+				lp.lggr.Warnf("Partially finished pruning LogPoller logs. Will resume in ~ %s", lp.pollPeriod*241)
 			}
 		}
 	}
@@ -1090,10 +1092,31 @@ func (lp *logPoller) PruneOldBlocks(ctx context.Context) (bool, error) {
 }
 
 // PruneExpiredLogs logs that are older than their retention period defined in Filter.
-// Returns whether all logs eligible for pruning were removed. If logPrunePageSize is set to 0, it will always return true.
-func (lp *logPoller) PruneExpiredLogs(ctx context.Context) (bool, error) {
+// Returns whether all logs eligible for pruning were removed. If logPrunePageSize is set to 0, it can only
+// return false if an actual error is encountered
+func (lp *logPoller) PruneExpiredLogs(ctx context.Context) (done bool) {
+	done = true
+
 	rowsRemoved, err := lp.orm.DeleteExpiredLogs(ctx, lp.logPrunePageSize)
-	return lp.logPrunePageSize == 0 || rowsRemoved < lp.logPrunePageSize, err
+	if err != nil {
+		lp.lggr.Errorw("Unable to prune expired logs", "err", err)
+		done = false
+	} else if lp.logPrunePageSize != 0 && rowsRemoved == lp.logPrunePageSize {
+		done = false
+	}
+
+	rowIds, err := lp.orm.FindExcessLogs(ctx, lp.logPrunePageSize)
+	if err != nil {
+		lp.lggr.Errorw("Unable to find excess logs for pruning", "err", err)
+		return false
+	}
+	rowsRemoved, err = lp.orm.DeleteLogsByRowId(ctx, rowIds)
+	if err != nil {
+		lp.lggr.Errorw("Unable to prune excess logs", "err", err)
+	} else if lp.logPrunePageSize != 0 && rowsRemoved == lp.logPrunePageSize {
+		done = false
+	}
+	return done
 }
 
 // Logs returns logs matching topics and address (exactly) in the given block range,

core/chains/evm/logpoller/observability.go

@@ -142,6 +142,18 @@ func (o *ObservedORM) DeleteExpiredLogs(ctx context.Context, limit int64) (int64
 	})
 }
 
+func (o *ObservedORM) FindExcessLogs(ctx context.Context, limit int64) ([]uint64, error) {
+	return withObservedQueryAndResults[uint64](o, "FindExcessLogs", func() ([]uint64, error) {
+		return o.ORM.FindExcessLogs(ctx, limit)
+	})
+}
+
+func (o *ObservedORM) DeleteLogsByRowId(ctx context.Context, rowIds []uint64) (int64, error) {
+	return withObservedExecAndRowsAffected(o, "DeleteLogsByRowId", del, func() (int64, error) {
+		return o.ORM.DeleteLogsByRowId(ctx, rowIds)
+	})
+}
+
 func (o *ObservedORM) SelectBlockByNumber(ctx context.Context, n int64) (*LogPollerBlock, error) {
 	return withObservedQuery(o, "SelectBlockByNumber", func() (*LogPollerBlock, error) {
 		return o.ORM.SelectBlockByNumber(ctx, n)