-
Notifications
You must be signed in to change notification settings - Fork 83
[FSSDK-11513] limit number of events in the eventStore #1053
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change | ||||
---|---|---|---|---|---|---|
|
@@ -18,10 +18,10 @@ import { EventProcessor, ProcessableEvent } from "./event_processor"; | |||||
import { getBatchedAsync, getBatchedSync, Store } from "../utils/cache/store"; | ||||||
import { EventDispatcher, EventDispatcherResponse, LogEvent } from "./event_dispatcher/event_dispatcher"; | ||||||
import { buildLogEvent } from "./event_builder/log_event"; | ||||||
import { BackoffController, ExponentialBackoff, IntervalRepeater, Repeater } from "../utils/repeater/repeater"; | ||||||
import { BackoffController, ExponentialBackoff, Repeater } from "../utils/repeater/repeater"; | ||||||
import { LoggerFacade } from '../logging/logger'; | ||||||
import { BaseService, ServiceState, StartupLog } from "../service"; | ||||||
import { Consumer, Fn, Producer } from "../utils/type"; | ||||||
import { Consumer, Fn, Maybe, Producer } from "../utils/type"; | ||||||
import { RunResult, runWithRetry } from "../utils/executor/backoff_retry_runner"; | ||||||
import { isSuccessStatusCode } from "../utils/http_request_handler/http_util"; | ||||||
import { EventEmitter } from "../utils/event_emitter/event_emitter"; | ||||||
|
@@ -31,13 +31,16 @@ import { FAILED_TO_DISPATCH_EVENTS, SERVICE_NOT_RUNNING } from "error_message"; | |||||
import { OptimizelyError } from "../error/optimizly_error"; | ||||||
import { sprintf } from "../utils/fns"; | ||||||
import { SERVICE_STOPPED_BEFORE_RUNNING } from "../service"; | ||||||
import { EVENT_STORE_FULL } from "../message/log_message"; | ||||||
|
||||||
export const DEFAULT_MIN_BACKOFF = 1000; | ||||||
export const DEFAULT_MAX_BACKOFF = 32000; | ||||||
export const MAX_EVENTS_IN_STORE = 500; | ||||||
|
||||||
export type EventWithId = { | ||||||
id: string; | ||||||
event: ProcessableEvent; | ||||||
notStored?: boolean; | ||||||
}; | ||||||
|
||||||
export type RetryConfig = { | ||||||
|
@@ -59,7 +62,7 @@ export type BatchEventProcessorConfig = { | |||||
|
||||||
type EventBatch = { | ||||||
request: LogEvent, | ||||||
ids: string[], | ||||||
events: EventWithId[], | ||||||
} | ||||||
|
||||||
export const LOGGER_NAME = 'BatchEventProcessor'; | ||||||
|
@@ -70,11 +73,13 @@ export class BatchEventProcessor extends BaseService implements EventProcessor { | |||||
private eventQueue: EventWithId[] = []; | ||||||
private batchSize: number; | ||||||
private eventStore?: Store<EventWithId>; | ||||||
private eventCountInStore: Maybe<number> = undefined; | ||||||
private maxEventsInStore: number = MAX_EVENTS_IN_STORE; | ||||||
private dispatchRepeater: Repeater; | ||||||
private failedEventRepeater?: Repeater; | ||||||
private idGenerator: IdGenerator = new IdGenerator(); | ||||||
private runningTask: Map<string, RunResult<EventDispatcherResponse>> = new Map(); | ||||||
private dispatchingEventIds: Set<string> = new Set(); | ||||||
private dispatchingEvents: Map<string, EventWithId> = new Map(); | ||||||
private eventEmitter: EventEmitter<{ dispatch: LogEvent }> = new EventEmitter(); | ||||||
private retryConfig?: RetryConfig; | ||||||
|
||||||
|
@@ -84,11 +89,13 @@ export class BatchEventProcessor extends BaseService implements EventProcessor { | |||||
this.closingEventDispatcher = config.closingEventDispatcher; | ||||||
this.batchSize = config.batchSize; | ||||||
this.eventStore = config.eventStore; | ||||||
|
||||||
this.retryConfig = config.retryConfig; | ||||||
|
||||||
this.dispatchRepeater = config.dispatchRepeater; | ||||||
this.dispatchRepeater.setTask(() => this.flush()); | ||||||
|
||||||
this.maxEventsInStore = Math.max(2 * config.batchSize, MAX_EVENTS_IN_STORE); | ||||||
this.failedEventRepeater = config.failedEventRepeater; | ||||||
this.failedEventRepeater?.setTask(() => this.retryFailedEvents()); | ||||||
if (config.logger) { | ||||||
|
@@ -111,7 +118,7 @@ export class BatchEventProcessor extends BaseService implements EventProcessor { | |||||
} | ||||||
|
||||||
const keys = (await this.eventStore.getKeys()).filter( | ||||||
(k) => !this.dispatchingEventIds.has(k) && !this.eventQueue.find((e) => e.id === k) | ||||||
(k) => !this.dispatchingEvents.has(k) && !this.eventQueue.find((e) => e.id === k) | ||||||
); | ||||||
|
||||||
const events = await (this.eventStore.operation === 'sync' ? | ||||||
|
@@ -138,7 +145,7 @@ export class BatchEventProcessor extends BaseService implements EventProcessor { | |||||
(currentBatch.length > 0 && !areEventContextsEqual(currentBatch[0].event, event.event))) { | ||||||
batches.push({ | ||||||
request: buildLogEvent(currentBatch.map((e) => e.event)), | ||||||
ids: currentBatch.map((e) => e.id), | ||||||
events: currentBatch, | ||||||
}); | ||||||
currentBatch = []; | ||||||
} | ||||||
|
@@ -148,7 +155,7 @@ export class BatchEventProcessor extends BaseService implements EventProcessor { | |||||
if (currentBatch.length > 0) { | ||||||
batches.push({ | ||||||
request: buildLogEvent(currentBatch.map((e) => e.event)), | ||||||
ids: currentBatch.map((e) => e.id), | ||||||
events: currentBatch, | ||||||
}); | ||||||
} | ||||||
|
||||||
|
@@ -163,15 +170,15 @@ export class BatchEventProcessor extends BaseService implements EventProcessor { | |||||
} | ||||||
|
||||||
const events: ProcessableEvent[] = []; | ||||||
const ids: string[] = []; | ||||||
const eventWithIds: EventWithId[] = []; | ||||||
|
||||||
this.eventQueue.forEach((event) => { | ||||||
events.push(event.event); | ||||||
ids.push(event.id); | ||||||
eventWithIds.push(event); | ||||||
}); | ||||||
|
||||||
this.eventQueue = []; | ||||||
return { request: buildLogEvent(events), ids }; | ||||||
return { request: buildLogEvent(events), events: eventWithIds }; | ||||||
} | ||||||
|
||||||
private async executeDispatch(request: LogEvent, closing = false): Promise<EventDispatcherResponse> { | ||||||
|
@@ -185,10 +192,10 @@ export class BatchEventProcessor extends BaseService implements EventProcessor { | |||||
} | ||||||
|
||||||
private dispatchBatch(batch: EventBatch, closing: boolean): void { | ||||||
const { request, ids } = batch; | ||||||
const { request, events } = batch; | ||||||
|
||||||
ids.forEach((id) => { | ||||||
this.dispatchingEventIds.add(id); | ||||||
events.forEach((event) => { | ||||||
this.dispatchingEvents.set(event.id, event); | ||||||
}); | ||||||
|
||||||
const runResult: RunResult<EventDispatcherResponse> = this.retryConfig | ||||||
|
@@ -205,9 +212,11 @@ export class BatchEventProcessor extends BaseService implements EventProcessor { | |||||
this.runningTask.set(taskId, runResult); | ||||||
|
||||||
runResult.result.then((res) => { | ||||||
ids.forEach((id) => { | ||||||
this.dispatchingEventIds.delete(id); | ||||||
this.eventStore?.remove(id); | ||||||
events.forEach((event) => { | ||||||
this.eventStore?.remove(event.id); | ||||||
if (!event.notStored && this.eventCountInStore) { | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The check 'this.eventCountInStore' may fail when the count is 0 (since 0 is falsy), which could prevent the intended decrement. Consider replacing it with 'this.eventCountInStore !== undefined' to ensure the condition works correctly even when the count is 0.
Suggested change
Copilot uses AI. Check for mistakes. Positive FeedbackNegative Feedback |
||||||
this.eventCountInStore--; | ||||||
} | ||||||
}); | ||||||
return Promise.resolve(); | ||||||
}).catch((err) => { | ||||||
|
@@ -216,7 +225,7 @@ export class BatchEventProcessor extends BaseService implements EventProcessor { | |||||
this.logger?.error(err); | ||||||
}).finally(() => { | ||||||
this.runningTask.delete(taskId); | ||||||
ids.forEach((id) => this.dispatchingEventIds.delete(id)); | ||||||
events.forEach((event) => this.dispatchingEvents.delete(event.id)); | ||||||
}); | ||||||
} | ||||||
|
||||||
|
@@ -235,12 +244,12 @@ export class BatchEventProcessor extends BaseService implements EventProcessor { | |||||
return Promise.reject(new OptimizelyError(SERVICE_NOT_RUNNING, 'BatchEventProcessor')); | ||||||
} | ||||||
|
||||||
const eventWithId = { | ||||||
const eventWithId: EventWithId = { | ||||||
id: this.idGenerator.getId(), | ||||||
event: event, | ||||||
}; | ||||||
|
||||||
await this.eventStore?.set(eventWithId.id, eventWithId); | ||||||
await this.storeEvent(eventWithId); | ||||||
|
||||||
if (this.eventQueue.length > 0 && !areEventContextsEqual(this.eventQueue[0].event, event)) { | ||||||
this.flush(); | ||||||
|
@@ -253,7 +262,35 @@ export class BatchEventProcessor extends BaseService implements EventProcessor { | |||||
} else if (!this.dispatchRepeater.isRunning()) { | ||||||
this.dispatchRepeater.start(); | ||||||
} | ||||||
} | ||||||
|
||||||
private async findEventCountInStore(): Promise<void> { | ||||||
if (this.eventStore && this.eventCountInStore === undefined) { | ||||||
try { | ||||||
const keys = await this.eventStore.getKeys(); | ||||||
this.eventCountInStore = keys.length; | ||||||
} catch (e) { | ||||||
this.logger?.error(e); | ||||||
} | ||||||
} | ||||||
} | ||||||
|
||||||
private async storeEvent(eventWithId: EventWithId): Promise<void> { | ||||||
await this.findEventCountInStore(); | ||||||
if (this.eventCountInStore !== undefined && this.eventCountInStore >= this.maxEventsInStore) { | ||||||
this.logger?.info(EVENT_STORE_FULL, eventWithId.event.uuid); | ||||||
eventWithId.notStored = true; | ||||||
return; | ||||||
} | ||||||
|
||||||
await Promise.resolve(this.eventStore?.set(eventWithId.id, eventWithId)).then(() => { | ||||||
if (this.eventCountInStore !== undefined) { | ||||||
this.eventCountInStore++; | ||||||
} | ||||||
}).catch((e) => { | ||||||
eventWithId.notStored = true; | ||||||
this.logger?.error(e); | ||||||
}); | ||||||
} | ||||||
|
||||||
start(): void { | ||||||
|
Uh oh!
There was an error while loading. Please reload this page.