Skip to content

Activity reset #1730

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions packages/client/src/async-completion-client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,13 @@ export class ActivityCancelledError extends Error {}
@SymbolBasedInstanceOfError('ActivityPausedError')
export class ActivityPausedError extends Error {}

/**
* Thrown by {@link AsyncCompletionClient.heartbeat} when the reporting Activity
* has been reset.
*/
@SymbolBasedInstanceOfError('ActivityResetError')
export class ActivityResetError extends Error {}

/**
* Options used to configure {@link AsyncCompletionClient}
*/
Expand Down Expand Up @@ -219,6 +226,7 @@ export class AsyncCompletionClient extends BaseClient {
const payloads = await encodeToPayloads(this.dataConverter, details);
let cancelRequested = false;
let paused = false;
let reset = false;
try {
if (taskTokenOrFullActivityId instanceof Uint8Array) {
const response = await this.workflowService.recordActivityTaskHeartbeat({
Expand All @@ -229,6 +237,7 @@ export class AsyncCompletionClient extends BaseClient {
});
cancelRequested = !!response.cancelRequested;
paused = !!response.activityPaused;
reset = !!response.activityReset;
} else {
const response = await this.workflowService.recordActivityTaskHeartbeatById({
identity: this.options.identity,
Expand All @@ -238,13 +247,17 @@ export class AsyncCompletionClient extends BaseClient {
});
cancelRequested = !!response.cancelRequested;
paused = !!response.activityPaused;
reset = !!response.activityReset;
}
} catch (err) {
this.handleError(err);
}
if (cancelRequested) {
throw new ActivityCancelledError('cancelled');
}
if (reset) {
throw new ActivityResetError('reset');
}
if (paused) {
throw new ActivityPausedError('paused');
}
Expand Down
31 changes: 23 additions & 8 deletions packages/test/src/helpers-integration.ts
Original file line number Diff line number Diff line change
Expand Up @@ -285,7 +285,11 @@ export function configurableHelpers<T>(
};
}

export async function setActivityPauseState(handle: WorkflowHandle, activityId: string, pause: boolean): Promise<void> {
export async function setActivityState(
handle: WorkflowHandle,
activityId: string,
state: 'pause' | 'unpause' | 'reset'
): Promise<void> {
const desc = await handle.describe();
const req = {
namespace: handle.client.options.namespace,
Expand All @@ -295,22 +299,33 @@ export async function setActivityPauseState(handle: WorkflowHandle, activityId:
},
id: activityId,
};
if (pause) {
if (state === 'pause') {
await handle.client.workflowService.pauseActivity(req);
} else {
} else if (state === 'unpause') {
await handle.client.workflowService.unpauseActivity(req);
} else {
const resetReq = { ...req, resetHeartbeat: true };
await handle.client.workflowService.resetActivity(resetReq);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am looking into doing this in other langs too, can you confirm the current dev server supports this and/or do we need to upgrade to an RC one for tests?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As in supports the resetActivity call? Yes should be fine. Latest CLI version is 1.4.1, which has temporal server version 1.28.0. ResetActivity was included in 1.27.1 with some improvements in 1.28.0

}
await waitUntil(async () => {
const { raw } = await handle.describe();
const activityInfo = raw.pendingActivities?.find((act) => act.activityId === activityId);
// If we are pausing: success when either
// • paused flag is true OR
// • the activity vanished (it completed / retried)
if (pause) return activityInfo ? activityInfo.paused ?? false : true;
// If we are unpausing: success when either
// • paused flag is false OR
// • the activity vanished (already completed)
return activityInfo ? !activityInfo.paused : true;
if (state === 'pause') {
return activityInfo ? activityInfo.paused ?? false : true;
} else if (state === 'unpause') {
// If we are unpausing: success when either
// • paused flag is false OR
// • the activity vanished (already completed)
return activityInfo ? !activityInfo.paused : true;
} else {
// If we are resetting, success when either
// • heartbeat details have been reset OR
// • the activity vanished (completed / retried)
return activityInfo ? activityInfo.heartbeatDetails === null : true;
}
}, 15000);
}

Expand Down
2 changes: 0 additions & 2 deletions packages/test/src/test-integration-split-three.ts
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,6 @@ test(
await worker.runUntil(handle.result());
let firstChild = true;
const history = await handle.fetchHistory();
console.log('events');
for (const event of history?.events ?? []) {
switch (event.eventType) {
case temporal.api.enums.v1.EventType.EVENT_TYPE_WORKFLOW_EXECUTION_STARTED:
Expand Down Expand Up @@ -184,7 +183,6 @@ test('workflow start without priorities sees undefined for the key', configMacro
const { env, createWorkerWithDefaults } = config;
const { startWorkflow } = configurableHelpers(t, t.context.workflowBundle, env);
const worker = await createWorkerWithDefaults(t, { activities });
console.log('STARTING WORKFLOW');

const handle1 = await startWorkflow(workflows.priorityWorkflow, {
args: [true, undefined],
Expand Down
40 changes: 36 additions & 4 deletions packages/test/src/test-integration-workflows.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ import {
hasActivityHeartbeat,
helpers,
makeTestFunction,
setActivityPauseState,
setActivityState,
} from './helpers-integration';
import { overrideSdkInternalFlag } from './mock-internal-flags';
import { heartbeatCancellationDetailsActivity } from './activities/heartbeat-cancellation-details';
Expand Down Expand Up @@ -1459,7 +1459,7 @@ test('Activity pause returns expected cancellation details', async (t) => {
}, 10000);

// Now pause the activity
await setActivityPauseState(handle, testActivityId, true);
await setActivityState(handle, testActivityId, 'pause');
// Get the result - should contain pause cancellation details
const result = await handle.result();

Expand Down Expand Up @@ -1494,15 +1494,47 @@ test('Activity can be cancelled via pause and retry after unpause', async (t) =>
return !!(activityInfo && (await hasActivityHeartbeat(handle, testActivityId, 'heartbeated')));
}, 10000);

await setActivityPauseState(handle, testActivityId, true);
await setActivityState(handle, testActivityId, 'pause');
await waitUntil(async () => hasActivityHeartbeat(handle, testActivityId, 'finally-complete'), 10000);
await setActivityPauseState(handle, testActivityId, false);
await setActivityState(handle, testActivityId, 'unpause');

const result = await handle.result();
t.true(result == null);
});
});

test('Activity reset returns expected cancellation details', async (t) => {
const { createWorker, startWorkflow } = helpers(t);
const worker = await createWorker({
activities: {
heartbeatCancellationDetailsActivity,
},
});

await worker.runUntil(async () => {
const testActivityId = randomUUID();
const handle = await startWorkflow(heartbeatPauseWorkflow, { args: [testActivityId, true, 1] });

// Wait for it to exist and heartbeat
await waitUntil(async () => {
const { raw } = await handle.describe();
const activityInfo = raw.pendingActivities?.find((act) => act.activityId === testActivityId);
return !!(activityInfo && (await hasActivityHeartbeat(handle, testActivityId, 'heartbeated')));
}, 10000);

await setActivityState(handle, testActivityId, 'reset');
const result = await handle.result();
t.deepEqual(result, {
cancelRequested: false,
notFound: false,
paused: false,
timedOut: false,
workerShutdown: false,
reset: true,
});
});
});

const reservedNames = [TEMPORAL_RESERVED_PREFIX, STACK_TRACE_QUERY_NAME, ENHANCED_STACK_TRACE_QUERY_NAME];

test('Cannot register activities using reserved prefixes', async (t) => {
Expand Down
17 changes: 14 additions & 3 deletions packages/worker/src/activity.ts
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,9 @@ export class Activity {
(error instanceof CancelledFailure || isAbortError(error)) &&
this.context.cancellationSignal.aborted
) {
if (this.context.cancellationDetails?.paused) {
if (this.context.cancellationDetails?.reset) {
this.workerLogger.debug('Activity reset', { durationMs });
} else if (this.context.cancellationDetails?.paused) {
this.workerLogger.debug('Activity paused', { durationMs });
} else {
this.workerLogger.debug('Activity completed as cancelled', { durationMs });
Expand Down Expand Up @@ -186,8 +188,17 @@ export class Activity {
} else if (this.cancelReason) {
// Either a CancelledFailure that we threw or AbortError from AbortController
if (err instanceof CancelledFailure) {
// If cancel due to activity pause, emit an application failure for the pause.
if (this.context.cancellationDetails?.paused) {
// If cancel due to activity pause or reset, emit an application failure.
if (this.context.cancellationDetails?.reset) {
return {
failed: {
failure: await encodeErrorToFailure(
this.dataConverter,
new ApplicationFailure('Activity reset', 'ActivityReset')
),
},
};
} else if (this.context.cancellationDetails?.paused) {
return {
failed: {
failure: await encodeErrorToFailure(
Expand Down
2 changes: 1 addition & 1 deletion packages/worker/src/worker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -995,7 +995,7 @@ export class Worker {
details,
onError() {
// activity must be defined
// empty cancellation details, not corresponding detail for heartbeat detail conversion failure
// empty cancellation details, no corresponding detail for heartbeat detail conversion failure
activity?.cancel(
'HEARTBEAT_DETAILS_CONVERSION_FAILED',
ActivityCancellationDetails.fromProto(undefined)
Expand Down