Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
abfdb16
Keep running jobs alive across dispatcher and task-pod restarts (job …
cigamit Nov 10, 2026
e7f0dbc
Send sinceTime on pod log reconnects through a raw API request
cigamit Nov 10, 2026
4469e76
Skip only what a pod log reconnect repeats, not lines that share a ti…
cigamit Nov 10, 2026
f14462a
Treat a pod log read that breaks off in its final pass as an error
cigamit Nov 10, 2026
2c7bfe2
Leave a local unit to its node while that node restarts its dispatcher
cigamit Nov 10, 2026
1c28bb8
Keep reading a job when recording its pod name hits a database error
cigamit Nov 10, 2026
84fcc59
Try again to record that the worker started when the write fails
cigamit Nov 10, 2026
2e165ff
Release a project update's lock when its run ends without the post-ru…
cigamit Nov 10, 2026
d860e0e
Remove a detached job's private data dir unless its work is local
cigamit Nov 10, 2026
eb10d99
Skip the post-run hook when a resume fails or cancels a job it never …
cigamit Nov 10, 2026
56bf1cf
Serialize lease claims and retirements on the lease row
cigamit Nov 10, 2026
1f90860
Re-queue a job only for the task that owns it, in one transaction
cigamit Nov 10, 2026
efe6af1
Keep the monitor lease renewed while a resume gets ready to read
cigamit Nov 10, 2026
9e03603
Do not re-queue a job on shutdown when its work could not be cancelled
cigamit Nov 10, 2026
91f24e2
Sweep a node's own expired leases first
cigamit Nov 10, 2026
6d67027
Do not count waiting for a pending pod against the deferral budget
cigamit Nov 10, 2026
77fee30
Track a resumed job's saved events as ranges read in one query
cigamit Nov 10, 2026
608429c
Let the post-run hooks of a resumed job find its artifacts directly
cigamit Nov 10, 2026
43aeed1
Pause before reopening a followed pod log that the kubelet ended
cigamit Nov 10, 2026
cf1b63e
Mark a re-queued attempt before its stream drains, so it sends no EOF
cigamit Nov 10, 2026
8396751
Lock the lease before the job when re-queueing it
cigamit Nov 10, 2026
cb137c6
Take receptor's "Adopt Pending" answer for an adopted unit
cigamit Nov 10, 2026
e6bd265
Wait for an execution node that is unavailable but not yet lost
cigamit Nov 10, 2026
e60b137
Count a resume that fails to get ready, and fail a job without that s…
cigamit Nov 10, 2026
5e9e8d6
Read a pod log line that spans many chunks in linear time
cigamit Nov 10, 2026
9a50990
Leave a lost control or hybrid node's leased jobs to the sweep when r…
cigamit Nov 11, 2026
a5fc7b3
Fail a job whose inline project sync was taken over, not run it
cigamit Nov 11, 2026
1624944
Remove the private data dir of a project update that lost its lease
cigamit Nov 11, 2026
b6df41d
Have the job monitor sweep see each job as its own type
cigamit Nov 11, 2026
a3aa821
Leave signal handling as the caller set it after waiting for the stre…
cigamit Nov 11, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions ascender/main/dispatch/pool.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
from jinja2 import Template
import psutil

from ascender.main.models import UnifiedJob
from ascender.main.models import UnifiedJob, JobMonitorLease
from ascender.main.dispatch import reaper
from ascender.main.utils.common import convert_mem_str_to_bytes, get_mem_effective_capacity, log_excess_runtime

Expand Down Expand Up @@ -427,7 +427,14 @@ def cleanup(self):
if w.current_task != 'QUIT':
try:
for j in UnifiedJob.objects.filter(celery_task_id=w.current_task['uuid']):
reaper.reap_job(j, 'failed')
# A job whose output was being read holds a monitor lease. Expiring it
# hands the job to the next sweep, which resumes reading, rather than
# failing a job whose work is still running somewhere. A lease another
# task has claimed since is that task's.
if JobMonitorLease.objects.release(j, w.current_task['uuid']):
logger.warning(f'{j.log_format}: its monitor died with worker pid:{w.pid}; its lease is released for the next sweep')
elif not JobMonitorLease.objects.filter(unified_job_id=j.pk).exists():
reaper.reap_job(j, 'failed')
except Exception:
logger.exception('failed to reap job UUID {}'.format(w.current_task['uuid']))
else:
Expand Down
27 changes: 24 additions & 3 deletions ascender/main/dispatch/reaper.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,17 @@
def startup_reaping():
"""
If this particular instance is starting, then we know that any running jobs are invalid
so we will reap those jobs as a special action here
so we will reap those jobs as a special action here.

A running job that holds a monitor lease is not one of them: its work was handed to
receptor, and may well still be running on an execution node or in a pod. The job
monitor sweep resumes or re-queues it, see ascender.main.tasks.monitor.
"""
jobs = UnifiedJob.objects.filter(status='running', controller_node=Instance.objects.my_hostname())
hostname = Instance.objects.my_hostname()
leased = list(UnifiedJob.objects.filter(status='running', controller_node=hostname, monitor_lease__isnull=False).values_list('id', flat=True))
if leased:
logger.warning(f'Unified jobs {leased} were running under a monitor lease at dispatch startup; left to the job monitor sweep')
jobs = UnifiedJob.objects.filter(status='running', controller_node=hostname, monitor_lease__isnull=True)
job_ids = []
for j in jobs:
job_ids.append(j.id)
Expand Down Expand Up @@ -73,13 +81,26 @@ def reap_waiting(instance=None, status='failed', job_explanation=None, grace_per
def reap(instance=None, status='failed', job_explanation=None, excluded_uuids=None, ref_time=None):
"""
Reap all jobs in running for this instance.

A job that holds a monitor lease is left to the job monitor sweep when this instance is a
control or hybrid node. Such a node executes only work it controls, so its leased jobs are
ones it was reading, whose lease lapses once nobody renews it, and ones a peer claimed after
this node was lost, whose unit only this node's receptor had: that peer re-queues or fails
them within moments. A leased job whose work is on an execution or hop node is reaped as
before: its controller is alive and still reading it, and would wait for a lost node for as
long as receptor does, which is indefinitely.
"""
if instance is None:
hostname = Instance.objects.my_hostname()
node_type = Instance.objects.filter(hostname=hostname).values_list('node_type', flat=True).first()
else:
hostname = instance.hostname
hostname, node_type = instance.hostname, instance.node_type
workflow_ctype_id = ContentType.objects.get_for_model(WorkflowJob).id
base_Q = Q(status='running') & (Q(execution_node=hostname) | Q(controller_node=hostname)) & ~Q(polymorphic_ctype_id=workflow_ctype_id)
if node_type in (Instance.Types.CONTROL, Instance.Types.HYBRID):
base_Q &= Q(monitor_lease__isnull=True)
else:
base_Q &= Q(monitor_lease__isnull=True) | ~Q(controller_node=hostname)
if ref_time:
jobs = UnifiedJob.objects.filter(base_Q & Q(started__lte=ref_time))
else:
Expand Down
27 changes: 25 additions & 2 deletions ascender/main/dispatch/worker/callback.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from ascender.settings.typed import settings
from django.utils.functional import cached_property
from django.utils.timezone import now as tz_now
from django.db import transaction, connection as django_connection
from django.db import transaction, IntegrityError, connection as django_connection
from django_guid import set_guid

import psutil
Expand All @@ -26,6 +26,15 @@
logger = logging.getLogger('ascender.main.commands.run_callback_receiver')


def _is_unique_constraint_violation(exc):
"""True for a unique-key IntegrityError: the event is already stored, so it is safe to drop."""
cause = getattr(exc, '__cause__', None)
# psycopg 3 names it sqlstate; psycopg 2 called it pgcode
if '23505' in (getattr(cause, 'sqlstate', None), getattr(cause, 'pgcode', None)):
return True
return 'duplicate key value violates unique constraint' in str(exc)


def job_stats_wrapup(job_identifier, event=None):
"""Fill in the unified job host_status_counts, fire off notifications if needed"""
try:
Expand All @@ -38,14 +47,17 @@ def job_stats_wrapup(job_identifier, event=None):
# Update host_status_counts while holding the row lock
with transaction.atomic():
uj = UnifiedJob.objects.select_for_update().get(pk=job_identifier)
# Already set means the stats event was processed before, by a controller that was
# lost after saving it; its replay must not send a second notification below.
already_processed = uj.host_status_counts is not None
uj.host_status_counts = host_status_counts
uj.save(update_fields=['host_status_counts'])

uj.log_lifecycle("stats_wrapup_finished")

# If the status was a finished state before this update was made, send notifications
# If not, we will send notifications when the status changes
if uj.status not in ACTIVE_STATES:
if uj.status not in ACTIVE_STATES and not already_processed:
uj.send_notification_templates('succeeded' if uj.status == 'successful' else 'failed')

except Exception:
Expand Down Expand Up @@ -163,6 +175,7 @@ def flush(self, force=False):
metrics_events_batch_save_errors = 0
metrics_events_broadcast = 0
metrics_events_missing_created = 0
metrics_duplicates_dropped = 0
metrics_total_job_event_processing_seconds = datetime.timedelta(seconds=0)
for cls, events in self.buff.items():
if not events:
Expand Down Expand Up @@ -198,6 +211,13 @@ def flush(self, force=False):
events.remove(e)
saved_events.append(e) # Importantly, remove successfully saved events from the buffer
except Exception as exc_indv:
if isinstance(exc_indv, IntegrityError) and _is_unique_constraint_violation(exc_indv):
# The same event arrived twice, from a stream that was replayed when
# the job's monitoring was resumed. It is stored; drop this copy.
events.remove(e)
metrics_duplicates_dropped += 1
continue
Comment thread
cigamit marked this conversation as resolved.
Resolved

retry_count = getattr(e, '_retry_count', 0) + 1
e._retry_count = retry_count

Expand All @@ -221,6 +241,9 @@ def flush(self, force=False):
if getattr(e, '_notification_trigger_event', False):
job_stats_wrapup(getattr(e, e.JOB_REFERENCE), event=e)
self.last_flush = time.time()
if metrics_duplicates_dropped:
logger.info(f'Dropped {metrics_duplicates_dropped} replayed event(s) that were already saved')
self.subsystem_metrics.inc('callback_receiver_events_in_memory', -metrics_duplicates_dropped)
# only update metrics if we saved events
if (metrics_bulk_events_saved + metrics_singular_events_saved) > 0:
self.subsystem_metrics.inc('callback_receiver_batch_events_errors', metrics_events_batch_save_errors)
Expand Down
95 changes: 95 additions & 0 deletions ascender/main/migrations/0232_event_counter_unique.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
# Generated by Django 6.1.1 on 2026-11-10 09:23, then edited: the constraints were split from
# the job monitor lease model (0233) and preceded by a de-duplication.

from django.db import migrations, models

# (table, column of the job reference)
EVENT_TABLES = (
('main_jobevent', 'job_id'),
('main_projectupdateevent', 'project_update_id'),
('main_adhoccommandevent', 'ad_hoc_command_id'),
('main_inventoryupdateevent', 'inventory_update_id'),
('main_systemjobevent', 'system_job_id'),
)


def dedup_events(apps, schema_editor):
"""
Remove duplicate (job, job_created, counter) rows before the unique constraints are added.

ansible-runner numbers the events of a run one by one, so a database should hold no
duplicates, and this deletes nothing; it is here so that the constraints apply even to one
that somehow does, keeping the row with the lowest id, the earliest insert. Each table is
first checked with a scan of its existing (job, job_created, counter) index, which is cheap
next to the self-join that removes duplicates when there are any.

NOTE: on a large deployment the event tables are partitioned by job_created with many
partitions, and ADD CONSTRAINT builds a unique index on every one of them under a lock.
Build the indexes CONCURRENTLY per partition beforehand if that lock is a concern.
"""
with schema_editor.connection.cursor() as cursor:
for table, job_column in EVENT_TABLES:
cursor.execute(f'SELECT 1 FROM {table} GROUP BY {job_column}, job_created, counter HAVING count(*) > 1 LIMIT 1')
if cursor.fetchone() is None:
continue
cursor.execute(
f"""
DELETE FROM {table} a
USING {table} b
WHERE a.id > b.id
AND a.{job_column} = b.{job_column}
AND a.job_created = b.job_created
AND a.counter = b.counter
"""
)


class Migration(migrations.Migration):
dependencies = [
('main', '0231_workload_identity_key'),
]

operations = [
migrations.RunPython(dedup_events, migrations.RunPython.noop),
# Each unique index replaces the plain index on the same columns, in the same order
migrations.AddConstraint(
model_name='adhoccommandevent',
constraint=models.UniqueConstraint(fields=('ad_hoc_command', 'job_created', 'counter'), name='main_adhoccommandevent_counter_unique'),
),
migrations.RemoveIndex(
model_name='adhoccommandevent',
name='main_adhocc_ad_hoc__a57777_idx',
),
migrations.AddConstraint(
model_name='inventoryupdateevent',
constraint=models.UniqueConstraint(fields=('inventory_update', 'job_created', 'counter'), name='main_inventoryupdateevent_counter_unique'),
),
migrations.RemoveIndex(
model_name='inventoryupdateevent',
name='main_invent_invento_364dcb_idx',
),
migrations.AddConstraint(
model_name='jobevent',
constraint=models.UniqueConstraint(fields=('job', 'job_created', 'counter'), name='main_jobevent_counter_unique'),
),
migrations.RemoveIndex(
model_name='jobevent',
name='main_jobeve_job_id_51c382_idx',
),
migrations.AddConstraint(
model_name='projectupdateevent',
constraint=models.UniqueConstraint(fields=('project_update', 'job_created', 'counter'), name='main_projectupdateevent_counter_unique'),
),
migrations.RemoveIndex(
model_name='projectupdateevent',
name='main_projec_project_69559a_idx',
),
migrations.AddConstraint(
model_name='systemjobevent',
constraint=models.UniqueConstraint(fields=('system_job', 'job_created', 'counter'), name='main_systemjobevent_counter_unique'),
),
migrations.RemoveIndex(
model_name='systemjobevent',
name='main_system_system__73537a_idx',
),
]
37 changes: 37 additions & 0 deletions ascender/main/migrations/0233_job_monitor_lease.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
# Generated by Django 6.1.1 on 2026-11-10 09:23

import django.db.models.deletion
from django.db import migrations, models


class Migration(migrations.Migration):
dependencies = [
('main', '0232_event_counter_unique'),
]

operations = [
migrations.CreateModel(
name='JobMonitorLease',
fields=[
(
'unified_job',
models.OneToOneField(
editable=False,
on_delete=django.db.models.deletion.CASCADE,
primary_key=True,
related_name='monitor_lease',
serialize=False,
to='main.unifiedjob',
),
),
('expires', models.DateTimeField(db_index=True, editable=False)),
('pod_name', models.CharField(blank=True, default='', editable=False, max_length=253)),
('pod_namespace', models.CharField(blank=True, default='', editable=False, max_length=253)),
('worker_started', models.BooleanField(default=False, editable=False)),
('resumed', models.PositiveIntegerField(default=0, editable=False)),
('deferrals', models.PositiveIntegerField(default=0, editable=False)),
('created', models.DateTimeField(auto_now_add=True)),
('modified', models.DateTimeField(auto_now=True)),
],
),
]
1 change: 1 addition & 0 deletions ascender/main/models/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@
from ascender.api.versioning import reverse
from ascender.main.models.oauth import OAuth2AccessToken, OAuth2Application, OAuth2RefreshToken, OAuth2IDToken # noqa
from ascender.main.models.workload_identity import WorkloadIdentityKey # noqa
from ascender.main.models.monitor import JobMonitorLease, LeaseLost # noqa
from oauth2_provider.models import Grant # noqa -- needed django-oauth-toolkit model migrations

# Add custom methods to User model for permissions checks.
Expand Down
Loading
Loading