Merge pull request #941 from opengisch/QF-4070-fix-delta-apply-stuck-…

…started

QF-4070 fix Delta Apply stuck in "Started"

docker-app/qfieldcloud/core/cron.py

@@ -7,7 +7,7 @@
 from invitations.utils import get_invitation_model
 from sentry_sdk import capture_message
 
-from ..core.models import Job, Project
+from ..core.models import ApplyJob, ApplyJobDelta, Delta, Job, Project
 from ..core.utils2 import storage
 from .invitations_utils import send_invitation
 
@@ -60,6 +60,22 @@ def do(self):
             capture_message(
                 f'Job "{job.id}" was with status "{job.status}", but worker container no longer exists. Job unexpectedly terminated.'
             )
+            if job.type == Job.Type.DELTA_APPLY:
+                ApplyJob.objects.get(id=job.id).deltas_to_apply.update(
+                    last_status=Delta.Status.ERROR,
+                    last_feedback=None,
+                    last_modified_pk=None,
+                    last_apply_attempt_at=job.started_at,
+                    last_apply_attempt_by=job.created_by,
+                )
+
+                ApplyJobDelta.objects.filter(
+                    apply_job_id=job.id,
+                ).update(
+                    status=Delta.Status.ERROR,
+                    feedback=None,
+                    modified_pk=None,
+                )
 
         jobs.update(
             status=Job.Status.FAILED,

docker-app/worker_wrapper/wrapper.py

@@ -157,13 +157,14 @@ def run(self):
                 feedback["error_stack"] = ""
 
                 try:
+                    logger.info(
+                        "Set job status to `failed` due to being killed by the docker engine.",
+                    )
+
                     self.job.output = output.decode("utf-8")
                     self.job.feedback = feedback
                     self.job.status = Job.Status.FAILED
                     self.job.save(update_fields=["output", "feedback", "status"])
-                    logger.info(
-                        "Set job status to `failed` due to being killed by the docker engine.",
-                    )
                 except Exception as err:
                     logger.error(
                         "Failed to update job status, probably does not exist in the database.",
@@ -549,13 +550,21 @@ def after_docker_run(self) -> None:
     def after_docker_exception(self) -> None:
         Delta.objects.filter(
             id__in=self.delta_ids,
-        ).update(last_status=Delta.Status.ERROR)
+        ).update(
+            last_status=Delta.Status.ERROR,
+            last_feedback=None,
+            last_modified_pk=None,
+            last_apply_attempt_at=self.job.started_at,
+            last_apply_attempt_by=self.job.created_by,
+        )
 
         ApplyJobDelta.objects.filter(
             apply_job_id=self.job_id,
             delta_id__in=self.delta_ids,
         ).update(
             status=Delta.Status.ERROR,
+            feedback=None,
+            modified_pk=None,
         )