Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

FIX: Correctly log errors during job initialisation (fixes #316) #374

Merged
merged 1 commit into from
Aug 22, 2022
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 47 additions & 27 deletions src/Services/QueuedJobService.php
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
use Symbiote\QueuedJobs\Jobs\RunBuildTaskJob;
use Symbiote\QueuedJobs\QJUtils;
use Symbiote\QueuedJobs\Tasks\Engines\TaskRunnerEngine;
use Throwable;

/**
* A service that can be used for starting, stopping and listing queued jobs.
Expand Down Expand Up @@ -753,7 +754,7 @@ protected function grabMutex(QueuedJobDescriptor $jobDescriptor)
});

return true;
} catch (\Throwable $e) {
} catch (Throwable $e) {
// note that error here may not be an issue as failing to acquire a job lock is a valid state
// which happens when other process claimed the job lock first
$this->getLogger()->debug(
Expand Down Expand Up @@ -829,7 +830,14 @@ public function runJob($jobId)
$job = null;

try {
$job = $this->initialiseJob($jobDescriptor);
try {
$job = $this->initialiseJob($jobDescriptor);
} catch (Throwable $e) {
$this->handleJobInitialisationException($jobDescriptor, $e);
$broken = true;
$this->finaliseLogging($logger);
return;
}

// get the job ready to begin.
if (!$jobDescriptor->JobStarted) {
Expand Down Expand Up @@ -911,7 +919,7 @@ public function runJob($jobId)

try {
$job->process();
} catch (\Throwable $e) {
} catch (Throwable $e) {
$logger->error($e->getMessage(), ['exception' => $e]);
$this->markJobAsBroken($jobDescriptor);
$this->extend('updateJobDescriptorAndJobOnException', $jobDescriptor, $job, $e);
Expand Down Expand Up @@ -991,36 +999,40 @@ public function runJob($jobId)

$this->extend('updateJobDescriptorAndJobOnCompletion', $jobDescriptor, $job);
}
} catch (\Throwable $e) {
// PHP 7 Error handling)
} catch (Throwable $e) {
$this->handleBrokenJobException($jobDescriptor, $job, $e);
$broken = true;
}

// Write any remaining batched messages at the end.
if ($logger instanceof Logger) {
foreach ($logger->getHandlers() as $handler) {
if ($handler instanceof BufferHandler) {
$handler->flush();
}
}
}

// If using a global singleton logger here,
// any messages added after this point will be auto-flushed on PHP shutdown through the handler.
// This causes a database write, and assumes the database and table will be available at this point.
if ($logger instanceof Logger) {
$logger->setHandlers(array_filter($logger->getHandlers() ?? [], function ($handler) {
return !($handler instanceof BufferHandler);
}));
}
$this->finaliseLogging($logger);
});

$this->unsetRunAsUser($runAsUser, $originalUser);

return !$broken;
}

protected function finaliseLogging(LoggerInterface $logger)
{
if (!$logger instanceof Logger) {
return;
}

// Write any remaining batched messages at the end.
foreach ($logger->getHandlers() as $handler) {
if ($handler instanceof BufferHandler) {
$handler->flush();
}
}

// If using a global singleton logger here,
// any messages added after this point will be auto-flushed on PHP shutdown through the handler.
// This causes a database write, and assumes the database and table will be available at this point.
$logger->setHandlers(array_filter($logger->getHandlers() ?? [], function ($handler) {
return !($handler instanceof BufferHandler);
}));
}

/**
* Provides a wrapper when executing arbitrary code contained in job implementation
* this ensures that job specific code doesn't alter the configuration of the queue runner execution
Expand All @@ -1044,19 +1056,27 @@ protected function withNestedState(callable $callback)
}
}

protected function handleJobInitialisationException(QueuedJobDescriptor $jobDescriptor, Throwable $e)
GuySartorelli marked this conversation as resolved.
Show resolved Hide resolved
{
$this->getLogger()->info(
$e->getMessage(),
['exception' => $e]
);
$this->markJobAsBroken($jobDescriptor);
$this->extend('updateJobDescriptorOnInitialisationException', $jobDescriptor, $e);
$jobDescriptor->write();
}

/**
* @param QueuedJobDescriptor $jobDescriptor
* @param QueuedJob $job
* @param Exception|\Throwable $e
* @param Exception|Throwable $e
*/
protected function handleBrokenJobException(QueuedJobDescriptor $jobDescriptor, QueuedJob $job, $e)
{
// okay, we'll just catch this exception for now
$this->getLogger()->info(
$e->getMessage(),
[
'exception' => $e,
]
['exception' => $e]
);
$this->markJobAsBroken($jobDescriptor);
$this->extend('updateJobDescriptorAndJobOnException', $jobDescriptor, $job, $e);
Expand Down