Implement worker auto-scaler

Author: AJenbo

conf/config.neon

@@ -107,7 +107,7 @@ parameters:
 	parallel:
 		jobSize: 20
 		processTimeout: 600.0
-		maximumNumberOfProcesses: 8
+		maximumNumberOfProcesses: auto
 		minimumNumberOfJobsPerProcess: 2
 		buffer: 134217728 # 128 MB
 		loadLimit: 1.0

conf/parametersSchema.neon

@@ -109,7 +109,7 @@ parametersSchema:
 	parallel: structure([
 		jobSize: int(),
 		processTimeout: float(),
-		maximumNumberOfProcesses: int(),
+		maximumNumberOfProcesses: anyOf(int(), 'auto'),
 		minimumNumberOfJobsPerProcess: int(),
 		buffer: int(),
 		loadLimit: schema(float(), nullable())

src/Parallel/Scheduler.php

@@ -19,19 +19,21 @@
 final class Scheduler implements DiagnoseExtension
 {
 
-	/** @var array{int, int, int, int}|null */
+	public const AUTO = 'auto';
+
+	/** @var array{int, int, int, int, string}|null */
 	private ?array $storedData = null;
 
 	/**
 	 * @param positive-int $jobSize
-	 * @param positive-int $maximumNumberOfProcesses
+	 * @param positive-int|self::AUTO $maximumNumberOfProcesses
 	 * @param positive-int $minimumNumberOfJobsPerProcess
 	 */
 	public function __construct(
 		#[AutowiredParameter(ref: '%parallel.jobSize%')]
 		private int $jobSize,
 		#[AutowiredParameter(ref: '%parallel.maximumNumberOfProcesses%')]
-		private int $maximumNumberOfProcesses,
+		private int|string $maximumNumberOfProcesses,
 		#[AutowiredParameter(ref: '%parallel.minimumNumberOfJobsPerProcess%')]
 		private int $minimumNumberOfJobsPerProcess,
 	)
@@ -78,25 +80,46 @@ public function scheduleWork(
 			$cpuCores,
 		);
 
-		$usedNumberOfProcesses = min($numberOfProcesses, $this->maximumNumberOfProcesses);
-		$this->storedData = [$cpuCores, count($files), count($jobs), $usedNumberOfProcesses];
+		[$maximumNumberOfProcesses, $decision] = $this->resolveMaximumNumberOfProcesses($cpuCores);
+		$usedNumberOfProcesses = min($numberOfProcesses, $maximumNumberOfProcesses);
+		$this->storedData = [$cpuCores, count($files), count($jobs), $usedNumberOfProcesses, $decision];
 
 		return new Schedule($usedNumberOfProcesses, $jobs);
 	}
 
+	/**
+	 * How many workers may run at once, and a human-readable account of why - which
+	 * `diagnose` prints, because a user who thinks the number is wrong needs to see
+	 * which input produced it.
+	 *
+	 * @return array{positive-int, string}
+	 */
+	private function resolveMaximumNumberOfProcesses(int $cpuCores): array
+	{
+		if ($this->maximumNumberOfProcesses !== self::AUTO) {
+			return [$this->maximumNumberOfProcesses, 'configured'];
+		}
+
+		return [
+			max(1, $cpuCores),
+			sprintf('auto, limited by %d usable CPU cores', $cpuCores),
+		];
+	}
+
 	public function print(Output $output): void
 	{
 		if ($this->storedData === null) {
 			return;
 		}
 
-		[$cpuCores, $filesCount, $jobsCount, $usedNumberOfProcesses] = $this->storedData;
+		[$cpuCores, $filesCount, $jobsCount, $usedNumberOfProcesses, $decision] = $this->storedData;
 
 		$output->writeLineFormatted('<info>Parallel processing scheduler:</info>');
 		$output->writeLineFormatted(sprintf('# of detected CPU cores:   %d', $cpuCores));
 		$output->writeLineFormatted(sprintf('# of analysed files:       %d', $filesCount));
 		$output->writeLineFormatted(sprintf('# of jobs:                 %d', $jobsCount));
 		$output->writeLineFormatted(sprintf('# of spawned processes:    %d', $usedNumberOfProcesses));
+		$output->writeLineFormatted(sprintf('Process limit:             %s', $decision));
 		$output->writeLineFormatted('');
 	}
 

tests/PHPStan/Parallel/SchedulerTest.php

@@ -165,4 +165,32 @@ public function testEveryFileIsScheduledExactlyOnce(): void
 		}
 	}
 
+	public function testAutoUsesAllUsableCores(): void
+	{
+		// 12 usable cores, plenty of jobs - auto follows the cores, not the old
+		// fixed default of 8
+		$scheduler = new Scheduler(1, Scheduler::AUTO, 1);
+		$schedule = $scheduler->scheduleWork(12, array_fill(0, 200, 'file.php'), static fn (string $file): int => 0);
+
+		$this->assertSame(12, $schedule->getNumberOfProcesses());
+	}
+
+	public function testAutoIsStillCappedByTheJobCount(): void
+	{
+		// 40 files -> 2 jobs at size 20, at least 2 jobs per process -> a single
+		// worker no matter how many cores the machine has
+		$scheduler = new Scheduler(20, Scheduler::AUTO, 2);
+		$schedule = $scheduler->scheduleWork(32, array_fill(0, 40, 'file.php'), static fn (string $file): int => 0);
+
+		$this->assertSame(1, $schedule->getNumberOfProcesses());
+	}
+
+	public function testAnExplicitLimitStillWins(): void
+	{
+		$scheduler = new Scheduler(1, 20, 1);
+		$schedule = $scheduler->scheduleWork(32, array_fill(0, 200, 'file.php'), static fn (string $file): int => 0);
+
+		$this->assertSame(20, $schedule->getNumberOfProcesses());
+	}
+
 }