diff --git a/src/Codeception/Module/AxeCeption.php b/src/Codeception/Module/AxeCeption.php index 03a1a73..7fed65e 100644 --- a/src/Codeception/Module/AxeCeption.php +++ b/src/Codeception/Module/AxeCeption.php @@ -5,6 +5,7 @@ namespace Codeception\Module; use Codeception\Module; +use Codeception\Step; use Codeception\Step\AxeStep; use Codeception\Test\Cest; use Codeception\TestInterface; @@ -50,11 +51,12 @@ public function _after(TestInterface $test): void * Preconditions: * - WebDriver module must be enabled; otherwise the test is skipped. * - Method is intended to be used inside a Cest scenario. - * + * + * @param array $baseline Axe-Core.run() Configuration options * @param array $axeRunConfiguration Axe-Core.run() Configuration options * @param array $axeConfiguration Axe-Core Configuration options */ - public function seeNoAccessibilityIssues(array $axeRunConfiguration = [], array $axeConfiguration = []): void + public function seeNoAccessibilityIssues(array $baseline = [], array $axeRunConfiguration = [], array $axeConfiguration = []): void { // Without WebDriver (e.g., PhpBrowser mode) we cannot run axe. if (!$this->hasModule('WebDriver')) { @@ -72,7 +74,9 @@ public function seeNoAccessibilityIssues(array $axeRunConfiguration = [], array assert($webDriver instanceof WebDriver, 'WebDriver module not loaded'); try { - $violations = $this->getViolations($webDriver, $axeRunConfiguration, $axeConfiguration); + $axeConfiguration = $this->addNeedsReviewFlagToBaselineRules($baseline, $axeConfiguration); + + $axeResults = $this->getAxeResult($webDriver, $axeRunConfiguration, $axeConfiguration); } catch (\Exception $e) { // Any error from JavaScript execution or return handling -> fail the test. $this->fail($e->getMessage()); @@ -80,7 +84,7 @@ public function seeNoAccessibilityIssues(array $axeRunConfiguration = [], array } // Record each violation as a separate AxeStep for reporting. - foreach ($violations as $violation) { + foreach ($axeResults['violations'] as $violation) { $this->addStep( 'seeAccessibilityTest' . ucfirst($violation['id']) . 'DoesNotFail', $violation, @@ -89,18 +93,115 @@ public function seeNoAccessibilityIssues(array $axeRunConfiguration = [], array ); } + $failed = false; + foreach ($baseline as $ruleId => $baseLineConfig) { + $rule = $this->getRuleEntryById($axeResults['needsReview'] ?? [], $ruleId); + + if ($rule === null) { + $this->addStep( + 'seeAccessibilityIssueFromBaseline', + [], + $webDriver->webDriver->getCurrentURL(), + true, + ['rule' => $ruleId] + ); + continue; + } + + if (is_int($baseLineConfig['count'] ?? null)) { + $numberOfErrors = array_sum(array_values($rule['errors'])); + $this->addStep( + 'seeNumberOfFoundNodesEqualsNumberFromBaseline', + [], + $webDriver->webDriver->getCurrentURL(), + $numberOfErrors !== (int)$baseLineConfig['count'], + [['rule' => $ruleId, 'expectedBaseLineCount' => $baseLineConfig['count'], 'actualCount' => $numberOfErrors]], + ); + continue; + } elseif (is_array($baseLineConfig['count'] ?? null)) { + foreach ($baseLineConfig['count'] as $selector => $count) { + $this->addStep( + 'seeNumberOfFoundNodesForSelectorEqualsNumberFromBaseline', + [], + $webDriver->webDriver->getCurrentURL(), + ($rule['errors'][$selector] ?? 0) !== $count, + [[ + 'rule' => $ruleId, + 'selector' => $selector, + 'expectedBaseLineCount' => $count, + 'actualCount' => $rule['errors'][$selector], + ]] + ); + } + $this->addStep( + 'seeNumberOfFoundNodesEqualsNumberFromBaseline', + [], + $webDriver->webDriver->getCurrentURL(), + array_keys($rule['errors']) !== array_keys($baseLineConfig['count']), + [[ + 'rule' => $ruleId, + 'expextedRuleSelectors' => array_keys($baseLineConfig['count']), + 'actualRuleSelectors' => array_keys($rule['errors']), + ]] + ); + } + } + // Fail the test if any violations were found. - if (count($violations) > 0) { - $this->fail(count($violations) . ' accessibility issues found'); + if (count($axeResults['violations']) > 0) { + $this->fail(count($axeResults['violations']) . ' accessibility issues found'); + } + + if (array_find($this->currentTest?->getScenario()->getSteps(), static fn(Step $step) => str_contains(strtolower($step->getAction()), 'baseline') && $step->hasFailed()) instanceof Step) { + $this->fail($numberOfBaselineViolations . ' baseline violations not met'); + } + } + + /** + * @template T of array + * + * @param array $baseline + * @param T $axeConfiguration + * @return T + */ + private function addNeedsReviewFlagToBaselineRules(array $baseline, array $axeConfiguration): array { + foreach ($baseline as $ruleId => $baseLineConfig) { + $rule = $this->getRuleEntryById($axeConfiguration['rules'] ?? [], $ruleId); + if ($rule === null) { + $axeConfiguration['rules'][] = ['id' => $ruleId, 'reviewOnFail' => true]; + } else { + $axeConfiguration['rules'][array_search($rule, $axeConfiguration['rules'])]['reviewOnFail'] = true; + } } + return $axeConfiguration; + } + + /** + * @param array $rules + * @param string $ruleId + * @return array|null + */ + private function getRuleEntryById(array $rules, string $ruleId): ?array + { + foreach ($rules as $rule) { + if ($rule['id'] === $ruleId) { + return $rule; + } + } + return null; + } + + private function getConfigurationIndexOfRule(string $ruleId, array $axeConfiguration): int + { + return array_search($ruleId, array_column($axeConfiguration['rules'], 'id')); } /** * Adds a custom AxeStep to the current scenario so the reporter can render it later. */ - private function addStep(string $action, array $violation, string $testedPageUrl, bool $failed = false): void + private function addStep(string $action, array $violation, string $testedPageUrl, bool $failed = false, array $arguments = []): void { - $step = new AxeStep($action); + $step = new AxeStep($action, $arguments); if ($failed) { $step->setFailed(true); @@ -120,10 +221,10 @@ private function addStep(string $action, array $violation, string $testedPageUrl * @param WebDriver $webDriver Codeception WebDriver module instance * @param array $axeRunConfiguration Axe-Core.run() Configuration options * @param array $axeConfiguration Axe-Core Configuration options - * @return array> List of axe violations as arrays (JSON from the browser) + * @return array{violations: array>, needsReview: array>} List of axe violations as arrays (JSON from the browser) * @throws \Exception when axe could not be executed or returned invalid data */ - private function getViolations(WebDriver $webDriver, array $axeRunConfiguration, array $axeConfiguration): array + private function getAxeResult(WebDriver $webDriver, array $axeRunConfiguration, array $axeConfiguration): array { // Configurable source for axe-core; double quotes are stripped to simplify injection into the script tag. // You can override this via module config: axeJavascript: "https://.../axe.min.js" @@ -144,14 +245,17 @@ private function getViolations(WebDriver $webDriver, array $axeRunConfiguration, $runConfigJavaScriptObject = json_encode($axeRunConfiguration, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE); } - $violations = $webDriver->executeJS( - /** @lang JavaScript */ <<