Skip to content

Implement baseline feature #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
nmaier95 wants to merge 1 commit into
base: main
Choose a base branch
from
+132 −16
Draft

Implement baseline feature #8

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
148 changes: 132 additions & 16 deletions src/Codeception/Module/AxeCeption.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
namespace Codeception\Module;

use Codeception\Module;
use Codeception\Step;
use Codeception\Step\AxeStep;
use Codeception\Test\Cest;
use Codeception\TestInterface;
Expand Down Expand Up @@ -50,11 +51,12 @@ public function _after(TestInterface $test): void
* Preconditions:
* - WebDriver module must be enabled; otherwise the test is skipped.
* - Method is intended to be used inside a Cest scenario.
*
*
* @param array<string, array{count: int, selector: string}> $baseline Axe-Core.run() Configuration options
* @param array<mixed> $axeRunConfiguration Axe-Core.run() Configuration options
* @param array<mixed> $axeConfiguration Axe-Core Configuration options
*/
public function seeNoAccessibilityIssues(array $axeRunConfiguration = [], array $axeConfiguration = []): void
public function seeNoAccessibilityIssues(array $baseline = [], array $axeRunConfiguration = [], array $axeConfiguration = []): void
{
// Without WebDriver (e.g., PhpBrowser mode) we cannot run axe.
if (!$this->hasModule('WebDriver')) {
Expand All @@ -72,15 +74,17 @@ public function seeNoAccessibilityIssues(array $axeRunConfiguration = [], array
assert($webDriver instanceof WebDriver, 'WebDriver module not loaded');

try {
$violations = $this->getViolations($webDriver, $axeRunConfiguration, $axeConfiguration);
$axeConfiguration = $this->addNeedsReviewFlagToBaselineRules($baseline, $axeConfiguration);

$axeResults = $this->getAxeResult($webDriver, $axeRunConfiguration, $axeConfiguration);
} catch (\Exception $e) {
// Any error from JavaScript execution or return handling -> fail the test.
$this->fail($e->getMessage());
return;
}

// Record each violation as a separate AxeStep for reporting.
foreach ($violations as $violation) {
foreach ($axeResults['violations'] as $violation) {
$this->addStep(
'seeAccessibilityTest' . ucfirst($violation['id']) . 'DoesNotFail',
$violation,
Expand All @@ -89,18 +93,115 @@ public function seeNoAccessibilityIssues(array $axeRunConfiguration = [], array
);
}

$failed = false;
foreach ($baseline as $ruleId => $baseLineConfig) {
$rule = $this->getRuleEntryById($axeResults['needsReview'] ?? [], $ruleId);

if ($rule === null) {
$this->addStep(
'seeAccessibilityIssueFromBaseline',
[],
$webDriver->webDriver->getCurrentURL(),
true,
['rule' => $ruleId]
);
continue;
}

if (is_int($baseLineConfig['count'] ?? null)) {
$numberOfErrors = array_sum(array_values($rule['errors']));
$this->addStep(
'seeNumberOfFoundNodesEqualsNumberFromBaseline',
[],
$webDriver->webDriver->getCurrentURL(),
$numberOfErrors !== (int)$baseLineConfig['count'],
[['rule' => $ruleId, 'expectedBaseLineCount' => $baseLineConfig['count'], 'actualCount' => $numberOfErrors]],
);
continue;
} elseif (is_array($baseLineConfig['count'] ?? null)) {
foreach ($baseLineConfig['count'] as $selector => $count) {
$this->addStep(
'seeNumberOfFoundNodesForSelectorEqualsNumberFromBaseline',
[],
$webDriver->webDriver->getCurrentURL(),
($rule['errors'][$selector] ?? 0) !== $count,
[[
'rule' => $ruleId,
'selector' => $selector,
'expectedBaseLineCount' => $count,
'actualCount' => $rule['errors'][$selector],
]]
);
}
$this->addStep(
'seeNumberOfFoundNodesEqualsNumberFromBaseline',
[],
$webDriver->webDriver->getCurrentURL(),
array_keys($rule['errors']) !== array_keys($baseLineConfig['count']),
[[
'rule' => $ruleId,
'expextedRuleSelectors' => array_keys($baseLineConfig['count']),
'actualRuleSelectors' => array_keys($rule['errors']),
]]
);
}
}

// Fail the test if any violations were found.
if (count($violations) > 0) {
$this->fail(count($violations) . ' accessibility issues found');
if (count($axeResults['violations']) > 0) {
$this->fail(count($axeResults['violations']) . ' accessibility issues found');
}

if (array_find($this->currentTest?->getScenario()->getSteps(), static fn(Step $step) => str_contains(strtolower($step->getAction()), 'baseline') && $step->hasFailed()) instanceof Step) {
$this->fail($numberOfBaselineViolations . ' baseline violations not met');
}
}

/**
* @template T of array
*
* @param array<mixed> $baseline
* @param T $axeConfiguration
* @return T
*/
private function addNeedsReviewFlagToBaselineRules(array $baseline, array $axeConfiguration): array {
foreach ($baseline as $ruleId => $baseLineConfig) {
$rule = $this->getRuleEntryById($axeConfiguration['rules'] ?? [], $ruleId);
if ($rule === null) {
$axeConfiguration['rules'][] = ['id' => $ruleId, 'reviewOnFail' => true];
} else {
$axeConfiguration['rules'][array_search($rule, $axeConfiguration['rules'])]['reviewOnFail'] = true;
}
}
return $axeConfiguration;
}

/**
* @param array<mixed> $rules
* @param string $ruleId
* @return array|null
*/
private function getRuleEntryById(array $rules, string $ruleId): ?array
{
foreach ($rules as $rule) {
if ($rule['id'] === $ruleId) {
return $rule;
}
}
return null;
}

private function getConfigurationIndexOfRule(string $ruleId, array $axeConfiguration): int
{
return array_search($ruleId, array_column($axeConfiguration['rules'], 'id'));
}

/**
* Adds a custom AxeStep to the current scenario so the reporter can render it later.
*/
private function addStep(string $action, array $violation, string $testedPageUrl, bool $failed = false): void
private function addStep(string $action, array $violation, string $testedPageUrl, bool $failed = false, array $arguments = []): void
{
$step = new AxeStep($action);
$step = new AxeStep($action, $arguments);

if ($failed) {
$step->setFailed(true);
Expand All @@ -120,10 +221,10 @@ private function addStep(string $action, array $violation, string $testedPageUrl
* @param WebDriver $webDriver Codeception WebDriver module instance
* @param array<mixed> $axeRunConfiguration Axe-Core.run() Configuration options
* @param array<mixed> $axeConfiguration Axe-Core Configuration options
* @return array<int, array<string, mixed>> List of axe violations as arrays (JSON from the browser)
* @return array{violations: array<int, array<string, mixed>>, needsReview: array<int, array<string, mixed>>} List of axe violations as arrays (JSON from the browser)
* @throws \Exception when axe could not be executed or returned invalid data
*/
private function getViolations(WebDriver $webDriver, array $axeRunConfiguration, array $axeConfiguration): array
private function getAxeResult(WebDriver $webDriver, array $axeRunConfiguration, array $axeConfiguration): array
{
// Configurable source for axe-core; double quotes are stripped to simplify injection into the script tag.
// You can override this via module config: axeJavascript: "https://.../axe.min.js"
Expand All @@ -144,14 +245,17 @@ private function getViolations(WebDriver $webDriver, array $axeRunConfiguration,
$runConfigJavaScriptObject = json_encode($axeRunConfiguration, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
}

$violations = $webDriver->executeJS(
/** @lang JavaScript */ <<<SCRIPT
$violations = $webDriver->executeJS(
/** @lang JavaScript */ <<<SCRIPT
return new Promise((resolve, reject) => {
const script = document.createElement("script");
script.onload = () => {
$configureJs
axe.run($runConfigJavaScriptObject)
.then(results => {
//
// resolve(results);
// return;
if (results.violations.length) {
results.violations.forEach((violation) => {
violation.errors = {};
Expand All @@ -161,10 +265,18 @@ private function getViolations(WebDriver $webDriver, array $axeRunConfiguration,
});
});
});
resolve(results.violations);
} else {
resolve([]);
}
if (results.incomplete.length) {
results.incomplete.forEach((violation) => {
violation.errors = {};
violation.nodes.forEach((node) => {
node.target.forEach((target) => {
violation.errors[target] = document.querySelectorAll(target).length;
});
});
});
}
resolve({violations: results.violations ?? [], needsReview: results.incomplete ?? []});
})
.catch(reject);
};
Expand All @@ -176,7 +288,11 @@ private function getViolations(WebDriver $webDriver, array $axeRunConfiguration,
);

// Validate the structure before returning to PHP.
if (!is_array($violations)) {
if (
!is_array($violations)
|| !array_key_exists('violations', $violations)
|| !array_key_exists('needsReview', $violations)
) {
throw new \Exception('Axe returned invalid data: ' . $violations);
}

Expand Down