diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 110b43b..5eadfe8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,34 +1,12 @@ name: CI on: - push: - branches: - - main - pull_request: + push env: PLAYWRIGHT_BROWSERS_PATH: 0 jobs: - infra: - name: Lint - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install -e . - python -m playwright install-deps - python -m playwright install - - name: Lint - uses: pre-commit/action@v2.0.3 - test: name: Test timeout-minutes: 30 @@ -36,7 +14,7 @@ jobs: fail-fast: true matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python-version: [3.7, 3.8, 3.9] + python-version: [3.8, 3.9, 3.10.4] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml new file mode 100644 index 0000000..bf08508 --- /dev/null +++ b/.github/workflows/publish-to-pypi.yml @@ -0,0 +1,40 @@ +name: Publish Python 🐍 distributions 📦 to PyPI and TestPyPI + +on: + push + +jobs: + build-n-publish: + name: Build and publish Python 🐍 distributions 📦 to PyPI and TestPyPI + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: Set up Python 3.10.4 + uses: actions/setup-python@v1 + with: + python-version: 3.10.4 + - name: Install pypa/build + run: >- + python -m + pip install + build + --user + - name: Build a binary wheel and a source tarball + run: >- + python -m + build + --sdist + --wheel + --outdir dist/ +# - name: Publish distribution 📦 to Test PyPI +# uses: pypa/gh-action-pypi-publish@release/v1 +# with: +# user: __token__ +# password: ${{ secrets.TEST_PYPI_API_TOKEN }} +# repository_url: https://test.pypi.org/legacy/ + - name: Publish distribution 📦 to PyPI + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/README.md b/README.md index 1573ecf..ed341a1 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,20 @@ -# Pytest Plugin for Snapshot Testing with Playwright +# Pytest Plugin for Visual Testing with Playwright -This plugin enables snapshot testing in playwright like snapshotting screenshots of pages, element handles etc. +Based on [pixelmatch-py](https://github.com/whtsky/pixelmatch-py) image comparison library. + +Expands `assert_snapshot` fixture from [pytest-playwright-snapshot](https://github.com/kumaraditya303/pytest-playwright-snapshot) library + +## Main Features: +- snapshots creation on the first run +- visual review of mismatches +- failing on `--update-snapshots` to make users manually review images +- snapshot name is optional, `test_name[browser][os].png` is auto-generated by default +- updated folder structure: `snapshots/file_name/test_name/test_name[browser][os].png` ## Installation ```bash -$ pip install pytest-playwright-snapshot +$ pip install pytest-playwright-visual ``` ## Usage @@ -17,27 +26,39 @@ Example: ```python def test_myapp(page, assert_snapshot): page.goto("https://example.com") - assert_snapshot(page.screenshot(), "example.png") + assert_snapshot(page.screenshot()) ``` - -Ths first time you run pytest, you will get error like +Then, run pytest: +```bash +$ pytest +``` +The first time you run pytest, snapshots will be created, and you will get the error: ```console -Failed: Snapshot not found, use --update-snapshots to create it. +Failed: --> New snapshot(s) created. Please review images ``` -As first you need to create golden snapshots to which this plugin will compare in future. +The next run, the snapshots comparison will take place. -To create snapshots run: +To update snapshots, run: ```bash $ pytest --update-snapshots ``` -This will create snapshots for your tests, after that you can run the tests are usual and this will compare the snapshots. +After updating, tests will fail and you will need to review images. + +In case of a mismatch, `snapshot_tests_failures` folder will be created with `Actual_..`, `Expected_..` and `Diff_..` images generated. + +## Folder Structure Example -There is `threshold` kwarg only option which can be used to set the threshold for the comparison of the screenshots and by default it is `0.1` +![img_2.png](img_2.png) +## API +**assert_snapshot(page.screenshot(), threshold: float = 0.1, name='test_name[browser][os].png', fail_fast=False)** +- `threshold` - sets the threshold for the comparison of the screenshots:`0` to `1`. Default is `0.1` +- `name` - `.png` extensions only. Default is `test_name[browser][os].png` (recommended) +- `fail_fast` - If `True`, will fail after first different pixel. `False` by default ## License Apache 2.0 LICENSE diff --git a/img_2.png b/img_2.png new file mode 100644 index 0000000..5807be9 Binary files /dev/null and b/img_2.png differ diff --git a/pytest_playwright_snapshot/plugin.py b/pytest_playwright_snapshot/plugin.py deleted file mode 100644 index a3c4057..0000000 --- a/pytest_playwright_snapshot/plugin.py +++ /dev/null @@ -1,43 +0,0 @@ -import sys -from io import BytesIO -from pathlib import Path -from typing import Any, Callable - -import pytest -from PIL import Image -from pixelmatch.contrib.PIL import pixelmatch - - -@pytest.fixture -def assert_snapshot(pytestconfig: Any, request: Any, browser_name: str) -> Callable: - def compare(img: bytes, name: str, *, threshold: float = 0.1) -> None: - update_snapshot = pytestconfig.getoption("--update-snapshots") - filepath = ( - Path(request.node.fspath).parent.resolve() - / "__snapshots__" - / browser_name - / sys.platform - ) - filepath.mkdir(parents=True, exist_ok=True) - file = filepath / name - if update_snapshot: - file.write_bytes(img) - return - if not file.exists(): - pytest.fail("Snapshot not found, use --update-snapshots to update it.") - image = Image.open(BytesIO(img)) - golden = Image.open(file) - diff_pixels = pixelmatch(image, golden, threshold=threshold) - assert diff_pixels == 0, "Snapshots does not match" - - return compare - - -def pytest_addoption(parser: Any) -> None: - group = parser.getgroup("playwright-snapshot", "Playwright Snapshot") - group.addoption( - "--update-snapshots", - action="store_true", - default=False, - help="Update snapshots.", - ) diff --git a/pytest_playwright_snapshot/__init__.py b/pytest_playwright_visual/__init__.py similarity index 100% rename from pytest_playwright_snapshot/__init__.py rename to pytest_playwright_visual/__init__.py diff --git a/pytest_playwright_visual/plugin.py b/pytest_playwright_visual/plugin.py new file mode 100644 index 0000000..e980907 --- /dev/null +++ b/pytest_playwright_visual/plugin.py @@ -0,0 +1,75 @@ +import sys +import os +import shutil +import math +from io import BytesIO +from pathlib import Path +from typing import Any, Callable +import pytest +from PIL import Image +from pixelmatch.contrib.PIL import pixelmatch + + +@pytest.fixture +def assert_snapshot(pytestconfig: Any, request: Any, browser_name: str) -> Callable: + test_name = f"{str(Path(request.node.name))}[{str(sys.platform)}]" + test_dir = str(Path(request.node.name)).split('[', 1)[0] + + def compare(img: bytes, *, diff_pixels: int = 0, diff_ratio: float = 0.0, + threshold: float = 0.1, name=f'{test_name}.png', fail_fast=False) -> None: + update_snapshot = pytestconfig.getoption("--update-snapshots") + test_file_name = str(os.path.basename(Path(request.node.fspath))).strip('.py') + filepath = ( + Path(request.node.fspath).parent.resolve() + / 'snapshots' + / test_file_name + / test_dir + ) + filepath.mkdir(parents=True, exist_ok=True) + file = filepath / name + # Create a dir where all snapshot test failures will go + results_dir_name = (Path(request.node.fspath).parent.resolve() + / "snapshot_tests_failures") + test_results_dir = (results_dir_name + / test_file_name / test_name) + # Remove a single test's past run dir with actual, diff and expected images + if test_results_dir.exists(): + shutil.rmtree(test_results_dir) + if update_snapshot: + file.write_bytes(img) + pytest.fail("--> Snapshots updated. Please review images") + if not file.exists(): + file.write_bytes(img) + # pytest.fail( + pytest.fail("--> New snapshot(s) created. Please review images") + img_a = Image.open(BytesIO(img)) + img_b = Image.open(file) + img_diff = Image.new("RGBA", img_a.size) + mismatch = pixelmatch(img_a, img_b, img_diff, threshold=threshold, fail_fast=fail_fast) + if mismatch == 0: + return + if mismatch <= diff_pixels: + return + if (mismatch/math.prod(img_a.size)) <= diff_ratio: + return + else: + # Create new test_results folder + test_results_dir.mkdir(parents=True, exist_ok=True) + img_diff.save(f'{test_results_dir}/Diff_{name}') + img_a.save(f'{test_results_dir}/Actual_{name}') + img_b.save(f'{test_results_dir}/Expected_{name}') + print(f"threshold is {threshold}") + print(f"number of mismatch pixels is {mismatch}") + pytest.fail("--> Snapshots DO NOT match!") + + return compare + + +def pytest_addoption(parser: Any) -> None: + group = parser.getgroup("playwright-snapshot", "Playwright Snapshot") + group.addoption( + "--update-snapshots", + action="store_true", + default=False, + help="Update snapshots.", + ) diff --git a/setup.py b/setup.py index 720abe8..3b06746 100644 --- a/setup.py +++ b/setup.py @@ -1,35 +1,35 @@ from pathlib import Path - from setuptools import setup + setup( - name="pytest-playwright-snapshot", - author="Kumar Aditya", - author_email="", - description="A pytest wrapper for snapshot testing with playwright", + name="pytest-playwright-visual", + author="Symon Storozhenko", + author_email="symon.storozhenko@gmail.com", + description="A pytest fixture for visual testing with Playwright", long_description=Path("README.md").read_text(), long_description_content_type="text/markdown", - url="https://github.com/kumaraditya303/pytest-playwright-snapshot", - packages=["pytest_playwright_snapshot"], + url="https://github.com/symon-storozhenko/pytest-playwright-visual", + packages=["pytest_playwright_visual"], include_package_data=True, install_requires=[ "pytest_playwright>=0.1.2", "Pillow>=8.2.0", - "pixelmatch==0.2.3", + "pixelmatch>=0.3.0", ], entry_points={ - "pytest11": ["playwright_snapshot = pytest_playwright_snapshot.plugin"] + "pytest11": ["playwright_visual = pytest_playwright_visual.plugin"] }, classifiers=[ "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", "Framework :: Pytest", ], - python_requires=">=3.7", + python_requires=">=3.8", use_scm_version=True, setup_requires=["setuptools_scm"], ) diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 07cd605..5394bf3 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -1,68 +1,230 @@ import sys from pathlib import Path - import pytest import requests +import os @pytest.mark.parametrize( "browser_name", ["chromium", "firefox", "webkit"], ) -def test_snapshot_create(browser_name: str, testdir: pytest.Testdir) -> None: +def test_filepath_exists(browser_name: str, testdir: pytest.Testdir) -> None: testdir.makepyfile( """ def test_snapshot(page, assert_snapshot): page.goto("https://example.com") - assert_snapshot(page.screenshot(), "test-snapshot.png") + assert_snapshot(page.screenshot()) """ ) filepath = ( - Path(testdir.tmpdir) - / "__snapshots__" - / browser_name - / sys.platform - / "test-snapshot.png" + Path(testdir.tmpdir) + / "snapshots" + / "test_filepath_exists" + / "test_snapshot" + / f"test_snapshot[{browser_name}][{sys.platform}].png" ).resolve() + result = testdir.runpytest("--browser", browser_name) + file_path_actual, file_name = "", "" + for dirpath, dirnames, filenames in os.walk("."): + for filename in [f for f in filenames + if f.endswith(".png")]: + file_path_actual = dirpath + file_name = filename + result.assert_outcomes(failed=1) + if not filepath.exists(): + pytest.fail(f"Filepath does not exist, but found this dir {file_path_actual} " + f"and filename :{file_name})") + +@pytest.mark.parametrize( + "browser_name", + ["chromium", "firefox", "webkit"], +) +def test_compare_pass(browser_name: str, testdir: pytest.Testdir) -> None: + testdir.makepyfile( + """ + def test_snapshot(page, assert_snapshot): + page.goto("https://example.com") + assert_snapshot(page.screenshot()) + """ + ) result = testdir.runpytest("--browser", browser_name) result.assert_outcomes(failed=1) - assert "Snapshot not found, use --update-snapshots to update it." in "".join( - result.outlines + assert "--> New snapshot(s) created. Please review images" in "".join(result.outlines) + result = testdir.runpytest("--browser", browser_name) + result.assert_outcomes(passed=1) + + +@pytest.mark.parametrize( + "browser_name", + ["chromium", "firefox", "webkit"], +) +def test_custom_image_name_generated(browser_name: str, testdir: pytest.Testdir) -> None: + testdir.makepyfile( + """ + def test_snapshot(page, assert_snapshot): + page.goto("https://example.com") + assert_snapshot(page.screenshot(), name="test.png") + """ ) - assert not filepath.exists() + filepath = ( + Path(testdir.tmpdir) + / "snapshots" + / "test_custom_image_name_generated" + / "test_snapshot" + / f"test.png" + ).resolve() + result = testdir.runpytest("--browser", browser_name) + file_path_actual, file_name = "", "" + for dirpath, dirnames, filenames in os.walk("."): + for filename in [f for f in filenames + if f.endswith(".png")]: + file_path_actual = dirpath + file_name = filename + result.assert_outcomes(failed=1) + if not filepath.exists(): + pytest.fail(f"Filepath does not exist, but found this dir {file_path_actual} " + f"and filename :{file_name})") + result.assert_outcomes(failed=1) + result = testdir.runpytest("--browser", browser_name) + result.assert_outcomes(passed=1) + +@pytest.mark.parametrize( + "browser_name", + ["chromium"], +) +def test_compare_fail(browser_name: str, testdir: pytest.Testdir) -> None: + testdir.makepyfile( + """ + def test_snapshot(page, assert_snapshot): + page.goto("https://via.placeholder.com/250/000000") + element = page.query_selector('img') + assert_snapshot(element.screenshot()) + """ + ) + # test_name = f"{str(Path(request.node.name))}[{str(sys.platform)}]" + filepath = ( + Path(testdir.tmpdir) + / "snapshots" + / "test_compare_fail" + / "test_snapshot" + / f"test_snapshot[{browser_name}][{sys.platform}].png" + ).resolve() + result = testdir.runpytest("--browser", browser_name) + result.assert_outcomes(failed=1) + assert "--> New snapshot(s) created. Please review images" in "".join(result.outlines) result = testdir.runpytest("--browser", browser_name, "--update-snapshots") - result.assert_outcomes(passed=1) - assert filepath.exists() + result.assert_outcomes(failed=1) + assert "--> Snapshots updated. Please review images" in "".join(result.outlines) + img = requests.get("https://via.placeholder.com/250/FFFFFF").content + filepath.write_bytes(img) + result = testdir.runpytest("--browser", browser_name) + result.assert_outcomes(failed=1) + assert "--> Snapshots DO NOT match!" in "".join(result.outlines) @pytest.mark.parametrize( "browser_name", - ["chromium", "firefox", "webkit"], + ["firefox", "webkit"], ) -def test_snapshot_fail(browser_name: str, testdir: pytest.Testdir) -> None: +def test_compare_with_fail_fast(browser_name: str, testdir: pytest.Testdir) -> None: testdir.makepyfile( """ def test_snapshot(page, assert_snapshot): page.goto("https://via.placeholder.com/250/000000") element = page.query_selector('img') - assert_snapshot(element.screenshot(), "test-snapshot.png") + assert_snapshot(element.screenshot(), fail_fast=True) """ ) + # test_name = f"{str(Path(request.node.name))}[{str(sys.platform)}]" filepath = ( - Path(testdir.tmpdir) - / "__snapshots__" - / browser_name - / sys.platform - / "test-snapshot.png" + Path(testdir.tmpdir) + / "snapshots" + / "test_compare_with_fail_fast" + / "test_snapshot" + / f"test_snapshot[{browser_name}][{sys.platform}].png" ).resolve() + result = testdir.runpytest("--browser", browser_name, "--update-snapshots") + result.assert_outcomes(failed=1) + assert "--> Snapshots updated. Please review images" in "".join(result.outlines) + img = requests.get("https://via.placeholder.com/250/FFFFFF").content + filepath.write_bytes(img) + result = testdir.runpytest("--browser", browser_name) + result.assert_outcomes(failed=1) + assert "--> Snapshots DO NOT match!" in "".join(result.outlines) + + +@pytest.mark.parametrize( + "browser_name", + ["chromium", "firefox", "webkit"], +) +def test_actual_expected_diff_images_generated(browser_name: str, testdir: pytest.Testdir) -> None: + testdir.makepyfile( + """ + def test_snapshot(page, assert_snapshot): + page.goto("https://via.placeholder.com/250/000000") + element = page.query_selector('img') + assert_snapshot(element.screenshot()) + """ + ) + # test_name = f"{str(Path(request.node.name))}[{str(sys.platform)}]" + filepath = ( + Path(testdir.tmpdir) + / "snapshots" + / "test_actual_expected_diff_images_generated" + / "test_snapshot" + / f"test_snapshot[{browser_name}][{sys.platform}].png" + ).resolve() result = testdir.runpytest("--browser", browser_name, "--update-snapshots") - result.assert_outcomes(passed=1) - assert filepath.exists() + result.assert_outcomes(failed=1) + assert "--> Snapshots updated. Please review images" in "".join( + result.outlines + ) img = requests.get("https://via.placeholder.com/250/FFFFFF").content filepath.write_bytes(img) result = testdir.runpytest("--browser", browser_name) result.assert_outcomes(failed=1) - assert "Snapshots does not match" in "".join(result.outlines) + results_dir_name = (Path(testdir.tmpdir) + / "snapshot_tests_failures") + test_results_dir = (results_dir_name + / "test_actual_expected_diff_images_generated" + / f"test_snapshot[{browser_name}][{sys.platform}]") + actual_img = (test_results_dir / f"Actual_test_snapshot[{browser_name}][{sys.platform}].png") + expected_img = (test_results_dir / f"Expected_test_snapshot[{browser_name}][{sys.platform}].png") + diff_img = (test_results_dir / f"Diff_test_snapshot[{browser_name}][{sys.platform}].png") + + # Validate the actual image exists in results folder + file_path_actual, file_name_actual = "", "" + for dirpath, dirnames, filenames in os.walk("."): + for filename in [f for f in filenames + if f.startswith("Actual_")]: + file_path_actual = dirpath + file_name_actual = filename + if not actual_img.exists(): + pytest.fail(f"Filepath does not exist, but found this dir {file_path_actual} " + f"and filename :{file_name_actual})") + + # Validate the expected image exists in results folder + file_path_expected, file_name_expected = "", "" + for dirpath, dirnames, filenames in os.walk("."): + for filename in [f for f in filenames + if f.startswith("Expected_")]: + file_path_expected = dirpath + file_name_expected = filename + if not expected_img.exists(): + pytest.fail(f"Filepath does not exist, but found this dir {file_path_expected} " + f"and filename :{file_name_expected})") + + # Validate the difference image exists in results folder + file_path_diff, file_name_diff = "", "" + for dirpath, dirnames, filenames in os.walk("."): + for filename in [f for f in filenames + if f.startswith("Diff_")]: + file_path_diff = dirpath + file_name_diff = filename + if not diff_img.exists(): + pytest.fail(f"Filepath does not exist, but found this dir {file_path_diff} " + f"and filename :{file_name_diff})")