Make tests just a runtime warning (#37)

Author: ssbarnea

pyproject.toml

@@ -33,9 +33,7 @@ description = "PyTest Plus Plugin :: extends pytest functionality"
 dynamic = ["version", "dependencies", "optional-dependencies"]
 keywords = ["testing", "pytest", "plugin"]
 license = {text = "MIT"}
-maintainers = [
-  {"email" = "[email protected]", "name" = "Sorin Sbarnea"}
-]
+maintainers = [{"email" = "[email protected]", "name" = "Sorin Sbarnea"}]
 name = "pytest-plus"
 readme = "README.md"
 # https://peps.python.org/pep-0621/#readme
@@ -58,7 +56,7 @@ show_missing = true
 omit = ["test/*", "/private/var/folders/*", "/tmp/*"]
 
 [tool.pytest.ini_options]
-addopts = "-p no:flaky"
+addopts = "-p no:flaky --showlocals"
 filterwarnings = [
   "error",
   "ignore:ast.(Str|Num|NameConstant) is deprecated and will be removed in Python 3.14:DeprecationWarning:_pytest.assertion.rewrite",

src/pytest_plus/__init__.py

@@ -2,13 +2,15 @@
 
 from __future__ import annotations
 
+import logging
 import os
 import re
 import shutil
 
 import pytest
 from _pytest.terminal import TerminalReporter
 
+_logger = logging.getLogger(__name__)
 PYTEST_CHECK_TEST_DUPLICATE = int(os.environ.get("PYTEST_CHECK_TEST_DUPLICATE", "1"))
 
 
@@ -61,7 +63,7 @@ def pytest_sessionfinish(session: pytest.Session) -> None:
 @pytest.hookimpl(tryfirst=True)  # type: ignore[misc,unused-ignore]
 def pytest_collection_modifyitems(items: list[pytest.Item]) -> None:
     """Ensure testing fails if tests have duplicate names."""
-    errors = []
+    messages = []
     names = {}
     max_test_id_length = get_max_test_id_length()
     test_id_regex = get_test_id_regex()
@@ -70,21 +72,18 @@ def pytest_collection_modifyitems(items: list[pytest.Item]) -> None:
         if base_name not in names:
             names[base_name] = item.location
         elif item.location[:2] != names[base_name][:2] and PYTEST_CHECK_TEST_DUPLICATE:
-            error = f"Duplicate test name '{base_name}', found at {item.location[0]}:{item.location[1]} and {names[base_name][0]}:{names[base_name][1]}"
-            if error not in errors:
-                errors.append(error)
+            msg = f"Duplicate test name '{base_name}', found at {item.location[0]}:{item.location[1]} and {names[base_name][0]}:{names[base_name][1]}"
+            if msg not in messages:
+                messages.append(msg)
         if hasattr(item, "callspec"):
             test_id = item.callspec.id
             if max_test_id_length and len(test_id) > max_test_id_length:
-                errors.append(
+                messages.append(
                     f"{item} has an id that looks above {max_test_id_length} characters.",
                 )
             elif test_id_regex and not test_id_regex.match(test_id):
-                errors.append(
+                messages.append(
                     f"Test {item} has an id that does not match our safe pattern '{test_id_regex.pattern}' for use with a terminal.",
                 )
-    if errors:
-        msg = f"Failed run due to following issues being identified:\n{os.linesep.join(errors)}\nTotal errors found: {len(errors)}"
-        raise pytest.UsageError(
-            msg,
-        )
+    for msg in messages:
+        _logger.warning(msg)

test/test_plugin.py

@@ -1,14 +1,21 @@
 """Tests."""
 
+import logging
 import os
 
 import pytest
 
 pytest_plugins = ["pytester"]
 
 
-def test_duplicate_test_name(pytester: pytest.Pytester) -> None:
+def test_duplicate_test_name(
+    pytester: pytest.Pytester,
+    caplog: pytest.LogCaptureFixture,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
     """Validates that we can detect duplicate test names."""
+    monkeypatch.delenv("PYTEST_REQPASS", raising=False)
+    caplog.set_level(logging.INFO)
     p1 = pytester.makepyfile(
         test_one="""
             def test_a():
@@ -24,25 +31,28 @@ def test_a():
 
     result = pytester.runpytest_inprocess(p1, p2)
     assert (
-        result.errlines[0]
-        == "ERROR: Failed run due to following issues being identified:"
+        "Duplicate test name 'test_a', found at test_two.py:0 and test_one.py:0"
+        in caplog.text
     )
-    assert (
-        result.errlines[1]
-        == "Duplicate test name 'test_a', found at test_two.py:0 and test_one.py:0"
-    )
-    assert result.ret == pytest.ExitCode.USAGE_ERROR
+    assert result.ret == 0, result.stdout.lines
 
 
 @pytest.mark.parametrize(
     ("rc", "disable"),
     [
-        pytest.param(pytest.ExitCode.USAGE_ERROR, False, id="0"),
+        pytest.param(pytest.ExitCode.OK, False, id="0"),
         pytest.param(pytest.ExitCode.OK, True, id="1"),
     ],
 )
-def test_check_test_id(pytester: pytest.Pytester, rc: int, *, disable: bool) -> None:
+def test_check_test_id(
+    pytester: pytest.Pytester,
+    rc: int,
+    *,
+    disable: bool,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
     """Validates that we can detect duplicate test names."""
+    caplog.set_level(logging.WARNING)
     if disable:
         os.environ["PYTEST_CHECK_TEST_ID_REGEX"] = "0"
     p1 = pytester.makepyfile(
@@ -62,13 +72,17 @@ def test_a(some: str):
     if not disable:
         assert (
             "Test <Function test_a[invalid name]> has an id that does not match our safe pattern '^[\\w_\\-\\.:]+$' for use with a terminal."
-            in result.stderr.lines
+            in caplog.text
         )
     assert result.ret == rc
 
 
-def test_check_test_id_length(pytester: pytest.Pytester) -> None:
+def test_check_test_id_length(
+    pytester: pytest.Pytester,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
     """Validates that we can detect duplicate test names."""
+    caplog.set_level(logging.WARNING)
     p1 = pytester.makepyfile(
         test_one="""
             import pytest
@@ -85,6 +99,6 @@ def test_a(some: str):
     result = pytester.runpytest_inprocess("--collect-only", p1)
     assert (
         "<Function test_a[this-is-too-long-for-our-taste-so-we-ask-you-to-make-it-shorter]> has an id that looks above 60 characters."
-        in result.stderr.lines
+        in caplog.text
     )
-    assert result.ret == pytest.ExitCode.USAGE_ERROR
+    assert result.ret == pytest.ExitCode.OK