From 1f94e78fada67823c49a0d07e5aed2c2add319e5 Mon Sep 17 00:00:00 2001 From: Bastian Krause Date: Sat, 4 Jan 2025 13:51:03 +0100 Subject: [PATCH] python: allow adding parameter names to parametrized test IDs By default, only the parameter's values make it into parametrized test IDs. The parameter names don't. Since parameter values do not always speak for themselves, the test function + test ID are often not descriptive/expressive. Allowing parameter name=value pairs in the test ID optionally to get an idea what parameters a test gets passed is beneficial. So add a kwarg `id_names` to @pytest.mark.parametrize() / pytest.Metafunc.parametrize(). It defaults to `False` to keep the auto-generated ID as before. If set to `True`, the argument parameter=value pairs in the auto-generated test IDs are enabled. Calling parametrize() with `ids` and `id_names=True` is considered an error. Auto-generated test ID with `id_names=False` (default behavior as before): test_something[100-10-True-False-True] Test ID with `id_names=True`: test_something[speed_down=100-speed_up=10-foo=True-bar=False-baz=True] Signed-off-by: Bastian Krause --- AUTHORS | 1 + changelog/13055.feature.rst | 1 + doc/en/example/parametrize.rst | 43 +++++++++++++++++++++------------- src/_pytest/mark/structures.py | 1 + src/_pytest/python.py | 37 +++++++++++++++++++++++------ testing/python/metafunc.py | 34 ++++++++++++++++++++++----- 6 files changed, 88 insertions(+), 29 deletions(-) create mode 100644 changelog/13055.feature.rst diff --git a/AUTHORS b/AUTHORS index 76099413072..0d87f0f1904 100644 --- a/AUTHORS +++ b/AUTHORS @@ -60,6 +60,7 @@ Aviv Palivoda Babak Keyvani Bahram Farahmand Barney Gale +Bastian Krause Ben Brown Ben Gartner Ben Leith diff --git a/changelog/13055.feature.rst b/changelog/13055.feature.rst new file mode 100644 index 00000000000..7b7fdb71dc5 --- /dev/null +++ b/changelog/13055.feature.rst @@ -0,0 +1 @@ +``@pytest.mark.parametrize()`` and ``pytest.Metafunc.parametrize()`` now support the ``id_names`` argument enabling auto-generated test IDs consisting of parameter name=value pairs. diff --git a/doc/en/example/parametrize.rst b/doc/en/example/parametrize.rst index 8e6479254bb..577b8300ef5 100644 --- a/doc/en/example/parametrize.rst +++ b/doc/en/example/parametrize.rst @@ -111,12 +111,18 @@ the argument name: assert diff == expected - @pytest.mark.parametrize("a,b,expected", testdata, ids=["forward", "backward"]) + @pytest.mark.parametrize("a,b,expected", testdata, id_names=True) def test_timedistance_v1(a, b, expected): diff = a - b assert diff == expected + @pytest.mark.parametrize("a,b,expected", testdata, ids=["forward", "backward"]) + def test_timedistance_v2(a, b, expected): + diff = a - b + assert diff == expected + + def idfn(val): if isinstance(val, (datetime,)): # note this wouldn't show any hours/minutes/seconds @@ -124,7 +130,7 @@ the argument name: @pytest.mark.parametrize("a,b,expected", testdata, ids=idfn) - def test_timedistance_v2(a, b, expected): + def test_timedistance_v3(a, b, expected): diff = a - b assert diff == expected @@ -140,16 +146,19 @@ the argument name: ), ], ) - def test_timedistance_v3(a, b, expected): + def test_timedistance_v4(a, b, expected): diff = a - b assert diff == expected In ``test_timedistance_v0``, we let pytest generate the test IDs. -In ``test_timedistance_v1``, we specified ``ids`` as a list of strings which were +In ``test_timedistance_v1``, we let pytest generate the test IDs using argument +name/value pairs. + +In ``test_timedistance_v2``, we specified ``ids`` as a list of strings which were used as the test IDs. These are succinct, but can be a pain to maintain. -In ``test_timedistance_v2``, we specified ``ids`` as a function that can generate a +In ``test_timedistance_v3``, we specified ``ids`` as a function that can generate a string representation to make part of the test ID. So our ``datetime`` values use the label generated by ``idfn``, but because we didn't generate a label for ``timedelta`` objects, they are still using the default pytest representation: @@ -160,22 +169,24 @@ objects, they are still using the default pytest representation: =========================== test session starts ============================ platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y rootdir: /home/sweet/project - collected 8 items + collected 10 items - - - - - - - - ======================== 8 tests collected in 0.12s ======================== - -In ``test_timedistance_v3``, we used ``pytest.param`` to specify the test IDs + + + + + + + + + + ======================== 10 tests collected in 0.12s ======================= + +In ``test_timedistance_v4``, we used ``pytest.param`` to specify the test IDs together with the actual data, instead of listing them separately. A quick port of "testscenarios" diff --git a/src/_pytest/mark/structures.py b/src/_pytest/mark/structures.py index f9261076ad0..6d41bef5c82 100644 --- a/src/_pytest/mark/structures.py +++ b/src/_pytest/mark/structures.py @@ -519,6 +519,7 @@ def __call__( # type: ignore[override] | Callable[[Any], object | None] | None = ..., scope: _ScopeName | None = ..., + id_names: bool = ..., ) -> MarkDecorator: ... class _UsefixturesMarkDecorator(MarkDecorator): diff --git a/src/_pytest/python.py b/src/_pytest/python.py index 8e4fb041532..0a88b6ef8a2 100644 --- a/src/_pytest/python.py +++ b/src/_pytest/python.py @@ -888,18 +888,21 @@ class IdMaker: # Used only for clearer error messages. func_name: str | None - def make_unique_parameterset_ids(self) -> list[str | _HiddenParam]: + def make_unique_parameterset_ids( + self, id_names: bool = False + ) -> list[str | _HiddenParam]: """Make a unique identifier for each ParameterSet, that may be used to identify the parametrization in a node ID. - Format is -...-[counter], where prm_x_token is + Format is [=]-...-[=][counter], + where prm_x is (only for id_names=True) and prm_x_token is - user-provided id, if given - else an id derived from the value, applicable for certain types - else The counter suffix is appended only in case a string wouldn't be unique otherwise. """ - resolved_ids = list(self._resolve_ids()) + resolved_ids = list(self._resolve_ids(id_names=id_names)) # All IDs must be unique! if len(resolved_ids) != len(set(resolved_ids)): # Record the number of occurrences of each ID. @@ -925,7 +928,7 @@ def make_unique_parameterset_ids(self) -> list[str | _HiddenParam]: ) return resolved_ids - def _resolve_ids(self) -> Iterable[str | _HiddenParam]: + def _resolve_ids(self, id_names: bool = False) -> Iterable[str | _HiddenParam]: """Resolve IDs for all ParameterSets (may contain duplicates).""" for idx, parameterset in enumerate(self.parametersets): if parameterset.id is not None: @@ -942,8 +945,9 @@ def _resolve_ids(self) -> Iterable[str | _HiddenParam]: yield self._idval_from_value_required(self.ids[idx], idx) else: # ID not provided - generate it. + idval_func = self._idval_named if id_names else self._idval yield "-".join( - self._idval(val, argname, idx) + idval_func(val, argname, idx) for val, argname in zip(parameterset.values, self.argnames) ) @@ -960,6 +964,11 @@ def _idval(self, val: object, argname: str, idx: int) -> str: return idval return self._idval_from_argname(argname, idx) + def _idval_named(self, val: object, argname: str, idx: int) -> str: + """Make an ID in argname=value format for a parameter in a + ParameterSet.""" + return "=".join((argname, self._idval(val, argname, idx))) + def _idval_from_function(self, val: object, argname: str, idx: int) -> str | None: """Try to make an ID for a parameter in a ParameterSet using the user-provided id callable, if given.""" @@ -1167,6 +1176,7 @@ def parametrize( indirect: bool | Sequence[str] = False, ids: Iterable[object | None] | Callable[[Any], object | None] | None = None, scope: _ScopeName | None = None, + id_names: bool = False, *, _param_mark: Mark | None = None, ) -> None: @@ -1236,6 +1246,11 @@ def parametrize( The scope is used for grouping tests by parameter instances. It will also override any fixture-function defined scope, allowing to set a dynamic scope using test context or configuration. + + :param id_names: + Whether the argument names should be part of the auto-generated + ids. Defaults to ``False``. Must not be ``True`` if ``ids`` is + given. """ nodeid = self.definition.nodeid @@ -1261,6 +1276,9 @@ def parametrize( else: scope_ = _find_parametrized_scope(argnames, self._arg2fixturedefs, indirect) + if id_names and ids is not None: + fail("'id_names' must not be combined with 'ids'", pytrace=False) + self._validate_if_using_arg_names(argnames, indirect) # Use any already (possibly) generated ids with parametrize Marks. @@ -1270,7 +1288,11 @@ def parametrize( ids = generated_ids ids = self._resolve_parameter_set_ids( - argnames, ids, parametersets, nodeid=self.definition.nodeid + argnames, + ids, + parametersets, + nodeid=self.definition.nodeid, + id_names=id_names, ) # Store used (possibly generated) ids with parametrize Marks. @@ -1356,6 +1378,7 @@ def _resolve_parameter_set_ids( ids: Iterable[object | None] | Callable[[Any], object | None] | None, parametersets: Sequence[ParameterSet], nodeid: str, + id_names: bool, ) -> list[str | _HiddenParam]: """Resolve the actual ids for the given parameter sets. @@ -1390,7 +1413,7 @@ def _resolve_parameter_set_ids( nodeid=nodeid, func_name=self.function.__name__, ) - return id_maker.make_unique_parameterset_ids() + return id_maker.make_unique_parameterset_ids(id_names=id_names) def _validate_ids( self, diff --git a/testing/python/metafunc.py b/testing/python/metafunc.py index 7ae26de3a18..62a406243ab 100644 --- a/testing/python/metafunc.py +++ b/testing/python/metafunc.py @@ -204,28 +204,50 @@ def find_scope(argnames, indirect): ) assert find_scope(["mixed_fix"], indirect=True) == Scope.Class - def test_parametrize_and_id(self) -> None: + @pytest.mark.parametrize("id_names", (False, True)) + def test_parametrize_and_id(self, id_names: bool) -> None: def func(x, y): - pass + """Dummy function""" metafunc = self.Metafunc(func) metafunc.parametrize("x", [1, 2], ids=["basic", "advanced"]) - metafunc.parametrize("y", ["abc", "def"]) + metafunc.parametrize("y", ["abc", "def"], id_names=id_names) ids = [x.id for x in metafunc._calls] - assert ids == ["basic-abc", "basic-def", "advanced-abc", "advanced-def"] + if id_names: + assert ids == [ + "basic-y=abc", + "basic-y=def", + "advanced-y=abc", + "advanced-y=def", + ] + else: + assert ids == ["basic-abc", "basic-def", "advanced-abc", "advanced-def"] - def test_parametrize_and_id_unicode(self) -> None: + @pytest.mark.parametrize("id_names", (False, True)) + def test_parametrize_and_id_unicode(self, id_names: bool) -> None: """Allow unicode strings for "ids" parameter in Python 2 (##1905)""" def func(x): - pass + """Dummy function""" metafunc = self.Metafunc(func) metafunc.parametrize("x", [1, 2], ids=["basic", "advanced"]) ids = [x.id for x in metafunc._calls] assert ids == ["basic", "advanced"] + def test_parametrize_with_bad_ids_name_combination(self) -> None: + def func(x): + """Dummy function""" + + metafunc = self.Metafunc(func) + + with pytest.raises( + fail.Exception, + match="'id_names' must not be combined with 'ids'", + ): + metafunc.parametrize("x", [1, 2], ids=["basic", "advanced"], id_names=True) + def test_parametrize_with_wrong_number_of_ids(self) -> None: def func(x, y): pass