Backtestspytestrun-20261106-234554#13431
ParameterSet is private
Our patch was judged better than the merged fix.
In this judgment, Patch A = our agent, Patch B = the merged fix.
Both patches make the identical core change: import ParameterSet into src/pytest/__init__.py and add it to __all__, remove the nitpick_ignore entry, and document it in reference.rst. The independent tests pass on both, so correctness is equal. The deciding criterion is tests: Patch A adds a __all__ membership check, an identity/isinstance check against the private class, and a pytester test that runs the issue's exact scenario (typed params() helper feeding parametrize under -W error). These would fail before the change and pass after. Patch B adds no tests at all. On scope, Patch B includes unrelated churn: an AUTHORS line, an unrelated nitpick_ignore for pluggy._compat.DistFacade, and it demotes the useful ParameterSet docstring examples into code comments (losing them from rendered docs) while leaving a '<version>' placeholder in versionadded. Its __module__ = 'pytest' override and ':meta private:' markers on param/extract_from are reasonable polish for autodoc, but that does not outweigh missing tests and the docstring regression. Patch A also adds a helpful note under pytest.param pointing readers to the type. A is the better fix.
Four judges ruled on this ticket in parallel, and the verdict comes from counting their votes. Each saw the issue and the two patches, labelled A and B in random order (two of them saw ours as A), with issue links and version numbers removed. Where tests could check the fix, they also saw the results of tests a separate agent wrote for the issue, run on both patches. None was told which patch was ours. The reasoning above is one judge's, shown verbatim.
Solve
8m 31s
Wall-clock time in the container
Full test suite
No new failures
3,952 passed · 50 skipped · 4,082 total
Contamination audit
Clean
No sign the agent saw the real fix
Patch size
59vs 69 merged
Changed lines, ours vs merged
The issue
View on GitHubWhen using pytest.param to create parameters for a test from a function, that function will return a list of ParameterSet but currently that is only available from _pytest.mark.structures which is private so importing it results in a private import warning.
It would be ideal if this structure was publicly available so developers don't need to disable warnings for uses of pytest.param when using fully hinted code.
The current situation looks like this, which requires a pyright ignore directive to be warning free.
This is simplified example, which could be defined inline but for more complex where the test cases were built by code using a function is more necessary.
from _pytest.mark.structures import ParameterSet # pyright: ignore[reportPrivateImportUsage]
import pytest
def params() -> list[ParameterSet]:
return [
pytest.param("basic", 10, True, id="basic")
pytest.param("advanced", 5, False, id="advanced")}
]
@pytest.mark.parametrize(("val1", "val2", "expected"), params())
async def test_ratelimiter(val1: str, val2: int, expected: bool) -> None:
"""Test code goes here..."""
Ideally this would be public to avoid this, for example:
from pytest.mark.structures import ParameterSet
import pytest
def params() -> list[ParameterSet]:
return [
pytest.param("basic", 10, True, id="basic")
pytest.param("advanced", 5, False, id="advanced")}
]
@pytest.mark.parametrize(("val1", "val2", "expected"), params())
async def test_ratelimiter(val1: str, val2: int, expected: bool) -> None:
"""Test code goes here..."""The two patches
changelog/13431.improvement.rstnew file mode 100644index 000000000..94332209a@@ -0,0 +1,2 @@+:class:`pytest.ParameterSet`, the type returned by :func:`pytest.param`, is now part of the public API,+so helper functions building parameter lists can be type-annotated without importing from ``_pytest``.doc/en/conf.pyindex 803a5b2a9..bca96227b 100644@@ -78,7 +78,6 @@ nitpick_ignore = [ ("py:class", "_pytest.fixtures.FixtureFunctionMarker"), ("py:class", "_pytest.fixtures.FixtureFunctionDefinition"), ("py:class", "_pytest.logging.LogCaptureHandler"),- ("py:class", "_pytest.mark.structures.ParameterSet"), # Intentionally undocumented/private ("py:class", "_pytest._code.code.Traceback"), ("py:class", "_pytest._py.path.LocalPath"),doc/en/reference/reference.rstindex 4e0f40f4f..42708b9b9 100644@@ -116,6 +116,9 @@ pytest.param .. autofunction:: pytest.param(*values, [id], [marks]) +The returned :class:`pytest.ParameterSet` can be used in type annotations, for example+to annotate a helper function which builds a list of parameters.+ pytest.raises ~~~~~~~~~~~~~ @@ -1037,6 +1040,11 @@ Metafunc .. autoclass:: pytest.Metafunc() :members: +ParameterSet+~~~~~~~~~~~~++.. autoclass:: pytest.ParameterSet()+ Parser ~~~~~~ src/pytest/__init__.pyindex f6ea24f95..ecd075ff5 100644@@ -42,6 +42,7 @@ from _pytest.mark import MARK_GEN as mark from _pytest.mark import MarkDecorator from _pytest.mark import MarkGenerator from _pytest.mark import param+from _pytest.mark import ParameterSet from _pytest.monkeypatch import MonkeyPatch from _pytest.nodes import Collector from _pytest.nodes import Directory@@ -130,6 +131,7 @@ __all__ = [ "MonkeyPatch", "OptionGroup", "Package",+ "ParameterSet", "Parser", "PytestApproxDecimalToleranceWarning", "PytestAssertRewriteWarning",testing/test_mark.pyindex 82a941df0..20e2bbe4e 100644@@ -24,11 +24,55 @@ import pytest class TestMark:- @pytest.mark.parametrize("attr", ["mark", "param"])+ @pytest.mark.parametrize("attr", ["mark", "param", "ParameterSet"]) def test_pytest_exists_in_namespace_all(self, attr: str) -> None: module = sys.modules["pytest"] assert attr in module.__all__ + def test_pytest_param_returns_public_parameterset(self) -> None:+ """``pytest.ParameterSet`` is the public name of the type returned by+ ``pytest.param``, so it can be used in annotations without importing+ from a private module (#13431)."""+ from _pytest.mark.structures import ParameterSet++ assert pytest.ParameterSet is ParameterSet++ def params() -> list[pytest.ParameterSet]:+ return [+ pytest.param("basic", 10, id="basic"),+ pytest.param("advanced", 5, id="advanced"),+ ]++ result = params()+ assert all(isinstance(p, pytest.ParameterSet) for p in result)+ assert result[0] == pytest.ParameterSet(+ values=("basic", 10), marks=(), id="basic"+ )++ def test_parametrize_with_annotated_parameterset_helper(+ self, pytester: Pytester+ ) -> None:+ pytester.makepyfile(+ """+ import pytest++ def params() -> list[pytest.ParameterSet]:+ return [+ pytest.param("basic", 10, True, id="basic"),+ pytest.param("advanced", 5, False, id="advanced"),+ ]++ @pytest.mark.parametrize(("val1", "val2", "expected"), params())+ def test_it(val1: str, val2: int, expected: bool) -> None:+ assert (val2 > 7) is expected+ """+ )+ result = pytester.runpytest("-v", "-W", "error")+ result.stdout.fnmatch_lines(+ ["*::test_it[[]basic[]] PASSED*", "*::test_it[[]advanced[]] PASSED*"]+ )+ result.assert_outcomes(passed=2)+ def test_pytest_mark_notcallable(self) -> None: mark = MarkGenerator(_ispytest=True) with pytest.raises(TypeError):
AUTHORSindex aa32b2185..1d1af0015 100644@@ -383,6 +383,7 @@ Omri Golan Ondřej Súkup Onkesh Bansal Oscar Benjamin+Paneendra Kumar Parth Patel Patrick Hayes Patrick Lanniganchangelog/13431.improvement.rstnew file mode 100644index 000000000..299642570@@ -0,0 +1,2 @@+Exposed :class:`pytest.ParameterSet` as the type of the return value of :func:`pytest.param()`.+This can be used in type annotations and ``isinstance`` checks.doc/en/conf.pyindex 803a5b2a9..2083e72df 100644@@ -67,6 +67,7 @@ nitpick_ignore = [ ("py:class", "HookCaller"), ("py:class", "HookspecMarker"), ("py:exc", "PluginValidationError"),+ ("py:class", "pluggy._compat.DistFacade"), # Might want to expose/TODO (https://github.com/pytest-dev/pytest/issues/7469) ("py:class", "ExceptionRepr"), ("py:class", "Exit"),@@ -78,7 +79,6 @@ nitpick_ignore = [ ("py:class", "_pytest.fixtures.FixtureFunctionMarker"), ("py:class", "_pytest.fixtures.FixtureFunctionDefinition"), ("py:class", "_pytest.logging.LogCaptureHandler"),- ("py:class", "_pytest.mark.structures.ParameterSet"), # Intentionally undocumented/private ("py:class", "_pytest._code.code.Traceback"), ("py:class", "_pytest._py.path.LocalPath"),doc/en/reference/reference.rstindex 4e0f40f4f..87abcc5f7 100644@@ -1049,6 +1049,11 @@ OptionGroup .. autoclass:: pytest.OptionGroup() :members: +ParameterSet+~~~~~~~~~~~~++.. autoclass:: pytest.ParameterSet()+ PytestPluginManager ~~~~~~~~~~~~~~~~~~~ src/_pytest/mark/structures.pyindex 34a004f48..aaee55771 100644@@ -81,33 +81,37 @@ def get_empty_parameterset_mark( class ParameterSet(NamedTuple):- """A set of values for a set of parameters along with associated marks and- an optional ID for the set.-- Examples::-- pytest.param(1, 2, 3)- # ParameterSet(values=(1, 2, 3), marks=(), id=None)+ """The return type of :func:`pytest.param`. - pytest.param("hello", id="greeting")- # ParameterSet(values=("hello",), marks=(), id="greeting")-- # Parameter set with marks- pytest.param(42, marks=pytest.mark.xfail)- # ParameterSet(values=(42,), marks=(MarkDecorator(...),), id=None)+ A set of values for a set of parameters along with associated marks and+ an optional ID for the set. - # From parametrize mark (parameter names + list of parameter sets)- pytest.mark.parametrize(- ("a", "b", "expected"),- [- (1, 2, 3),- pytest.param(40, 2, 42, id="everything"),- ],- )- # ParameterSet(values=(1, 2, 3), marks=(), id=None)- # ParameterSet(values=(40, 2, 42), marks=(), id="everything")+ .. versionadded:: 9.2 """ + # Examples:+ #+ # pytest.param(1, 2, 3)+ # # ParameterSet(values=(1, 2, 3), marks=(), id=None)+ #+ # pytest.param("hello", id="greeting")+ # # ParameterSet(values=("hello",), marks=(), id="greeting")+ #+ # # Parameter set with marks+ # pytest.param(42, marks=pytest.mark.xfail)+ # # ParameterSet(values=(42,), marks=(MarkDecorator(...),), id=None)+ #+ # # From parametrize mark (parameter names + list of parameter sets)+ # pytest.mark.parametrize(+ # ("a", "b", "expected"),+ # [+ # (1, 2, 3),+ # pytest.param(40, 2, 42, id="everything"),+ # ],+ # )+ # # ParameterSet(values=(1, 2, 3), marks=(), id=None)+ # # ParameterSet(values=(40, 2, 42), marks=(), id="everything")+ values: Sequence[object | NotSetType] marks: Collection[MarkDecorator | Mark] id: str | _HiddenParam | None@@ -119,6 +123,7 @@ class ParameterSet(NamedTuple): marks: MarkDecorator | Collection[MarkDecorator | Mark] = (), id: str | _HiddenParam | None = None, ) -> ParameterSet:+ """:meta private:""" if isinstance(marks, MarkDecorator): marks = (marks,) else:@@ -145,6 +150,8 @@ class ParameterSet(NamedTuple): ) -> ParameterSet: """Extract from an object or objects. + :meta private:+ :param parameterset: A legacy style parameterset that may or may not be a tuple, and may or may not be wrapped into a mess of mark objects.@@ -259,6 +266,10 @@ class ParameterSet(NamedTuple): return argnames, parameters +# mypy doesn't like `__module__` defined inline in the NamedTuple.+ParameterSet.__module__ = "pytest"++ @final @dataclasses.dataclass(frozen=True) class Mark:src/pytest/__init__.pyindex f6ea24f95..ecd075ff5 100644@@ -42,6 +42,7 @@ from _pytest.mark import MARK_GEN as mark from _pytest.mark import MarkDecorator from _pytest.mark import MarkGenerator from _pytest.mark import param+from _pytest.mark import ParameterSet from _pytest.monkeypatch import MonkeyPatch from _pytest.nodes import Collector from _pytest.nodes import Directory@@ -130,6 +131,7 @@ __all__ = [ "MonkeyPatch", "OptionGroup", "Package",+ "ParameterSet", "Parser", "PytestApproxDecimalToleranceWarning", "PytestAssertRewriteWarning",