Backtestspytestrun-20261106-234554#6997
xfail reports incorrect number of testcases if there is exception in fixture teardown
Our patch was judged better than the merged fix.
In this judgment, Patch B = our agent, Patch A = the merged fix.
The issue is that one collected item produces two xfailed outcomes because the xfail mark converts the teardown failure into a second xfail report. Patch B addresses the root cause in skipping.py by excluding the teardown phase from xfail conversion, so the teardown exception becomes a regular error (1 xfailed, 1 error), matching the behaviour the issue cites for the non-xfail case. The independent tests all go from FAIL-ASSERT on base to PASS on B. Patch A only changes the terminal summary line to say 'at teardown of', leaving the count at 2 xfailed and the short summary still listing the test twice; every independent test still fails on A, so it masks the symptom's wording without fixing it. B's tests are substantive: they assert counts, summary lines, exit code, strict-xfail interaction, and junitxml output, and would fail before the change. B also updates an existing test whose expectation encoded the old double-counting; that behaviour change is deliberate and documented in a code comment. A's tests merely lock in the cosmetic phase label. B is the clearly better fix on correctness, tests, and scope.
Four judges ruled on this ticket in parallel, and the verdict comes from counting their votes. Each saw the issue and the two patches, labelled A and B in random order (two of them saw ours as A), with issue links and version numbers removed. Where tests could check the fix, they also saw the results of tests a separate agent wrote for the issue, run on both patches. None was told which patch was ours. The reasoning above is one judge's, shown verbatim.
Solve
3m 36s
Wall-clock time in the container
Full test suite
No new failures
377 passed · 1 skipped · 378 total
Contamination audit
Clean
No sign the agent saw the real fix
Patch size
84vs 60 merged
Changed lines, ours vs merged
The issue
View on GitHubI have a simple testcase which is expected to fail (xfail)
@pytest.fixture
def my_fix():
yield
raise Exception()
class TestMyFix:
@pytest.mark.xfail(reason="Some reason")
def test_1(self, my_fix):
raise Exception()
which is deliberately raising Exception and the fixture is raising it too
The test execution summary reports 2 testcases even when it collects 1 item
# pytest -ras test_fixture.py
...
collected 1 item
test_fixture.py xx [100%]
XFAIL test_fixture.py::TestMyFix::test_1
Some reason
XFAIL test_fixture.py::TestMyFix::test_1
Some reason
============ 2 xfailed in 0.10 seconds ==========
Without xfail it correctly report 1 error in teardown=============== 1 failed, 1 error in 0.10 seconds ===================
The two patches
src/_pytest/skipping.pyindex aa97e3b7d..b1477d099 100644@@ -285,7 +285,10 @@ def pytest_runtest_makereport( assert call.excinfo.value.msg is not None rep.wasxfail = call.excinfo.value.msg rep.outcome = "skipped"- elif not rep.skipped and xfailed:+ elif not rep.skipped and xfailed and call.when != "teardown":+ # The xfail mark covers setup and call only: an error during teardown+ # is reported as a regular error, otherwise the test would be counted+ # twice (once for call and once for teardown). if call.excinfo: raises = xfailed.raises if raises is None or (testing/test_skipping.pyindex 9335f6db5..69284c3dd 100644@@ -791,7 +791,84 @@ class TestXFailwithSetupTeardown: """ ) result = pytester.runpytest()- result.stdout.fnmatch_lines(["*1 xfail*"])+ # The test itself passes (xpass); the teardown failure is an error,+ # not a second xfail.+ result.assert_outcomes(xpassed=1, errors=1)++ def test_failing_teardown_and_call(self, pytester: Pytester) -> None:+ """A failing teardown of an xfailed test is an error, not a second xfail."""+ pytester.makepyfile(+ """+ import pytest++ @pytest.fixture+ def my_fix():+ yield+ raise Exception()++ class TestMyFix:+ @pytest.mark.xfail(reason="Some reason")+ def test_1(self, my_fix):+ raise Exception()+ """+ )+ result = pytester.runpytest("-rxE")+ result.assert_outcomes(xfailed=1, errors=1)+ result.stdout.fnmatch_lines(+ [+ "collected 1 item",+ "*ERROR at teardown of TestMyFix.test_1*",+ "XFAIL *::TestMyFix::test_1 - Some reason",+ "ERROR *::TestMyFix::test_1 - Exception",+ "*= 1 xfailed, 1 error in *",+ ]+ )+ result.stdout.no_fnmatch_line("*2 xfailed*")+ assert result.ret == 1++ def test_failing_teardown_strict_xpass(self, pytester: Pytester) -> None:+ """A failing teardown is not converted by a strict xfail mark either."""+ pytester.makepyfile(+ """+ import pytest++ @pytest.fixture+ def my_fix():+ yield+ raise Exception()++ @pytest.mark.xfail(strict=True)+ def test_func(my_fix):+ pass+ """+ )+ result = pytester.runpytest()+ # The call is a strict xpass (failed); the teardown failure is an error.+ result.assert_outcomes(failed=1, errors=1)+ result.stdout.no_fnmatch_line("*xfailed*")++ def test_failing_teardown_junitxml(self, pytester: Pytester) -> None:+ """The junitxml report records the teardown failure as an error."""+ pytester.makepyfile(+ """+ import pytest++ @pytest.fixture+ def my_fix():+ yield+ raise Exception()++ @pytest.mark.xfail+ def test_func(my_fix):+ raise Exception()+ """+ )+ result = pytester.runpytest("--junitxml=junit.xml")+ result.assert_outcomes(xfailed=1, errors=1)+ xml = pytester.path.joinpath("junit.xml").read_text(encoding="utf-8")+ assert 'errors="1"' in xml+ assert 'skipped="1"' in xml+ assert 'tests="1"' in xml class TestSkip:
AUTHORSindex 47e310c23..2805077f4 100644@@ -422,6 +422,7 @@ Roland Puntaier Romain Dorgueil Roman Bolshakov Ronny Pfannschmidt+Roqia Salah Ross Lawley Ruaridh Williamson Russel Winderchangelog/6997.bugfix.rstnew file mode 100644index 000000000..2364d541f@@ -0,0 +1,2 @@+XFAIL reports from teardown phases now include the phase in the short test+summary.src/_pytest/terminal.pyindex 852153b92..e9aa36e94 100644@@ -1328,7 +1328,12 @@ class TerminalReporter: ) markup_word = self._tw.markup(verbose_word, **verbose_markup) nodeid = _get_node_id_with_markup(self._tw, self.config, rep)- line = f"{markup_word} {nodeid}"++ if rep.when == "call":+ line = f"{markup_word} {nodeid}"+ else:+ line = f"{markup_word} at {rep.when} of {nodeid}"+ reason = rep.wasxfail if reason: line += " - " + str(reason)testing/test_skipping.pyindex 9335f6db5..fbe196915 100644@@ -793,6 +793,56 @@ class TestXFailwithSetupTeardown: result = pytester.runpytest() result.stdout.fnmatch_lines(["*1 xfail*"]) + def test_xfail_call_and_teardown_reports_show_phase(+ self, pytester: Pytester+ ) -> None:+ pytester.makepyfile(+ test_case="""+ import pytest++ @pytest.fixture+ def my_fix():+ yield+ raise Exception("teardown")++ @pytest.mark.xfail(reason="Some reason")+ def test_func(my_fix):+ raise Exception("call")+ """+ )++ result = pytester.runpytest("-rx")++ result.stdout.fnmatch_lines(+ [+ "*XFAIL*test_case.py::test_func*",+ "*XFAIL at teardown of*test_case.py::test_func*",+ ]+ )++ def test_xfail_setup_report_shows_phase(self, pytester: Pytester) -> None:+ pytester.makepyfile(+ test_case="""+ import pytest++ @pytest.fixture+ def my_fix():+ raise Exception("setup")++ @pytest.mark.xfail(reason="Some reason")+ def test_func(my_fix):+ pass+ """+ )++ result = pytester.runpytest("-rx")++ result.stdout.fnmatch_lines(+ [+ "*XFAIL at setup of*test_case.py::test_func*",+ ]+ )+ class TestSkip: def test_skip_class(self, pytester: Pytester) -> None: