Backtestspytestrun-20261106-234554#12223
junit-xml output attribute can be too big to handle with -vv
The harness couldn't finish this ticket.
The solver hit its turn or time cap.. No judgment was produced for this ticket. It counts as an error in the run total.
Solve
1h 30m
Wall-clock time in the container
Full test suite
Not run
The harness did not reach the test stage
Contamination audit
Clean
No sign the agent saw the real fix
Patch size
0vs 127 merged
Changed lines, ours vs merged
The issue
View on GitHubSuppose you have a test that may fail with a very large diff; say:
def test_xyz() -> None:
assert ["x"] * 10_000 == ["x\n"] * 10_000
With normal verbosity (0), the junit-xml output would look like this:
<?xml version="1.0" encoding="utf-8"?>
<testsuites>
<testsuite errors="0" failures="1" hostname="..." name="pytest" skipped="0" tests="1" time="1.172" timestamp="...">
<testcase classname="test_file" file="test_file.py" line="0" name="test_xyz" time="0.767">
<failure message="AssertionError: assert ['x', 'x', 'x...'x', 'x', ...] == ['x\n', 'x\n'...', 'x\n', ...]
At index 0 diff: 'x' != 'x\n'
Use -v to get more diff">
def test_xyz() -> None:
> assert ["x"] * 10_000 == ["x\n"] * 10_000
E AssertionError: assert ['x', 'x', 'x...'x', 'x', ...] == ['x\n', 'x\n'...', 'x\n', ...]
E At index 0 diff: 'x' != 'x\n'
E Use -v to get more diff
test_file.py:2: AssertionError
</failure>
</testcase>
</testsuite>
</testsuites>
noting the <failure message='...'> tag's message and body both have a Use -v to get more diff.
If you do supply a -vv, then both the body of the <failure> tag, AND the content of the message attribute can be very large. The latter can be problematic for parsers, and projects like BeautifulSoup and lxml will fail to parse this correctly (truncating or crashing, respectively).
Maybe this is a limitation of the junit schema, but is there anything acceptable that can be done to reduce only the content of the message attribute? I think it's more expected and "fine" for the body of a tag to be really long, but an attribute less so.
applies to pytest 8.1.1
The two patches
No patch was produced.changelog/12223.bugfix.rstnew file mode 100644index 000000000..f2379c80b@@ -0,0 +1 @@+The ``message`` attribute of ``<failure>`` and ``<error>`` elements in ``--junitxml`` reports is now shortened to whole lines within 500 characters. When something is dropped, it ends with a note pointing to the element text, which still holds the full report. Previously, very verbose assertion output (e.g. with ``-vv``) produced attributes tens of thousands of characters long, which some XML parsers fail on.src/_pytest/junitxml.pyindex 06bb9fd5d..d33b81a74 100644@@ -69,6 +69,41 @@ def bin_xml_escape(arg: object) -> str: return re.sub(illegal_xml_re, repl, str(arg)) +# Budget for the ``message`` attribute of <failure> and <error> elements.+# The full report is always written as the element text.+MESSAGE_ATTR_BUDGET = 500+++def shorten_message(msg: str, budget: int = MESSAGE_ATTR_BUDGET) -> str:+ """Shorten ``msg`` for use in a ``message`` attribute.++ Whole lines are kept while they fit in ``budget`` characters; a first+ line that is longer than ``budget`` on its own is cut. If anything was+ dropped, a final line points readers to the element text, which holds+ the full report.+ """+ lines = msg.splitlines() or [""]+ kept: list[str] = []+ size = 0+ for line in lines:+ if kept and size + len(line) + 1 > budget:+ break+ kept.append(line)+ size += len(line) + 1+ shortened = "\n".join(kept)+ first_line_cut = len(shortened) > budget+ if first_line_cut:+ shortened = shortened[:budget] + "..."+ remaining = len(lines) - len(kept)+ if remaining:+ note = f"+{remaining} more line{'s' if remaining > 1 else ''}"+ elif first_line_cut:+ note = "truncated"+ else:+ return msg+ return f"{shortened}\n[{note}; full report in element text]"++ def merge_family(left, right) -> None: result = {} for kl, vl in left.items():@@ -213,7 +248,7 @@ class _NodeReporter: message = reprcrash.message else: message = str(report.longrepr)- message = bin_xml_escape(message)+ message = bin_xml_escape(shorten_message(message)) self._add_simple("failure", message, str(report.longrepr)) def append_collect_error(self, report: CollectReport) -> None:@@ -236,7 +271,9 @@ class _NodeReporter: msg = f'failed on teardown with "{reason}"' else: msg = f'failed on setup with "{reason}"'- self._add_simple("error", bin_xml_escape(msg), str(report.longrepr))+ self._add_simple(+ "error", bin_xml_escape(shorten_message(msg)), str(report.longrepr)+ ) def append_skipped(self, report: TestReport) -> None: if hasattr(report, "wasxfail"):testing/test_junitxml.pyindex 3b51495ac..c607d9f68 100644@@ -18,6 +18,7 @@ from _pytest.junitxml import _JunitFamily from _pytest.junitxml import _JunitLogging from _pytest.junitxml import bin_xml_escape from _pytest.junitxml import LogXML+from _pytest.junitxml import shorten_message from _pytest.monkeypatch import MonkeyPatch from _pytest.pytester import Pytester from _pytest.pytester import RunResult@@ -655,6 +656,49 @@ class TestPython: fnode = tnode.get_first_by_tag("failure") fnode.assert_attr(message="AssertionError: An error\nassert 0") + def test_failure_message_is_shortened(+ self, pytester: Pytester, run_and_parse: RunAndParse+ ) -> None:+ """Long failure messages are shortened in the attribute, but kept in+ full in the element text (#12223)."""+ pytester.makepyfile(+ """+ def test_fail():+ assert list(range(200)) == list(range(199)) + [999]+ """+ )+ _result, dom = run_and_parse("-vv")+ fnode = dom.get_first_by_tag("failure")+ message = fnode["message"]+ assert message.startswith("assert [0, 1, 2, ")+ assert message.endswith("; full report in element text]")+ assert len(message) < 1000+ assert "Full diff:" in fnode.text+ assert "999" in fnode.text++ def test_error_message_is_shortened(+ self, pytester: Pytester, run_and_parse: RunAndParse+ ) -> None:+ pytester.makepyfile(+ """+ import pytest++ @pytest.fixture+ def arg():+ raise ValueError("a" * 2000)++ def test_function(arg):+ pass+ """+ )+ _result, dom = run_and_parse()+ fnode = dom.get_first_by_tag("error")+ assert fnode["message"].startswith('failed on setup with "ValueError: aaa')+ assert fnode["message"].endswith(+ "...\n[truncated; full report in element text]"+ )+ assert "a" * 2000 in fnode.text+ @parametrize_families def test_failure_escape( self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: _JunitFamily@@ -1881,3 +1925,44 @@ def test_invalid_junit_option_value(pytester: Pytester, name: str, value: str) - result.stderr.fnmatch_lines( [f"*ERROR: *config option '{name}' expects one of *, got '{value}'"] )+++@pytest.mark.parametrize(+ "msg",+ [+ "",+ "assert 0",+ "AssertionError: Regex pattern did not match.\n"+ " Expected regex: 'foo'\n"+ " Actual message: 'bar'",+ "x" * 500,+ "trailing newline\n",+ ],+)+def test_shorten_message_keeps_short_messages(msg: str) -> None:+ assert shorten_message(msg) == msg+++def test_shorten_message_cuts_on_line_boundary() -> None:+ msg = "\n".join(f"line {i:3}" for i in range(100))+ assert shorten_message(msg, budget=20) == (+ "line 0\nline 1\n[+98 more lines; full report in element text]"+ )+++def test_shorten_message_cuts_single_long_line() -> None:+ assert shorten_message("x" * 30, budget=10) == (+ "xxxxxxxxxx...\n[truncated; full report in element text]"+ )+++def test_shorten_message_cuts_long_first_line_with_more_lines() -> None:+ assert shorten_message("x" * 30 + "\nsecond", budget=10) == (+ "xxxxxxxxxx...\n[+1 more line; full report in element text]"+ )+++def test_shorten_message_before_escaping() -> None:+ """Shortening happens on the raw text, so escapes are never split."""+ escaped = bin_xml_escape(shorten_message("\x1b" * 20, budget=5))+ assert escaped == "#x1B" * 5 + "...\n[truncated; full report in element text]"