Backtestsscrapyrun-20261025-211112#7010
scrapy settings --get does not account for add-ons
Our patch was judged better than the merged fix.
In this judgment, Patch A = our agent, Patch B = the merged fix.
Both patches fix the reported behaviour the same way at the core: in the settings command they call AddonManager.load_pre_crawler_settings, build a Crawler with the base Spider, and run crawler.addons.load_settings so both pre-crawler and update_settings changes are visible. All eight independent tests pass on both, so correctness on the issue's scenarios is equal. Differences: Patch A copies the settings before mutating them, keeping the command's self.settings untouched, which is slightly cleaner than B mutating self.settings in place. A also applies the same pre-crawler loading to list/edit/genspider, which share the exact root cause (commands that never build a CrawlerProcess miss add-on changes such as SPIDER_MODULES); this is beyond the literal ask but low risk and directly tested, so it does not count against A. B instead adds a new --spider feature with docs; that is also out of scope and neutral on its own, but it is more product-surface than fix. Tests tip the balance. A's new test class has a distinct name (TestSettingsCommandAddons) and covers get/getbool/getint/getfloat/getlist, pre-crawler settings, project and command-line priority. B names its new class TestSettingsCommand, which already exists earlier in tests/test_commands.py (visible in A's diff context, the base class ends right before TestCommandCrawlerProcess). Redefining it at module level shadows the original, so pytest would silently stop collecting the existing TestSettingsCommand tests, and ruff F811 would flag the redefinition. That is a concrete craft defect that quietly removes coverage. A's edit test has a win32 skip, which is a reasonable platform restriction and still runs on the POSIX hosts the issue's scenario would use (though I cannot verify its write_recording_editor helper exists, which slightly lowers confidence). On balance A is the better patch: equivalent fix, cleaner handling of settings, tests that extend rather than clobber existing coverage.
Four judges ruled on this ticket in parallel, and the verdict comes from counting their votes. Each saw the issue and the two patches, labelled A and B in random order (two of them saw ours as A), with issue links and version numbers removed. Where tests could check the fix, they also saw the results of tests a separate agent wrote for the issue, run on both patches. None was told which patch was ours. The reasoning above is one judge's, shown verbatim.
Solve
19m 44s
Wall-clock time in the container
Full test suite
No new failures
191 passed · 3 skipped · 194 total
Contamination audit
Clean
No sign the agent saw the real fix
Patch size
160vs 127 merged
Changed lines, ours vs merged
The issue
View on GitHubFor example, if you run the scrapy settings --get ZYTE_API_TRANSPARENT_MODE command on a project whose settings.py file contains ADDONS = {"scrapy_zyte_api.Addon": 100}, you do not get True, even though the add-on sets it.
The two patches
scrapy/commands/edit.pyindex 243c17c24..9e4668a83 100644@@ -7,6 +7,7 @@ import sys from pathlib import Path from typing import TYPE_CHECKING, Any, ClassVar +from scrapy.addons import AddonManager from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError from scrapy.spiderloader import get_spider_loader@@ -51,6 +52,7 @@ class Command(ScrapyCommand): raise UsageError assert self.settings is not None+ AddonManager.load_pre_crawler_settings(self.settings) editor = self.settings["EDITOR"] # pylint: disable=unsubscriptable-object spider_loader = get_spider_loader(self.settings) try:scrapy/commands/genspider.pyindex 2790310ba..ca978b000 100644@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, cast from urllib.parse import urlparse import scrapy+from scrapy.addons import AddonManager from scrapy.commands import ScrapyCommand from scrapy.commands.edit import _edit_file from scrapy.exceptions import UsageError@@ -92,6 +93,7 @@ class Command(ScrapyCommand): def run(self, args: list[str], opts: argparse.Namespace) -> None: assert self.settings is not None+ AddonManager.load_pre_crawler_settings(self.settings) if opts.list: self._list_templates() returnscrapy/commands/list.pyindex a037f5c9b..571602bb5 100644@@ -2,6 +2,7 @@ from __future__ import annotations from typing import TYPE_CHECKING, Any, ClassVar +from scrapy.addons import AddonManager from scrapy.commands import ScrapyCommand from scrapy.spiderloader import get_spider_loader @@ -19,5 +20,6 @@ class Command(ScrapyCommand): def run(self, args: list[str], opts: argparse.Namespace) -> None: assert self.settings is not None+ AddonManager.load_pre_crawler_settings(self.settings) spider_loader = get_spider_loader(self.settings) print("\n".join(sorted(spider_loader.list())))scrapy/commands/settings.pyindex 26a97ccbb..887f225f8 100644@@ -2,8 +2,11 @@ import argparse import json from typing import Any, ClassVar +from scrapy.addons import AddonManager from scrapy.commands import ScrapyCommand-from scrapy.settings import BaseSettings+from scrapy.crawler import Crawler+from scrapy.settings import BaseSettings, Settings+from scrapy.spiders import Spider class Command(ScrapyCommand):@@ -48,7 +51,7 @@ class Command(ScrapyCommand): def run(self, args: list[str], opts: argparse.Namespace) -> None: assert self.settings is not None- settings = self.settings+ settings = self._get_settings_with_addons(self.settings) if opts.get: s = settings.get(opts.get) if isinstance(s, BaseSettings):@@ -63,3 +66,14 @@ class Command(ScrapyCommand): print(settings.getfloat(opts.getfloat)) elif opts.getlist: print(settings.getlist(opts.getlist))++ @staticmethod+ def _get_settings_with_addons(settings: Settings) -> Settings:+ """Return a copy of *settings* with the changes of all enabled+ :ref:`add-ons <topics-addons>` applied, as they would be applied when+ running a spider."""+ settings = settings.copy()+ AddonManager.load_pre_crawler_settings(settings)+ crawler = Crawler(Spider, settings)+ crawler.addons.load_settings(crawler.settings)+ return crawler.settingstests/test_commands.pyindex bb5c20f18..f58845e1c 100644@@ -280,6 +280,142 @@ class TestSettingsCommand: assert not out +class TestSettingsCommandAddons(TestProjectBase):+ """Settings changed by add-ons enabled in the project settings are+ reported by the settings command."""++ @pytest.fixture(autouse=True)+ def create_files(self, proj_path: Path) -> None:+ self._append_settings(+ proj_path / self.project_name,+ """++class MyAddon:+ @classmethod+ def update_pre_crawler_settings(cls, settings):+ settings.set("ADDON_PRE_CRAWLER_SETTING", "pre", priority="addon")++ def update_settings(self, settings):+ settings.set("ADDON_BOOL_SETTING", True, priority="addon")+ settings.set("ADDON_INT_SETTING", 42, priority="addon")+ settings.set("ADDON_FLOAT_SETTING", 1.5, priority="addon")+ settings.set("ADDON_LIST_SETTING", ["a", "b"], priority="addon")+ settings.set("ADDON_DICT_SETTING", {"key": "value"}, priority="addon")+ settings.set("BOT_NAME", "addonbot", priority="addon")+++ADDONS = {f"{__name__}.MyAddon": 100}+""",+ )++ @pytest.mark.parametrize(+ ("option", "setting", "expected"),+ [+ ("--get", "ADDON_BOOL_SETTING", "True"),+ ("--get", "ADDON_PRE_CRAWLER_SETTING", "pre"),+ ("--get", "ADDON_DICT_SETTING", "{'key': 'value'}"),+ ("--getbool", "ADDON_BOOL_SETTING", "True"),+ ("--getint", "ADDON_INT_SETTING", "42"),+ ("--getfloat", "ADDON_FLOAT_SETTING", "1.5"),+ ("--getlist", "ADDON_LIST_SETTING", "['a', 'b']"),+ ],+ )+ def test_get(+ self, proj_path: Path, option: str, setting: str, expected: str+ ) -> None:+ returncode, out, err = proc("settings", option, setting, cwd=proj_path)+ assert returncode == 0, err+ assert out.strip() == expected++ def test_project_priority_wins(self, proj_path: Path) -> None:+ """BOT_NAME is set in the project settings, which have a higher+ priority than add-ons."""+ returncode, out, err = proc("settings", "--get", "BOT_NAME", cwd=proj_path)+ assert returncode == 0, err+ assert out.strip() == self.project_name++ def test_cmdline_priority_wins(self, proj_path: Path) -> None:+ returncode, out, err = proc(+ "settings",+ "--getint",+ "ADDON_INT_SETTING",+ "-s",+ "ADDON_INT_SETTING=7",+ cwd=proj_path,+ )+ assert returncode == 0, err+ assert out.strip() == "7"+++class TestSpiderModulesAddon(TestProjectBase):+ """Spider modules added by add-ons enabled in the project settings are+ used by commands that load spiders without a crawler process."""++ @pytest.fixture(autouse=True)+ def create_files(self, proj_path: Path) -> None:+ proj_mod_path = proj_path / self.project_name+ addon_spiders = proj_mod_path / "addon_spiders"+ addon_spiders.mkdir()+ (addon_spiders / "__init__.py").write_text(+ """+import scrapy+++class AddonSpider(scrapy.Spider):+ name = "addon_spider"+""",+ encoding="utf-8",+ )+ self._append_settings(+ proj_mod_path,+ f"""++class MyAddon:+ @classmethod+ def update_pre_crawler_settings(cls, settings):+ settings.set(+ "SPIDER_MODULES",+ [*settings.getlist("SPIDER_MODULES"), "{self.project_name}.addon_spiders"],+ priority=settings.getpriority("SPIDER_MODULES"),+ )+++ADDONS = {{f"{{__name__}}.MyAddon": 100}}+""",+ )++ def test_list(self, proj_path: Path) -> None:+ returncode, out, err = proc("list", cwd=proj_path)+ assert returncode == 0, err+ assert out.strip() == "addon_spider"++ @pytest.mark.skipif(+ sys.platform == "win32", reason="requires a POSIX shell editor script"+ )+ def test_edit(self, proj_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:+ edited = proj_path / "edited.txt"+ editor = proj_path / "fake-editor.sh"+ write_recording_editor(editor)+ monkeypatch.setenv("EDITOR", f"{editor} {edited}")++ returncode, _, err = proc("edit", "addon_spider", cwd=proj_path)++ assert returncode == 0, err+ assert (proj_path / edited.read_text(encoding="utf-8")).resolve() == (+ proj_path / self.project_name / "addon_spiders" / "__init__.py"+ ).resolve()++ def test_genspider_exists(self, proj_path: Path) -> None:+ returncode, out, err = proc(+ "genspider", "addon_spider", "example.com", cwd=proj_path+ )+ assert returncode == 0, err+ assert "Spider 'addon_spider' already exists in module" in out+ assert not (+ proj_path / self.project_name / "spiders" / "addon_spider.py"+ ).exists()++ class TestCommandCrawlerProcess(TestProjectBase): """Test that the command uses the expected kind of *CrawlerProcess and produces expected errors when needed."""
docs/topics/commands.rstindex 2294f2d23..71a1d003d 100644@@ -630,6 +630,17 @@ Example usage:: $ scrapy settings --get DOWNLOAD_DELAY 0 +The settings of enabled :ref:`add-ons <topics-addons>` are taken into account.+:ref:`Spider settings <spider-settings>` are only taken into account if you name+a spider with ``--spider``::++ $ scrapy settings --get DOWNLOAD_DELAY --spider myspider+ 2.0++.. versionchanged:: VERSION+ Add-on settings are now taken into account, and the ``--spider`` option was+ added.+ .. command:: runspider runspiderscrapy/commands/settings.pyindex 26a97ccbb..edd697965 100644@@ -1,9 +1,20 @@-import argparse+from __future__ import annotations+ import json-from typing import Any, ClassVar+from typing import TYPE_CHECKING, Any, ClassVar +from scrapy import Spider+from scrapy.addons import AddonManager from scrapy.commands import ScrapyCommand+from scrapy.crawler import Crawler+from scrapy.exceptions import UsageError from scrapy.settings import BaseSettings+from scrapy.spiderloader import get_spider_loader++if TYPE_CHECKING:+ import argparse++ from scrapy.settings import Settings class Command(ScrapyCommand):@@ -45,10 +56,33 @@ class Command(ScrapyCommand): metavar="SETTING", help="print setting value, interpreted as a list", )+ parser.add_argument(+ "--spider",+ dest="spider",+ metavar="SPIDER",+ help="also apply the settings of this spider",+ ) - def run(self, args: list[str], opts: argparse.Namespace) -> None:+ def _build_settings(self, spider_name: str | None) -> Settings:+ """Return the settings as they would be during a crawl.""" assert self.settings is not None- settings = self.settings+ # Must run before get_spider_loader(), because add-ons may change+ # SPIDER_MODULES and other pre-crawler settings that it relies on.+ AddonManager.load_pre_crawler_settings(self.settings)+ spidercls: type[Spider] = Spider+ if spider_name is not None:+ try:+ spidercls = get_spider_loader(self.settings).load(spider_name)+ except KeyError:+ raise UsageError(f"Unable to find spider: {spider_name}") from None+ # Building a Crawler applies the spider settings and does not install a+ # reactor or build any component.+ crawler = Crawler(spidercls, self.settings)+ crawler.addons.load_settings(crawler.settings)+ return crawler.settings++ def run(self, args: list[str], opts: argparse.Namespace) -> None:+ settings = self._build_settings(opts.spider) if opts.get: s = settings.get(opts.get) if isinstance(s, BaseSettings):tests/test_commands.pyindex bb5c20f18..4eefa058f 100644@@ -773,6 +773,80 @@ class TestExecute: assert capsys.readouterr().out.strip() == "scrapybot" +class TestSettingsCommand(TestProjectBase):+ @pytest.fixture(autouse=True)+ def create_files(self, proj_path: Path) -> None:+ proj_mod_path = proj_path / self.project_name+ (proj_mod_path / "addons.py").write_text("""+class MyAddon:+ def update_settings(self, settings):+ settings.set("FROM_ADDON", "addon", priority="addon")+ settings.set("SPIDER_WINS", "addon", priority="addon")+ settings.set("SEEN_BY_ADDON", settings.get("FROM_SPIDER"), priority="addon")++ @classmethod+ def update_pre_crawler_settings(cls, settings):+ settings.set("FROM_ADDON_PRE_CRAWLER", "addon", priority="addon")+""")+ (proj_mod_path / "spiders" / "sp.py").write_text("""+import scrapy++class MySpider(scrapy.Spider):+ name = "sp"+ custom_settings = {+ "FROM_SPIDER": "spider",+ "SPIDER_WINS": "spider",+ }+""")+ self._append_settings(+ proj_mod_path,+ f'ADDONS = {{"{self.project_name}.addons.MyAddon": 100}}\n',+ )++ @staticmethod+ def _get(proj_path: Path, setting: str, *args: str) -> str:+ returncode, out, err = proc("settings", "--get", setting, *args, cwd=proj_path)+ assert returncode == 0, err+ return out.strip()++ def test_project_setting(self, proj_path: Path) -> None:+ assert self._get(proj_path, "BOT_NAME") == self.project_name++ def test_addon(self, proj_path: Path) -> None:+ assert self._get(proj_path, "FROM_ADDON") == "addon"++ def test_addon_pre_crawler(self, proj_path: Path) -> None:+ assert self._get(proj_path, "FROM_ADDON_PRE_CRAWLER") == "addon"++ def test_cmdline_beats_addon(self, proj_path: Path) -> None:+ assert (+ self._get(proj_path, "FROM_ADDON", "-s", "FROM_ADDON=cmdline") == "cmdline"+ )++ def test_no_spider(self, proj_path: Path) -> None:+ assert self._get(proj_path, "FROM_SPIDER") == "None"+ assert self._get(proj_path, "SPIDER_WINS") == "addon"++ def test_spider(self, proj_path: Path) -> None:+ assert self._get(proj_path, "FROM_SPIDER", "--spider", "sp") == "spider"+ assert self._get(proj_path, "FROM_ADDON", "--spider", "sp") == "addon"++ def test_spider_beats_addon(self, proj_path: Path) -> None:+ assert self._get(proj_path, "SPIDER_WINS", "--spider", "sp") == "spider"++ def test_addons_see_spider_settings(self, proj_path: Path) -> None:+ """Add-ons are loaded after spider settings are applied, as in a crawl."""+ assert self._get(proj_path, "SEEN_BY_ADDON") == "None"+ assert self._get(proj_path, "SEEN_BY_ADDON", "--spider", "sp") == "spider"++ def test_unknown_spider(self, proj_path: Path) -> None:+ returncode, _, err = proc(+ "settings", "--get", "FROM_SPIDER", "--spider", "nope", cwd=proj_path+ )+ assert returncode == 2+ assert "Unable to find spider: nope" in err++ class TestBenchCommand: @pytest.mark.parametrize("use_reactor", [True, False]) def test_run(self, use_reactor: bool) -> None: