From 24fe00392339f97cec63a0c66adc4c67c6b265e8 Mon Sep 17 00:00:00 2001 From: "quantconnect-kronos[bot]" <346399058+quantconnect-kronos[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:53:52 +0000 Subject: [PATCH 1/7] bug: ship the modules json as a placeholder so uninstall removes it The CLI downloads modules-.json into its package folder. pip did not install that file, so uninstalling left it and the lean folder behind, where Python picks it up as a namespace package that shadows a later editable install. setup.py now writes an empty placeholder and ships it as package data, so pip records the file and removes it on uninstall. lean.models treats an empty file as not downloaded. Closes #703 Co-Authored-By: Claude Opus 5.5 --- lean/models/__init__.py | 6 +++--- setup.py | 8 +++++++- tests/models/test_modules.py | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/lean/models/__init__.py b/lean/models/__init__.py index 115e20e57..e16ed86f3 100644 --- a/lean/models/__init__.py +++ b/lean/models/__init__.py @@ -40,8 +40,8 @@ def _download_modules_json() -> None: error = None try: - # fetch if file not available or fetched before 1 day - if not path.exists(file_path) or (time() - path.getmtime(file_path) > 86400): + # fetch if file not available, still the empty placeholder the package ships, or fetched before 1 day + if not path.exists(file_path) or path.getsize(file_path) == 0 or (time() - path.getmtime(file_path) > 86400): _download_modules_json() except Exception as e: # No need to do anything if file isn't available @@ -49,7 +49,7 @@ def _download_modules_json() -> None: pass # check if file exists -if not Path(file_path).is_file(): +if not Path(file_path).is_file() or path.getsize(file_path) == 0: error_message = f": {error}" if error is not None else "" raise FileNotFoundError( f"Modules json not found in the given path {file_path}{error_message}") diff --git a/setup.py b/setup.py index 4d188a974..df60c0d81 100644 --- a/setup.py +++ b/setup.py @@ -44,6 +44,12 @@ def get_stubs_version_range() -> str: return "" +# An empty placeholder for the modules json the CLI downloads into its package makes pip record it, so uninstalling +# removes it. The name is read rather than imported, as importing lean.models downloads the file +modules_file_name = re.search(r"^file_name = \"([^\"]+)\"", read("lean/models/__init__.py"), re.M).group(1) +with open(os.path.join(current_dir, "lean", modules_file_name), "w"): + pass + # Production dependencies install_requires = [ "click>=8.0.4", @@ -73,7 +79,7 @@ def get_stubs_version_range() -> str: long_description_content_type="text/markdown", packages=find_packages(include=["lean", "lean.*"]), package_data={ - "lean": ["ssh/*"] + "lean": ["ssh/*", "modules-*.json"] }, entry_points={ "console_scripts": ["lean=lean.main:main"] diff --git a/tests/models/test_modules.py b/tests/models/test_modules.py index 642042d29..2a4eb15db 100644 --- a/tests/models/test_modules.py +++ b/tests/models/test_modules.py @@ -11,9 +11,44 @@ # See the License for the specific language governing permissions and # limitations under the License. +import runpy +from pathlib import Path + +import pytest +from pyfakefs.fake_filesystem import FakeFilesystem +from requests.exceptions import ConnectionError as RequestsConnectionError +from responses import RequestsMock + +import lean.models from lean.models.modules import NuGetPackage +def _write_modules_json_placeholder(fs: FakeFilesystem) -> None: + fs.add_real_file(lean.models.__file__) + fs.create_file(lean.models.file_path) + + +def test_modules_json_placeholder_is_replaced_by_the_downloaded_file(fs: FakeFilesystem, + requests_mock: RequestsMock) -> None: + _write_modules_json_placeholder(fs) + modules = {"SomeModule": {"version": "1.0"}} + requests_mock.add(requests_mock.GET, lean.models.url, json={"modules": modules}) + + module_globals = runpy.run_path(lean.models.__file__) + + assert module_globals["json_modules"] == modules + assert Path(lean.models.file_path).stat().st_size > 0 + + +def test_modules_json_placeholder_raises_not_found_when_download_fails(fs: FakeFilesystem, + requests_mock: RequestsMock) -> None: + _write_modules_json_placeholder(fs) + requests_mock.add(requests_mock.GET, lean.models.url, body=RequestsConnectionError()) + + with pytest.raises(FileNotFoundError, match="Modules json not found"): + runpy.run_path(lean.models.__file__) + + def test_get_file_name_returns_correct_value() -> None: package = NuGetPackage(name="QuantConnect.Lean", version="2.5.11940") From df7dbbe46671a19970dba4b121f92d4cfeff48ee Mon Sep 17 00:00:00 2001 From: "quantconnect-kronos[bot]" <346399058+quantconnect-kronos[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 22:02:53 +0000 Subject: [PATCH 2/7] bug: ship only the current modules json placeholder Co-Authored-By: Claude Opus 5.5 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index df60c0d81..8c88239aa 100644 --- a/setup.py +++ b/setup.py @@ -79,7 +79,7 @@ def get_stubs_version_range() -> str: long_description_content_type="text/markdown", packages=find_packages(include=["lean", "lean.*"]), package_data={ - "lean": ["ssh/*", "modules-*.json"] + "lean": ["ssh/*", modules_file_name] }, entry_points={ "console_scripts": ["lean=lean.main:main"] From 0170c9ec1aebf86723af02bea97494083aa42046 Mon Sep 17 00:00:00 2001 From: "quantconnect-kronos[bot]" <346399058+quantconnect-kronos[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 22:48:19 +0000 Subject: [PATCH 3/7] bug: write the modules json placeholder to the build output only Writing the placeholder when setup.py loads emptied a checkout's downloaded modules json on any setup command, such as --version, leaving the CLI unable to start offline. build_py now writes it into the build output, so wheels, including those built from the sdist, still record an empty placeholder while the checkout keeps its file. Co-Authored-By: Claude Opus 5.5 --- setup.py | 27 ++++++++++++---- tests/test_setup.py | 78 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 6 deletions(-) create mode 100644 tests/test_setup.py diff --git a/setup.py b/setup.py index 8c88239aa..8e0501fd2 100644 --- a/setup.py +++ b/setup.py @@ -17,6 +17,7 @@ import urllib.request from setuptools import find_packages, setup +from setuptools.command.build_py import build_py current_dir = os.path.abspath(os.path.dirname(__file__)) @@ -44,11 +45,22 @@ def get_stubs_version_range() -> str: return "" -# An empty placeholder for the modules json the CLI downloads into its package makes pip record it, so uninstalling -# removes it. The name is read rather than imported, as importing lean.models downloads the file -modules_file_name = re.search(r"^file_name = \"([^\"]+)\"", read("lean/models/__init__.py"), re.M).group(1) -with open(os.path.join(current_dir, "lean", modules_file_name), "w"): - pass +class BuildPyWithModulesPlaceholder(build_py): + """Adds an empty placeholder for the modules json the CLI downloads into its package, so pip records the file and + uninstalling removes it. It is written to the build output only, keeping a checkout's downloaded file intact.""" + + def run(self) -> None: + super().run() + + # Editable builds link to the checkout, where the downloaded file has to stay + if getattr(self, "editable_mode", False): + return + + # The name is read rather than imported, as importing lean.models downloads the file + modules_file_name = re.search(r"^file_name = \"([^\"]+)\"", read("lean/models/__init__.py"), re.M).group(1) + with open(os.path.join(self.build_lib, "lean", modules_file_name), "w"): + pass + # Production dependencies install_requires = [ @@ -79,7 +91,10 @@ def get_stubs_version_range() -> str: long_description_content_type="text/markdown", packages=find_packages(include=["lean", "lean.*"]), package_data={ - "lean": ["ssh/*", modules_file_name] + "lean": ["ssh/*"] + }, + cmdclass={ + "build_py": BuildPyWithModulesPlaceholder }, entry_points={ "console_scripts": ["lean=lean.main:main"] diff --git a/tests/test_setup.py b/tests/test_setup.py new file mode 100644 index 000000000..d5d5991d8 --- /dev/null +++ b/tests/test_setup.py @@ -0,0 +1,78 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import shutil +import subprocess +import sys +import tarfile +import tempfile +import zipfile +from pathlib import Path + +import pytest +from pyfakefs.fake_filesystem import FakeFilesystem + +import lean.models + +REPOSITORY_ROOT = Path(__file__).parent.parent +MODULES_FILE = f"lean/{lean.models.file_name}" +DOWNLOADED_MODULES_JSON = '{"modules": {}}' + + +@pytest.fixture +def source_tree(fs: FakeFilesystem) -> Path: + """A minimal real copy of the package sources holding a downloaded modules json, as a checkout has.""" + # setup.py runs in a subprocess, which needs the real filesystem + fs.pause() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for relative_path in ["setup.py", "README.md", "lean/__init__.py", "lean/models/__init__.py"]: + (root / relative_path).parent.mkdir(parents=True, exist_ok=True) + shutil.copy(REPOSITORY_ROOT / relative_path, root / relative_path) + (root / MODULES_FILE).write_text(DOWNLOADED_MODULES_JSON) + yield root + fs.resume() + + +def _run_setup(directory: Path, *arguments: str) -> None: + subprocess.run([sys.executable, "setup.py", *arguments], cwd=directory, check=True, capture_output=True) + + +def _assert_wheel_has_empty_modules_placeholder(dist_directory: Path) -> None: + with zipfile.ZipFile(next(dist_directory.glob("*.whl"))) as wheel: + assert wheel.getinfo(MODULES_FILE).file_size == 0 + assert MODULES_FILE in wheel.read(next(name for name in wheel.namelist() if name.endswith("RECORD"))).decode() + + +def test_setup_metadata_keeps_downloaded_modules_json(source_tree: Path) -> None: + _run_setup(source_tree, "--version") + + assert (source_tree / MODULES_FILE).read_text() == DOWNLOADED_MODULES_JSON + + +def test_setup_wheel_records_empty_modules_placeholder_and_keeps_downloaded_file(source_tree: Path) -> None: + _run_setup(source_tree, "sdist", "bdist_wheel") + + _assert_wheel_has_empty_modules_placeholder(source_tree / "dist") + assert (source_tree / MODULES_FILE).read_text() == DOWNLOADED_MODULES_JSON + + +def test_setup_wheel_built_from_sdist_records_empty_modules_placeholder(source_tree: Path) -> None: + _run_setup(source_tree, "sdist") + with tarfile.open(next((source_tree / "dist").glob("*.tar.gz"))) as sdist: + sdist.extractall(source_tree / "extracted") + sdist_tree = next((source_tree / "extracted").iterdir()) + + _run_setup(sdist_tree, "bdist_wheel") + + _assert_wheel_has_empty_modules_placeholder(sdist_tree / "dist") From be6a506d7741a15686b360a7f54a6b53511b76f9 Mon Sep 17 00:00:00 2001 From: "quantconnect-kronos[bot]" <346399058+quantconnect-kronos[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 22:54:29 +0000 Subject: [PATCH 4/7] bug: add the modules json placeholder to the sdist release tree Co-Authored-By: Claude Opus 5.5 --- setup.py | 32 ++++++++++++++++++++++++-------- tests/test_setup.py | 10 +++++++++- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index 8e0501fd2..dc81a5417 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,7 @@ from setuptools import find_packages, setup from setuptools.command.build_py import build_py +from setuptools.command.sdist import sdist current_dir = os.path.abspath(os.path.dirname(__file__)) @@ -45,10 +46,21 @@ def get_stubs_version_range() -> str: return "" -class BuildPyWithModulesPlaceholder(build_py): - """Adds an empty placeholder for the modules json the CLI downloads into its package, so pip records the file and - uninstalling removes it. It is written to the build output only, keeping a checkout's downloaded file intact.""" +def write_modules_placeholder(package_root: str) -> None: + """Writes an empty placeholder for the modules json the CLI downloads into its package, so pip records the file and + uninstalling removes it. Only build outputs get it, keeping a checkout's downloaded file intact.""" + # The name is read rather than imported, as importing lean.models downloads the file + modules_file_name = re.search(r"^file_name = \"([^\"]+)\"", read("lean/models/__init__.py"), re.M).group(1) + placeholder_path = os.path.join(package_root, "lean", modules_file_name) + + # Build trees may hard link the checkout's files, so a copied file is replaced rather than truncated + if os.path.exists(placeholder_path): + os.remove(placeholder_path) + with open(placeholder_path, "w"): + pass + +class BuildPyWithModulesPlaceholder(build_py): def run(self) -> None: super().run() @@ -56,10 +68,13 @@ def run(self) -> None: if getattr(self, "editable_mode", False): return - # The name is read rather than imported, as importing lean.models downloads the file - modules_file_name = re.search(r"^file_name = \"([^\"]+)\"", read("lean/models/__init__.py"), re.M).group(1) - with open(os.path.join(self.build_lib, "lean", modules_file_name), "w"): - pass + write_modules_placeholder(self.build_lib) + + +class SdistWithModulesPlaceholder(sdist): + def make_release_tree(self, base_dir: str, files: list) -> None: + super().make_release_tree(base_dir, files) + write_modules_placeholder(base_dir) # Production dependencies @@ -94,7 +109,8 @@ def run(self) -> None: "lean": ["ssh/*"] }, cmdclass={ - "build_py": BuildPyWithModulesPlaceholder + "build_py": BuildPyWithModulesPlaceholder, + "sdist": SdistWithModulesPlaceholder }, entry_points={ "console_scripts": ["lean=lean.main:main"] diff --git a/tests/test_setup.py b/tests/test_setup.py index d5d5991d8..3deb240b8 100644 --- a/tests/test_setup.py +++ b/tests/test_setup.py @@ -61,12 +61,20 @@ def test_setup_metadata_keeps_downloaded_modules_json(source_tree: Path) -> None def test_setup_wheel_records_empty_modules_placeholder_and_keeps_downloaded_file(source_tree: Path) -> None: - _run_setup(source_tree, "sdist", "bdist_wheel") + _run_setup(source_tree, "bdist_wheel") _assert_wheel_has_empty_modules_placeholder(source_tree / "dist") assert (source_tree / MODULES_FILE).read_text() == DOWNLOADED_MODULES_JSON +def test_setup_sdist_has_empty_modules_placeholder_and_keeps_downloaded_file(source_tree: Path) -> None: + _run_setup(source_tree, "sdist") + + with tarfile.open(next((source_tree / "dist").glob("*.tar.gz"))) as sdist: + assert next(member for member in sdist.getmembers() if member.name.endswith(MODULES_FILE)).size == 0 + assert (source_tree / MODULES_FILE).read_text() == DOWNLOADED_MODULES_JSON + + def test_setup_wheel_built_from_sdist_records_empty_modules_placeholder(source_tree: Path) -> None: _run_setup(source_tree, "sdist") with tarfile.open(next((source_tree / "dist").glob("*.tar.gz"))) as sdist: From 4e7742b0043cea72d3b20c1eb3a8ad7cfcbdb920 Mon Sep 17 00:00:00 2001 From: "quantconnect-kronos[bot]" <346399058+quantconnect-kronos[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 23:59:42 +0000 Subject: [PATCH 5/7] bug: skip the modules json placeholder for dry runs and in-place builds Co-Authored-By: Claude Opus 5.5 --- setup.py | 12 ++++++++---- tests/test_setup.py | 9 +++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index dc81a5417..ba7c3de18 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ import re import urllib.request -from setuptools import find_packages, setup +from setuptools import Command, find_packages, setup from setuptools.command.build_py import build_py from setuptools.command.sdist import sdist @@ -46,9 +46,13 @@ def get_stubs_version_range() -> str: return "" -def write_modules_placeholder(package_root: str) -> None: +def write_modules_placeholder(command: Command, package_root: str) -> None: """Writes an empty placeholder for the modules json the CLI downloads into its package, so pip records the file and uninstalling removes it. Only build outputs get it, keeping a checkout's downloaded file intact.""" + # A dry run changes nothing, and an in-place build's package is the checkout itself + if command.dry_run or os.path.samefile(package_root, current_dir): + return + # The name is read rather than imported, as importing lean.models downloads the file modules_file_name = re.search(r"^file_name = \"([^\"]+)\"", read("lean/models/__init__.py"), re.M).group(1) placeholder_path = os.path.join(package_root, "lean", modules_file_name) @@ -68,13 +72,13 @@ def run(self) -> None: if getattr(self, "editable_mode", False): return - write_modules_placeholder(self.build_lib) + write_modules_placeholder(self, self.build_lib) class SdistWithModulesPlaceholder(sdist): def make_release_tree(self, base_dir: str, files: list) -> None: super().make_release_tree(base_dir, files) - write_modules_placeholder(base_dir) + write_modules_placeholder(self, base_dir) # Production dependencies diff --git a/tests/test_setup.py b/tests/test_setup.py index 3deb240b8..d3349a806 100644 --- a/tests/test_setup.py +++ b/tests/test_setup.py @@ -60,6 +60,15 @@ def test_setup_metadata_keeps_downloaded_modules_json(source_tree: Path) -> None assert (source_tree / MODULES_FILE).read_text() == DOWNLOADED_MODULES_JSON +@pytest.mark.parametrize("arguments", [["build_py", "--build-lib", "."], + ["--dry-run", "build_py", "--build-lib", "."], + ["--dry-run", "build_py", "--build-lib", "build/lib"]]) +def test_setup_in_place_and_dry_run_builds_keep_downloaded_modules_json(source_tree: Path, arguments: list) -> None: + _run_setup(source_tree, *arguments) + + assert (source_tree / MODULES_FILE).read_text() == DOWNLOADED_MODULES_JSON + + def test_setup_wheel_records_empty_modules_placeholder_and_keeps_downloaded_file(source_tree: Path) -> None: _run_setup(source_tree, "bdist_wheel") From 9968c406951a81f1e409b42156cc4faac13e0703 Mon Sep 17 00:00:00 2001 From: "quantconnect-kronos[bot]" <346399058+quantconnect-kronos[bot]@users.noreply.github.com> Date: Wed, 4 Nov 2026 00:08:41 +0000 Subject: [PATCH 6/7] test: skip the dry-run setup cases when setuptools has no --dry-run setuptools 84 removed the global --dry-run option. Co-Authored-By: Claude Opus 5.5 --- tests/test_setup.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/test_setup.py b/tests/test_setup.py index d3349a806..e40d050fc 100644 --- a/tests/test_setup.py +++ b/tests/test_setup.py @@ -21,12 +21,16 @@ import pytest from pyfakefs.fake_filesystem import FakeFilesystem +from setuptools.dist import Distribution import lean.models REPOSITORY_ROOT = Path(__file__).parent.parent MODULES_FILE = f"lean/{lean.models.file_name}" DOWNLOADED_MODULES_JSON = '{"modules": {}}' +# Newer setuptools releases removed the --dry-run option +requires_dry_run = pytest.mark.skipif("dry-run" not in [option[0] for option in Distribution.global_options], + reason="This setuptools version has no --dry-run option") @pytest.fixture @@ -60,9 +64,11 @@ def test_setup_metadata_keeps_downloaded_modules_json(source_tree: Path) -> None assert (source_tree / MODULES_FILE).read_text() == DOWNLOADED_MODULES_JSON -@pytest.mark.parametrize("arguments", [["build_py", "--build-lib", "."], - ["--dry-run", "build_py", "--build-lib", "."], - ["--dry-run", "build_py", "--build-lib", "build/lib"]]) +@pytest.mark.parametrize("arguments", [ + ["build_py", "--build-lib", "."], + pytest.param(["--dry-run", "build_py", "--build-lib", "."], marks=requires_dry_run), + pytest.param(["--dry-run", "build_py", "--build-lib", "build/lib"], marks=requires_dry_run) +]) def test_setup_in_place_and_dry_run_builds_keep_downloaded_modules_json(source_tree: Path, arguments: list) -> None: _run_setup(source_tree, *arguments) From 5fe501bb2c97b411b3ba4c0955b1ca420d4b3f96 Mon Sep 17 00:00:00 2001 From: "quantconnect-kronos[bot]" <346399058+quantconnect-kronos[bot]@users.noreply.github.com> Date: Wed, 4 Nov 2026 01:12:54 +0000 Subject: [PATCH 7/7] bug: compare the destination package with the checkout's package A build output whose lean folder links to the checkout's skipped the earlier guard, which compared only the build roots. Co-Authored-By: Claude Opus 5.5 --- setup.py | 7 ++++--- tests/test_setup.py | 10 ++++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index ba7c3de18..f94232091 100644 --- a/setup.py +++ b/setup.py @@ -49,13 +49,14 @@ def get_stubs_version_range() -> str: def write_modules_placeholder(command: Command, package_root: str) -> None: """Writes an empty placeholder for the modules json the CLI downloads into its package, so pip records the file and uninstalling removes it. Only build outputs get it, keeping a checkout's downloaded file intact.""" - # A dry run changes nothing, and an in-place build's package is the checkout itself - if command.dry_run or os.path.samefile(package_root, current_dir): + # A dry run changes nothing, and an in-place or linked build's package is the checkout's package itself + package_directory = os.path.join(package_root, "lean") + if command.dry_run or os.path.samefile(package_directory, os.path.join(current_dir, "lean")): return # The name is read rather than imported, as importing lean.models downloads the file modules_file_name = re.search(r"^file_name = \"([^\"]+)\"", read("lean/models/__init__.py"), re.M).group(1) - placeholder_path = os.path.join(package_root, "lean", modules_file_name) + placeholder_path = os.path.join(package_directory, modules_file_name) # Build trees may hard link the checkout's files, so a copied file is replaced rather than truncated if os.path.exists(placeholder_path): diff --git a/tests/test_setup.py b/tests/test_setup.py index e40d050fc..c9cc47054 100644 --- a/tests/test_setup.py +++ b/tests/test_setup.py @@ -75,6 +75,16 @@ def test_setup_in_place_and_dry_run_builds_keep_downloaded_modules_json(source_t assert (source_tree / MODULES_FILE).read_text() == DOWNLOADED_MODULES_JSON +@pytest.mark.skipif(sys.platform == "win32", reason="Creating symbolic links may require elevated rights on Windows") +def test_setup_build_into_linked_package_keeps_downloaded_modules_json(source_tree: Path) -> None: + (source_tree / "build" / "lib").mkdir(parents=True) + (source_tree / "build" / "lib" / "lean").symlink_to(source_tree / "lean", target_is_directory=True) + + _run_setup(source_tree, "build_py") + + assert (source_tree / MODULES_FILE).read_text() == DOWNLOADED_MODULES_JSON + + def test_setup_wheel_records_empty_modules_placeholder_and_keeps_downloaded_file(source_tree: Path) -> None: _run_setup(source_tree, "bdist_wheel")