Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
00579c1
bump version
wangxingjun778 Jul 20, 2026
31f224c
fix(download): forward progress_callbacks through HubApi.download_rep…
wangxingjun778 Jul 20, 2026
85436d9
merge main
wangxingjun778 Jul 20, 2026
306f145
fix(download): harden legacy cache auto-detection for pre-1.38 layouts
wangxingjun778 Jul 21, 2026
e4acbfe
ok Merge branch 'main' of github.com:modelscope/modelscope_hub into f…
wangxingjun778 Jul 21, 2026
07aec55
fix(packaging): rename console scripts to modelscope-hub/ms-hub to av…
wangxingjun778 Jul 21, 2026
47b866a
update cli: ms/modelscope -> ms-hub/modelscope-hub
wangxingjun778 Jul 21, 2026
84b6e64
ok Merge branch 'main' of github.com:modelscope/modelscope_hub into f…
wangxingjun778 Jul 21, 2026
dde1875
docs(readme): expand recent version news, fold older, group by type
wangxingjun778 Jul 21, 2026
d0ea9e6
ok Merge branch 'main' of github.com:modelscope/modelscope_hub into f…
wangxingjun778 Jul 22, 2026
3665978
fix revision pass
wangxingjun778 Jul 22, 2026
a9839e6
ok Merge branch 'main' of github.com:modelscope/modelscope_hub into f…
wangxingjun778 Jul 31, 2026
17244b2
bump version
wangxingjun778 Jul 31, 2026
91a5489
fix lint and NixOS UT
wangxingjun778 Jul 31, 2026
402f7c5
update readme
wangxingjun778 Jul 31, 2026
2375141
fix 3.10 citest
wangxingjun778 Jul 31, 2026
74a6357
ok Merge branch 'main' of github.com:modelscope/modelscope_hub into f…
wangxingjun778 Aug 1, 2026
2d082bd
fix(auth): stop misreporting login failures and revoking credentials
wangxingjun778 Aug 1, 2026
bb2eb63
ok Merge branch 'main' of github.com:modelscope/modelscope_hub into f…
wangxingjun778 Aug 1, 2026
ce5c7b8
update news and bump version
wangxingjun778 Aug 1, 2026
3facbce
ok Merge branch 'main' of github.com:modelscope/modelscope_hub into f…
wangxingjun778 Aug 3, 2026
d59c734
fix(api): recover full repo file list past the server's 3000-entry cap
wangxingjun778 Aug 4, 2026
8287606
ok Merge branch 'main' of github.com:modelscope/modelscope_hub into f…
wangxingjun778 Aug 6, 2026
89f2924
ok Merge branch 'main' of github.com:modelscope/modelscope_hub into f…
wangxingjun778 Aug 13, 2026
9ae5a5c
Move CLI script ownership to modelscope-hub
wangxingjun778 Aug 18, 2026
cc83c47
ok Merge branch 'main' of github.com:modelscope/modelscope_hub into f…
wangxingjun778 Aug 19, 2026
dfa1487
fix logout
wangxingjun778 Aug 24, 2026
836c3fe
ok Merge branch 'main' of github.com:modelscope/modelscope_hub into f…
wangxingjun778 Aug 24, 2026
2497d0f
fix lock file path
wangxingjun778 Aug 24, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 21 additions & 6 deletions src/modelscope_hub/_download.py
Original file line number Diff line number Diff line change
Expand Up @@ -904,13 +904,28 @@ def _lock_path(
cache_dir: Path | None = None,
file_path: str | None = None,
) -> Path:
"""Compute the lock file path for a given repo or file."""
"""Compute a fixed-length lock file path for a repo or file.

File systems commonly cap a single path component at 255 bytes. Older
versions embedded the full ``file_path`` in the lock basename, so a
perfectly valid remote filename could make the local lock filename too
long before the download even started. Keep the lock *key* semantics
(repo type + repo id + optional file path), but store only a stable
SHA-256 digest in the basename.
"""
base = cache_dir or self._config.cache_dir
safe_id = repo_id.replace("/", "___")
if file_path is not None:
safe_file = file_path.replace("/", "___").replace(".", "_")
return base / ".lock" / f"{repo_type}_{safe_id}_{safe_file}.lock"
return base / ".lock" / f"{repo_type}_{safe_id}.lock"
scope = "file" if file_path is not None else "repo"
key = "\0".join(
(
"modelscope-hub-download-lock-v2",
scope,
str(repo_type),
repo_id,
file_path or "",
)
)
digest = hashlib.sha256(key.encode("utf-8")).hexdigest()
return base / ".lock" / f"{scope}_{digest}.lock"

def _download_with_resume(
self,
Expand Down
113 changes: 113 additions & 0 deletions tests/test_download_lock_path.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
"""Regression tests for download lock filename safety.

Remote repositories can contain perfectly valid path components close to the
common 255-byte filesystem basename limit. The local lock file must therefore
never embed the raw remote path in its own basename; otherwise the lock fails
before the actual target file is even opened.
"""

from __future__ import annotations

from pathlib import Path

from modelscope_hub._download import DownloadManager, _optional_file_lock
from modelscope_hub.api import HubApi
from modelscope_hub.compat.file_download import dataset_file_download

_LONG_IMAGE_NAME = "a" * 215 + ".jpg"
_LONG_FILE_PATH = f"images/{_LONG_IMAGE_NAME}"


def _download_manager() -> DownloadManager:
return HubApi().downloader


class TestDownloadLockPath:
def test_file_lock_name_is_fixed_length_for_long_remote_paths(self, tmp_path):
dm = _download_manager()

lock_path = dm._lock_path(
"OmniDocBench/OmniDocBench",
"dataset",
cache_dir=tmp_path,
file_path=_LONG_FILE_PATH,
)

assert lock_path.parent == tmp_path / ".lock"
assert lock_path.name.startswith("file_")
assert lock_path.name.endswith(".lock")
assert len(lock_path.name.encode("utf-8")) < 255
assert _LONG_IMAGE_NAME not in lock_path.name

def test_repo_lock_name_is_fixed_length_for_long_repo_ids(self, tmp_path):
dm = _download_manager()
long_repo_id = f"{'owner' * 30}/{'repo' * 40}"

lock_path = dm._lock_path(long_repo_id, "model", cache_dir=tmp_path)

assert lock_path.name.startswith("repo_")
assert lock_path.name.endswith(".lock")
assert len(lock_path.name.encode("utf-8")) < 255
assert "owner" not in lock_path.name

def test_lock_key_is_deterministic_and_file_specific(self, tmp_path):
dm = _download_manager()

first = dm._lock_path("owner/repo", "model", cache_dir=tmp_path, file_path="config.json")
second = dm._lock_path("owner/repo", "model", cache_dir=tmp_path, file_path="config.json")
other_file = dm._lock_path("owner/repo", "model", cache_dir=tmp_path, file_path="tokenizer.json")
repo_lock = dm._lock_path("owner/repo", "model", cache_dir=tmp_path)

assert first == second
assert first != other_file
assert first != repo_lock

def test_optional_file_lock_can_acquire_hashed_long_path(self, tmp_path):
dm = _download_manager()
lock_path = dm._lock_path(
"OmniDocBench/OmniDocBench",
"dataset",
cache_dir=tmp_path,
file_path=_LONG_FILE_PATH,
)

with _optional_file_lock(lock_path):
assert lock_path.exists()


class TestCompatDownloadUsesSafeLockNames:
def test_dataset_file_download_accepts_long_file_path(self, tmp_path, monkeypatch):
"""The legacy compat entry point reaches the same safe lock path.

No network is needed: patch the low-level transfer after the lock has
been acquired, and write the expected target file directly.
"""

def fake_download_with_resume(
self,
repo_id: str,
repo_type: str,
file_path: str,
revision: str,
target: Path,
**kwargs,
) -> Path:
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(b"image-bytes")
return target

monkeypatch.setattr(DownloadManager, "_download_with_resume", fake_download_with_resume)

cache_dir = tmp_path / "cache"
result = dataset_file_download(
"OmniDocBench/OmniDocBench",
_LONG_FILE_PATH,
cache_dir=str(cache_dir),
local_dir=str(tmp_path / "local"),
endpoint="https://modelscope.cn",
)

assert Path(result).read_bytes() == b"image-bytes"
lock_files = list((cache_dir / ".lock").glob("*.lock"))
assert lock_files
assert all(len(path.name.encode("utf-8")) < 255 for path in lock_files)
Loading