mirror of
https://github.com/jamiepine/voicebox.git
synced 2026-10-03 17:15:19 -07:00
fix(cache): share the orphaned-.incomplete check with /models/status
This commit is contained in:
committed by
capy-ai-staging[bot]
parent
f62aff0809
commit
e86a2dcaa5
+19
-10
@@ -22,6 +22,24 @@ from ..utils.tasks import get_task_manager
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def has_in_progress_download(blobs_dir: Path) -> bool:
|
||||
"""
|
||||
Whether a HuggingFace repo's ``blobs`` dir holds a genuinely in-progress download.
|
||||
|
||||
An ``.incomplete`` blob means a download is still in progress -- unless a
|
||||
completed blob with the same hash already sits next to it, which happens
|
||||
when a retried/concurrent download leaves a stale ``.incomplete`` behind
|
||||
after the real transfer already finished. Only orphaned ``.incomplete``
|
||||
files (no matching completed blob) count as "in progress".
|
||||
"""
|
||||
if not blobs_dir.exists():
|
||||
return False
|
||||
return any(
|
||||
not incomplete.with_name(incomplete.name.removesuffix(".incomplete")).exists()
|
||||
for incomplete in blobs_dir.glob("*.incomplete")
|
||||
)
|
||||
|
||||
|
||||
def is_model_cached(
|
||||
hf_repo: str,
|
||||
*,
|
||||
@@ -48,16 +66,7 @@ def is_model_cached(
|
||||
if not repo_cache.exists():
|
||||
return False
|
||||
|
||||
# An .incomplete blob means a download is still in progress -- unless
|
||||
# a completed blob with the same hash already sits next to it, which
|
||||
# happens when a retried/concurrent download leaves a stale .incomplete
|
||||
# behind after the real transfer already finished. Only orphaned
|
||||
# .incomplete files (no matching completed blob) count as "in progress".
|
||||
blobs_dir = repo_cache / "blobs"
|
||||
if blobs_dir.exists():
|
||||
for incomplete in blobs_dir.glob("*.incomplete"):
|
||||
completed = incomplete.with_name(incomplete.name.removesuffix(".incomplete"))
|
||||
if not completed.exists():
|
||||
if has_in_progress_download(repo_cache / "blobs"):
|
||||
logger.debug(f"Found in-progress .incomplete file for {hf_repo}")
|
||||
return False
|
||||
|
||||
|
||||
@@ -244,6 +244,7 @@ async def get_model_status():
|
||||
use_scan_cache = False
|
||||
|
||||
from ..backends import get_all_model_configs, check_model_loaded
|
||||
from ..backends.base import has_in_progress_download
|
||||
|
||||
registry_configs = get_all_model_configs()
|
||||
model_configs = [
|
||||
@@ -293,8 +294,7 @@ async def get_model_status():
|
||||
try:
|
||||
cache_dir = hf_constants.HF_HUB_CACHE
|
||||
blobs_dir = Path(cache_dir) / ("models--" + repo_id.replace("/", "--")) / "blobs"
|
||||
if blobs_dir.exists():
|
||||
has_incomplete = any(blobs_dir.glob("*.incomplete"))
|
||||
has_incomplete = has_in_progress_download(blobs_dir)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -314,7 +314,7 @@ async def get_model_status():
|
||||
|
||||
if repo_cache.exists():
|
||||
blobs_dir = repo_cache / "blobs"
|
||||
has_incomplete = blobs_dir.exists() and any(blobs_dir.glob("*.incomplete"))
|
||||
has_incomplete = has_in_progress_download(blobs_dir)
|
||||
|
||||
if not has_incomplete:
|
||||
snapshots_dir = repo_cache / "snapshots"
|
||||
|
||||
@@ -20,10 +20,10 @@ from typing import Optional
|
||||
|
||||
_SOURCE = (Path(__file__).parent.parent / "backends" / "base.py").read_text()
|
||||
_MODULE = ast.parse(_SOURCE)
|
||||
_FUNC_SRC = next(
|
||||
_FUNC_SRC = "\n\n".join(
|
||||
ast.get_source_segment(_SOURCE, node)
|
||||
for node in _MODULE.body
|
||||
if isinstance(node, ast.FunctionDef) and node.name == "is_model_cached"
|
||||
if isinstance(node, ast.FunctionDef) and node.name in ("has_in_progress_download", "is_model_cached")
|
||||
)
|
||||
|
||||
_namespace = {
|
||||
|
||||
Reference in New Issue
Block a user