mirror of
https://github.com/jamiepine/voicebox.git
synced 2026-10-03 17:15:19 -07:00
fix(cache): share the orphaned-.incomplete check with /models/status
This commit is contained in:
committed by
capy-ai-staging[bot]
parent
f62aff0809
commit
e86a2dcaa5
+21
-12
@@ -22,6 +22,24 @@ from ..utils.tasks import get_task_manager
|
|||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def has_in_progress_download(blobs_dir: Path) -> bool:
|
||||||
|
"""
|
||||||
|
Whether a HuggingFace repo's ``blobs`` dir holds a genuinely in-progress download.
|
||||||
|
|
||||||
|
An ``.incomplete`` blob means a download is still in progress -- unless a
|
||||||
|
completed blob with the same hash already sits next to it, which happens
|
||||||
|
when a retried/concurrent download leaves a stale ``.incomplete`` behind
|
||||||
|
after the real transfer already finished. Only orphaned ``.incomplete``
|
||||||
|
files (no matching completed blob) count as "in progress".
|
||||||
|
"""
|
||||||
|
if not blobs_dir.exists():
|
||||||
|
return False
|
||||||
|
return any(
|
||||||
|
not incomplete.with_name(incomplete.name.removesuffix(".incomplete")).exists()
|
||||||
|
for incomplete in blobs_dir.glob("*.incomplete")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def is_model_cached(
|
def is_model_cached(
|
||||||
hf_repo: str,
|
hf_repo: str,
|
||||||
*,
|
*,
|
||||||
@@ -48,18 +66,9 @@ def is_model_cached(
|
|||||||
if not repo_cache.exists():
|
if not repo_cache.exists():
|
||||||
return False
|
return False
|
||||||
|
|
||||||
# An .incomplete blob means a download is still in progress -- unless
|
if has_in_progress_download(repo_cache / "blobs"):
|
||||||
# a completed blob with the same hash already sits next to it, which
|
logger.debug(f"Found in-progress .incomplete file for {hf_repo}")
|
||||||
# happens when a retried/concurrent download leaves a stale .incomplete
|
return False
|
||||||
# behind after the real transfer already finished. Only orphaned
|
|
||||||
# .incomplete files (no matching completed blob) count as "in progress".
|
|
||||||
blobs_dir = repo_cache / "blobs"
|
|
||||||
if blobs_dir.exists():
|
|
||||||
for incomplete in blobs_dir.glob("*.incomplete"):
|
|
||||||
completed = incomplete.with_name(incomplete.name.removesuffix(".incomplete"))
|
|
||||||
if not completed.exists():
|
|
||||||
logger.debug(f"Found in-progress .incomplete file for {hf_repo}")
|
|
||||||
return False
|
|
||||||
|
|
||||||
snapshots_dir = repo_cache / "snapshots"
|
snapshots_dir = repo_cache / "snapshots"
|
||||||
if not snapshots_dir.exists():
|
if not snapshots_dir.exists():
|
||||||
|
|||||||
@@ -244,6 +244,7 @@ async def get_model_status():
|
|||||||
use_scan_cache = False
|
use_scan_cache = False
|
||||||
|
|
||||||
from ..backends import get_all_model_configs, check_model_loaded
|
from ..backends import get_all_model_configs, check_model_loaded
|
||||||
|
from ..backends.base import has_in_progress_download
|
||||||
|
|
||||||
registry_configs = get_all_model_configs()
|
registry_configs = get_all_model_configs()
|
||||||
model_configs = [
|
model_configs = [
|
||||||
@@ -293,8 +294,7 @@ async def get_model_status():
|
|||||||
try:
|
try:
|
||||||
cache_dir = hf_constants.HF_HUB_CACHE
|
cache_dir = hf_constants.HF_HUB_CACHE
|
||||||
blobs_dir = Path(cache_dir) / ("models--" + repo_id.replace("/", "--")) / "blobs"
|
blobs_dir = Path(cache_dir) / ("models--" + repo_id.replace("/", "--")) / "blobs"
|
||||||
if blobs_dir.exists():
|
has_incomplete = has_in_progress_download(blobs_dir)
|
||||||
has_incomplete = any(blobs_dir.glob("*.incomplete"))
|
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@@ -314,7 +314,7 @@ async def get_model_status():
|
|||||||
|
|
||||||
if repo_cache.exists():
|
if repo_cache.exists():
|
||||||
blobs_dir = repo_cache / "blobs"
|
blobs_dir = repo_cache / "blobs"
|
||||||
has_incomplete = blobs_dir.exists() and any(blobs_dir.glob("*.incomplete"))
|
has_incomplete = has_in_progress_download(blobs_dir)
|
||||||
|
|
||||||
if not has_incomplete:
|
if not has_incomplete:
|
||||||
snapshots_dir = repo_cache / "snapshots"
|
snapshots_dir = repo_cache / "snapshots"
|
||||||
|
|||||||
@@ -20,10 +20,10 @@ from typing import Optional
|
|||||||
|
|
||||||
_SOURCE = (Path(__file__).parent.parent / "backends" / "base.py").read_text()
|
_SOURCE = (Path(__file__).parent.parent / "backends" / "base.py").read_text()
|
||||||
_MODULE = ast.parse(_SOURCE)
|
_MODULE = ast.parse(_SOURCE)
|
||||||
_FUNC_SRC = next(
|
_FUNC_SRC = "\n\n".join(
|
||||||
ast.get_source_segment(_SOURCE, node)
|
ast.get_source_segment(_SOURCE, node)
|
||||||
for node in _MODULE.body
|
for node in _MODULE.body
|
||||||
if isinstance(node, ast.FunctionDef) and node.name == "is_model_cached"
|
if isinstance(node, ast.FunctionDef) and node.name in ("has_in_progress_download", "is_model_cached")
|
||||||
)
|
)
|
||||||
|
|
||||||
_namespace = {
|
_namespace = {
|
||||||
|
|||||||
Reference in New Issue
Block a user