Fix poisoned HF cache: fresh path + self-healing wipe

The /data/hf_cache tree was poisoned by partial downloads — xet and
huggingface_hub left files where directories belong, causing
[Errno 20] Not a directory on every generation attempt.

- Bump cache path to /data/hf_cache_v3 to escape the poisoned tree
- Set HF_HOME + HF_HUB_CACHE explicitly (no per-call cache_dir)
- Validate cache by checking for model_index.json in a snapshot
- If invalid, shutil.rmtree the whole subtree before re-downloading so
every attempt starts from clean ground (xet/.locks/blobs can create
real dirs)

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

image_generator.py

@@ -35,29 +35,42 @@ if IS_HF_SPACE:
 # ---------------------------------------------------------------------------
 # Persistent storage cache — survives sleep/restart on HF Spaces
 # ---------------------------------------------------------------------------
-if IS_HF_SPACE and os.path.isdir("/data"):
-    _cache_dir = "/data/hf_cache"
-    os.makedirs(_cache_dir, exist_ok=True)
-    os.environ.setdefault("HF_HOME", _cache_dir)
-    print(f"Persistent cache active → {_cache_dir}")
+# NOTE: a partially-written cache poisons the tree — huggingface_hub/xet leave
+# files where directories are expected ([Errno 20] Not a directory). We use a
+# fresh cache path and self-heal: if the model isn't *validly* cached, wipe the
+# whole subtree so every download attempt starts from clean ground.
+_MODEL_ID  = "black-forest-labs/FLUX.2-klein-4B"
+_MODEL_DIR = "models--black-forest-labs--FLUX.2-klein-4B"
 
-    # Clean up stale placeholder files left by previous failed downloads.
-    # huggingface_hub creates files where it expects directories — remove them.
+if IS_HF_SPACE and os.path.isdir("/data"):
     import shutil
-    _model_cache = os.path.join(_cache_dir, "models--black-forest-labs--FLUX.2-klein-4B")
-    _locks_path  = os.path.join(_cache_dir, ".locks", "models--black-forest-labs--FLUX.2-klein-4B")
-    for _stale in [_model_cache, _locks_path]:
-        try:
-            if os.path.isfile(_stale):
-                os.remove(_stale)
-                print(f"Removed stale placeholder file: {_stale}")
-        except Exception as _e:
-            print(f"⚠️  Could not remove {_stale}: {_e}")
-
-    if os.path.isdir(_model_cache):
-        print(f"✅ FLUX.2-klein-4B already cached — skipping download")
+    _cache_dir = "/data/hf_cache_v3"   # bump path to escape any poisoned older cache
+    os.environ["HF_HOME"]      = _cache_dir
+    os.environ["HF_HUB_CACHE"] = os.path.join(_cache_dir, "hub")
+
+    # A model is "validly cached" only if its snapshot has model_index.json.
+    def _is_valid_cache():
+        hub = os.path.join(_cache_dir, "hub", _MODEL_DIR, "snapshots")
+        if not os.path.isdir(hub):
+            return False
+        for snap in os.listdir(hub):
+            if os.path.isfile(os.path.join(hub, snap, "model_index.json")):
+                return True
+        return False
+
+    if _is_valid_cache():
+        print(f"✅ FLUX.2-klein-4B already cached at {_cache_dir} — skipping download")
     else:
-        print(f"Downloading FLUX.2-klein-4B to persistent cache…")
+        # Wipe any partial/poisoned state, then recreate clean.
+        if os.path.exists(_cache_dir):
+            print(f"Cache incomplete — wiping {_cache_dir} for a clean download")
+            try:
+                shutil.rmtree(_cache_dir)
+            except Exception as _e:
+                print(f"⚠️  Could not wipe cache: {_e}")
+        os.makedirs(os.path.join(_cache_dir, "hub"), exist_ok=True)
+
+        print(f"Downloading FLUX.2-klein-4B to {_cache_dir}…")
         try:
             from huggingface_hub import snapshot_download
             _hf_token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN")
@@ -67,8 +80,7 @@ if IS_HF_SPACE and os.path.isdir("/data"):
                 asyncio.set_event_loop(loop)
                 try:
                     snapshot_download(
-                        "black-forest-labs/FLUX.2-klein-4B",
-                        cache_dir=_cache_dir,
+                        _MODEL_ID,
                         token=_hf_token,
                         ignore_patterns=["*.msgpack", "*.h5", "flax_model*"],
                     )