Retry embedding startup for DeepFashion demo

demo.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 
 import os
 import re
+import time
 from collections import Counter
 from pathlib import Path
 from typing import Any
@@ -37,6 +38,8 @@ ALLOW_CANDIDATE_FALLBACK = os.environ.get("HYPERVIEW_ALLOW_CANDIDATE_FALLBACK",
     "true",
     "yes",
 }
+EMBEDDING_MAX_ATTEMPTS = max(1, int(os.environ.get("HYPERVIEW_EMBEDDING_MAX_ATTEMPTS", "4")))
+EMBEDDING_RETRY_DELAY_SECONDS = float(os.environ.get("HYPERVIEW_EMBEDDING_RETRY_DELAY_SECONDS", "15"))
 RUNTIME_WARNINGS: list[str] = []
 
 MODEL_SPECS = [
@@ -252,18 +255,40 @@ def add_deepfashion_samples(dataset: hv.Dataset) -> None:
     print(f"Prepared DeepFashion samples ({added} added, {updated} updated).", flush=True)
 
 
-def ensure_layouts(dataset: hv.Dataset) -> dict[str, str]:
-    layouts: dict[str, str] = {}
-    for spec in MODEL_SPECS:
-        print(f"Ensuring {spec['display_name']} embeddings...", flush=True)
+def compute_embeddings_with_retry(dataset: hv.Dataset, spec: dict[str, Any]) -> str:
+    for attempt in range(1, EMBEDDING_MAX_ATTEMPTS + 1):
         try:
-            space_key = dataset.compute_embeddings(
+            return dataset.compute_embeddings(
                 model=spec["model"],
                 provider=spec["provider"],
                 batch_size=32,
                 show_progress=True,
             )
-        except Exception as exc:
+        except BaseException as exc:
+            if isinstance(exc, (KeyboardInterrupt, SystemExit)):
+                raise
+            if attempt >= EMBEDDING_MAX_ATTEMPTS:
+                raise
+            delay = EMBEDDING_RETRY_DELAY_SECONDS * attempt
+            print(
+                f"Embedding load failed for {spec['display_name']} "
+                f"({type(exc).__name__}: {exc}). Retrying in {delay:.0f}s "
+                f"({attempt + 1}/{EMBEDDING_MAX_ATTEMPTS})...",
+                flush=True,
+            )
+            time.sleep(delay)
+    raise RuntimeError(f"Failed to compute embeddings for {spec['display_name']}")
+
+
+def ensure_layouts(dataset: hv.Dataset) -> dict[str, str]:
+    layouts: dict[str, str] = {}
+    for spec in MODEL_SPECS:
+        print(f"Ensuring {spec['display_name']} embeddings...", flush=True)
+        try:
+            space_key = compute_embeddings_with_retry(dataset, spec)
+        except BaseException as exc:
+            if isinstance(exc, (KeyboardInterrupt, SystemExit)):
+                raise
             if spec["key"] == "candidate" and ALLOW_CANDIDATE_FALLBACK and "clip" in layouts:
                 warning = (
                     f"Hyper3-CLIP embeddings are unavailable ({type(exc).__name__}: {exc}). "