Add Hugging Face-safe crypto data fallbacks

backend/main.py

@@ -911,6 +911,73 @@ SYMBOLS: Dict[str, SymbolConfig] = {
     "DIA":   _s("DIA","Dow Jones ETF","DIA","ETF",{"yfinance":"DIA"}),
 }
 
+CRYPTO_QUOTE_SUFFIXES: Tuple[str, ...] = ("USDT", "USDC", "USD", "BTC", "ETH")
+HF_CRYPTO_SOURCE_PRIORITY_DAILY: List[str] = [
+    "yfinance",
+    "twelvedata",
+    "finnhub",
+    "coingecko",
+    "binance",
+    "bybit",
+]
+HF_CRYPTO_SOURCE_PRIORITY_INTRADAY: List[str] = [
+    "twelvedata",
+    "yfinance",
+    "finnhub",
+    "coingecko",
+    "binance",
+    "bybit",
+]
+
+
+def _split_crypto_symbol(symbol: str) -> Tuple[Optional[str], Optional[str]]:
+    upper_symbol = symbol.upper()
+    for quote in CRYPTO_QUOTE_SUFFIXES:
+        if upper_symbol.endswith(quote) and len(upper_symbol) > len(quote):
+            return upper_symbol[: -len(quote)], quote
+    return None, None
+
+
+def _get_dynamic_crypto_mappings(symbol: str) -> Dict[str, str]:
+    cfg = SYMBOLS[symbol]
+    if cfg.category != "Crypto":
+        return {}
+
+    base, quote = _split_crypto_symbol(cfg.symbol)
+    if not base or not quote:
+        return {}
+
+    derived: Dict[str, str] = {}
+    normalized_quote = "USD" if quote == "USDT" else quote
+    binance_symbol = cfg.mappings.get("binance")
+
+    if "twelvedata" not in cfg.mappings:
+        derived["twelvedata"] = f"{base}/{normalized_quote}"
+
+    if normalized_quote == "USD" and "yfinance" not in cfg.mappings:
+        derived["yfinance"] = f"{base}-USD"
+
+    if "finnhub" not in cfg.mappings:
+        if binance_symbol:
+            derived["finnhub"] = f"BINANCE:{binance_symbol}"
+        elif quote in {"USD", "USDT", "USDC"}:
+            finnhub_quote = "USDT" if quote == "USD" else quote
+            derived["finnhub"] = f"BINANCE:{base}{finnhub_quote}"
+
+    return derived
+
+
+def _get_symbol_mapping(symbol: str, source: str) -> Optional[str]:
+    cfg = SYMBOLS[symbol]
+    direct_mapping = cfg.mappings.get(source)
+    if direct_mapping:
+        return direct_mapping
+    return _get_dynamic_crypto_mappings(symbol).get(source)
+
+
+def _has_source_mapping(symbol: str, source: str) -> bool:
+    return _get_symbol_mapping(symbol, source) is not None
+
 
 # ──────────────────────────────────────────────────────────────────────────────
 # TTL Cache (unchanged from v3, with improved stats)
@@ -1052,9 +1119,11 @@ def _get_canonical_symbol(sym: str) -> str:
     if s.startswith("BINANCE:") and s[8:] in SYMBOLS:
         return s[8:]
     # Search in mappings
-    for reg_id, cfg in SYMBOLS.items():
-        if s == cfg.mappings.get("binance") or s == cfg.mappings.get("coingecko") or s == cfg.mappings.get("twelvedata"):
-            return reg_id
+    for reg_id in SYMBOLS:
+        for source_name in ("binance", "bybit", "coingecko", "twelvedata", "yfinance", "finnhub"):
+            mapping = _get_symbol_mapping(reg_id, source_name)
+            if mapping and s == mapping.upper():
+                return reg_id
     return s
 
 
@@ -1216,7 +1285,9 @@ def _normalize_ohlcv(records: List[Dict[str, Any]], interval: str = "1h") -> Lis
 async def fetch_binance(symbol: str, interval: str, limit: int) -> List[Dict[str, Any]]:
     await _rate_limit("binance")
     cfg = SYMBOLS[symbol]
-    endpoint_symbol = cfg.mappings["binance"]
+    endpoint_symbol = _get_symbol_mapping(symbol, "binance")
+    if not endpoint_symbol:
+        raise RuntimeError("No Binance mapping for this symbol")
     
     # Define request details
     endpoint = "/fapi/v1/klines" if cfg.binance_type == "futures" else "/api/v3/klines"
@@ -1276,7 +1347,9 @@ async def fetch_binance(symbol: str, interval: str, limit: int) -> List[Dict[str
 async def fetch_bybit(symbol: str, interval: str, limit: int) -> List[Dict[str, Any]]:
     """Bybit V5 kline endpoint — free, no API key."""
     await _rate_limit("bybit")
-    endpoint_symbol = SYMBOLS[symbol].mappings["bybit"]
+    endpoint_symbol = _get_symbol_mapping(symbol, "bybit")
+    if not endpoint_symbol:
+        raise RuntimeError("No Bybit mapping for this symbol")
     bybit_cat = SYMBOLS[symbol].bybit_category
     params = {
         "category": bybit_cat,
@@ -1360,7 +1433,7 @@ async def fetch_coingecko(symbol: str, interval: str, limit: int) -> List[Dict[s
       90+ days   → weekly candles
     """
     await _rate_limit("coingecko")
-    cg_id = SYMBOLS[symbol].coingecko_id
+    cg_id = _get_symbol_mapping(symbol, "coingecko") or SYMBOLS[symbol].coingecko_id
     if not cg_id:
         raise RuntimeError("No CoinGecko ID for this symbol")
 
@@ -1409,7 +1482,9 @@ async def fetch_coingecko(symbol: str, interval: str, limit: int) -> List[Dict[s
 
 async def fetch_twelvedata(symbol: str, interval: str, limit: int) -> List[Dict[str, Any]]:
     await _rate_limit("twelvedata")
-    endpoint_symbol = SYMBOLS[symbol].mappings["twelvedata"]
+    endpoint_symbol = _get_symbol_mapping(symbol, "twelvedata")
+    if not endpoint_symbol:
+        raise RuntimeError("No TwelveData mapping for this symbol")
     params = {
         "symbol":     endpoint_symbol,
         "interval":   TWELVE_INTERVAL_MAP[interval],
@@ -1459,8 +1534,8 @@ async def fetch_twelvedata(symbol: str, interval: str, limit: int) -> List[Dict[
 
 
 async def fetch_finnhub(symbol: str, interval: str, limit: int) -> List[Dict[str, Any]]:
-    mappings = SYMBOLS[symbol].mappings
-    if "finnhub" not in mappings:
+    finnhub_symbol = _get_symbol_mapping(symbol, "finnhub")
+    if not finnhub_symbol:
         return []
 
     cb = source_breakers.get("finnhub")
@@ -1474,7 +1549,7 @@ async def fetch_finnhub(symbol: str, interval: str, limit: int) -> List[Dict[str
         r = await client.get(
             "https://finnhub.io/api/v1/stock/candle",
             params={
-                "symbol":   mappings["finnhub"],
+                "symbol":   finnhub_symbol,
                 "resolution": FINNHUB_RESOLUTION_MAP.get(interval, "D"),
                 "count":    limit,
                 "token":    finnhub_pool.next_key(),
@@ -1512,7 +1587,9 @@ def _resample_4h(df: pd.DataFrame) -> pd.DataFrame:
 
 async def fetch_yfinance(symbol: str, interval: str, limit: int) -> List[Dict[str, Any]]:
     await _rate_limit("yfinance")
-    ticker      = SYMBOLS[symbol].mappings["yfinance"]
+    ticker = _get_symbol_mapping(symbol, "yfinance")
+    if not ticker:
+        raise RuntimeError("No yfinance mapping for this symbol")
     yf_interval = YF_INTERVAL_MAP[interval]
     period      = YF_PERIOD_MAP[interval]
     logger.info("[yfinance] %s %s", symbol, interval)
@@ -1942,10 +2019,16 @@ async def _build_synthetic_symbol_history(
     return synthetic_rows, source
 
 
-def _get_source_priority(symbol: str) -> List[str]:
+def _get_source_priority(symbol: str, interval: Optional[str] = None) -> List[str]:
     cfg = SYMBOLS[symbol]
     priority = CATEGORY_SOURCE_PRIORITY.get(cfg.category, DEFAULT_SOURCE_PRIORITY)
-    return [s for s in priority if s in cfg.mappings]
+    if settings.is_hf and cfg.category == "Crypto":
+        priority = (
+            HF_CRYPTO_SOURCE_PRIORITY_DAILY
+            if interval in {"1d", "1w"}
+            else HF_CRYPTO_SOURCE_PRIORITY_INTRADAY
+        )
+    return [source_name for source_name in priority if _has_source_mapping(symbol, source_name)]
 
 
 async def _run_historical_fetch(
@@ -1957,7 +2040,7 @@ async def _run_historical_fetch(
     if _is_synthetic_symbol(symbol):
         return await _build_synthetic_symbol_history(symbol, interval, fetch_limit, cache_key)
 
-    priority = _get_source_priority(symbol)
+    priority = _get_source_priority(symbol, interval)
     errors: List[str] = []
 
     for source in priority:
@@ -2091,18 +2174,21 @@ async def fetch_ticker(symbol: str, interval: Optional[str] = None) -> Dict[str,
             return result
 
         cfg = SYMBOLS[symbol]
-        priority = _get_source_priority(symbol)
+        priority = _get_source_priority(symbol, interval=interval)
         client = await GlobalHTTPClient.get_client()
 
         for source in priority:
             try:
                 if source == "binance":
+                    binance_symbol = _get_symbol_mapping(symbol, "binance")
+                    if not binance_symbol:
+                        continue
                     await _rate_limit("binance")
                     base_url = "https://fapi.binance.com" if cfg.binance_type == "futures" else "https://api.binance.com"
                     endpoint = "/fapi/v1/ticker/24hr" if cfg.binance_type == "futures" else "/api/v3/ticker/24hr"
                     r = await client.get(
                         f"{base_url}{endpoint}",
-                        params={"symbol": cfg.mappings["binance"]},
+                        params={"symbol": binance_symbol},
                         timeout=10.0,
                     )
                     d = r.json()
@@ -2116,10 +2202,13 @@ async def fetch_ticker(symbol: str, interval: Optional[str] = None) -> Dict[str,
                         "source": "binance",
                     }
                 elif source == "twelvedata":
+                    twelvedata_symbol = _get_symbol_mapping(symbol, "twelvedata")
+                    if not twelvedata_symbol:
+                        continue
                     await _rate_limit("twelvedata")
                     r = await client.get(
                         "https://api.twelvedata.com/quote",
-                        params={"symbol": cfg.mappings["twelvedata"], "apikey": twelvedata_pool.next_key()},
+                        params={"symbol": twelvedata_symbol, "apikey": twelvedata_pool.next_key()},
                         timeout=10.0,
                     )
                     d = r.json()
@@ -2137,9 +2226,12 @@ async def fetch_ticker(symbol: str, interval: Optional[str] = None) -> Dict[str,
                         "source": "twelvedata",
                     }
                 elif source == "bybit":
+                    bybit_symbol = _get_symbol_mapping(symbol, "bybit")
+                    if not bybit_symbol:
+                        continue
                     await _rate_limit("bybit")
                     url = "https://api.bybit.com/v5/market/tickers"
-                    query_params = {"category": cfg.bybit_category, "symbol": cfg.mappings["bybit"]}
+                    query_params = {"category": cfg.bybit_category, "symbol": bybit_symbol}
                     headers = {}
 
                     if settings.bybit_api_key and settings.bybit_api_secret:
@@ -2175,8 +2267,11 @@ async def fetch_ticker(symbol: str, interval: Optional[str] = None) -> Dict[str,
                         "source": "bybit",
                     }
                 elif source == "yfinance":
+                    yfinance_symbol = _get_symbol_mapping(symbol, "yfinance")
+                    if not yfinance_symbol:
+                        continue
                     def _yf_info():
-                        ticker = yf.Ticker(cfg.mappings["yfinance"])
+                        ticker = yf.Ticker(yfinance_symbol)
                         history = ticker.history(period="5d")
                         if history.empty:
                             return None

backend/test_api_regressions.py

@@ -95,6 +95,55 @@ class ApiRegressionTests(unittest.TestCase):
         self.assertNotIn("binance", main._get_source_priority("EURUSD"))
         self.assertEqual(main._get_source_priority("DXY"), ["yfinance", "twelvedata"])
 
+    def test_crypto_dynamic_source_mappings_are_derived_for_hf_safe_fallbacks(self) -> None:
+        self.assertEqual(main._get_symbol_mapping("BTCUSD", "twelvedata"), "BTC/USD")
+        self.assertEqual(main._get_symbol_mapping("BTCUSD", "yfinance"), "BTC-USD")
+        self.assertEqual(main._get_symbol_mapping("BTCUSD", "finnhub"), "BINANCE:BTCUSDT")
+        self.assertEqual(main._get_canonical_symbol("BTC-USD"), "BTCUSD")
+
+    def test_crypto_source_priority_prefers_hf_safe_providers_on_hf(self) -> None:
+        with patch.object(main.settings, "is_hf", True):
+            self.assertEqual(
+                main._get_source_priority("BTCUSD", "1d"),
+                ["yfinance", "twelvedata", "finnhub", "coingecko", "binance", "bybit"],
+            )
+            self.assertEqual(
+                main._get_source_priority("BTCUSD", "1h"),
+                ["twelvedata", "yfinance", "finnhub", "coingecko", "binance", "bybit"],
+            )
+
+    def test_crypto_historical_fetch_on_hf_uses_hf_safe_fallback_first(self) -> None:
+        sample_rows = [
+            {"time": i, "open": 1.0, "high": 1.1, "low": 0.9, "close": 1.0, "volume": 0.0}
+            for i in range(1, 41)
+        ]
+        attempts: list[str] = []
+
+        async def fake_twelvedata(symbol: str, interval: str, limit: int) -> list[dict[str, float]]:
+            attempts.append("twelvedata")
+            return sample_rows[-limit:]
+
+        async def fake_yfinance(symbol: str, interval: str, limit: int) -> list[dict[str, float]]:
+            attempts.append("yfinance")
+            raise RuntimeError("should not be needed after TwelveData success")
+
+        with patch.object(main.settings, "is_hf", True), patch.object(
+            main,
+            "fetch_twelvedata",
+            side_effect=fake_twelvedata,
+        ), patch.object(
+            main,
+            "fetch_yfinance",
+            side_effect=fake_yfinance,
+        ):
+            rows, source = asyncio.run(
+                main._run_historical_fetch("BTCUSD", "1h", 40, "hist_btcusd_1h_hf_test")
+            )
+
+        self.assertEqual(source, "twelvedata")
+        self.assertEqual(len(rows), 40)
+        self.assertEqual(attempts, ["twelvedata"])
+
     def test_historical_fetch_falls_back_after_provider_http_error(self) -> None:
         sample_rows = [
             {"time": i, "open": 1.0, "high": 1.1, "low": 0.9, "close": 1.0, "volume": 0.0}