Optimize requests: remove background prefetching, disable market peers, fix Binance NameError

backend/main.py

@@ -100,6 +100,9 @@ class Settings(BaseModel):
     alphavantage_api_key: Optional[str] = os.getenv("ALPHAVANTAGE_API_KEY")
     admin_token:        str           = os.getenv("ADMIN_TOKEN", "kronos_v6_default_secret")
 
+    # Environment detection
+    is_hf: bool = os.getenv("SPACE_ID") is not None
+
     # App Config
     host: str = os.getenv("HOST", "0.0.0.0")
     port: int = int(os.getenv("PORT", 8000))
@@ -803,44 +806,60 @@ async def fetch_binance(symbol: str, interval: str, limit: int) -> List[Dict[str
     await _rate_limit("binance")
     cfg = SYMBOLS[symbol]
     endpoint_symbol = cfg.mappings["binance"]
-    base_url = "https://fapi.binance.com" if cfg.binance_type == "futures" else "https://api.binance.com"
-    endpoint = "/fapi/v1/klines" if cfg.binance_type == "futures" else "/api/v3/klines"
     
+    # Define request details
+    endpoint = "/fapi/v1/klines" if cfg.binance_type == "futures" else "/api/v3/klines"
     params = {
         "symbol":   endpoint_symbol,
-        "interval": BINANCE_INTERVAL_MAP.get(interval, interval),
-        "limit":    min(max(limit, 30), 1000),
+        "interval": BINANCE_INTERVAL_MAP.get(interval, "1h"),
+        "limit":    min(max(limit, 1), 1000),
     }
-    logger.info("[Binance] %s %s (%s)", symbol, interval, cfg.binance_type)
 
-    async def _fetch():
-        cb = source_breakers["binance"]
-        if not cb.allow_request():
-            raise HTTPException(status_code=503, detail="Binance circuit is OPEN")
-            
+    # B-2: Endpoint Rotation for HF/Cloud environments
+    endpoints_spot = ["https://api.binance.com", "https://api1.binance.com", "https://api2.binance.com", "https://api3.binance.com", "https://data-api.binance.com"]
+    endpoints_fapi = ["https://fapi.binance.com"] # fapi usually more restricted, but try first
+    
+    selected_endpoints = endpoints_fapi if cfg.binance_type == "futures" else endpoints_spot
+    
+    # If on HF, we know api.binance.com is likely blocked, so we can try data-api or alternates faster
+    if settings.is_hf and cfg.binance_type == "spot":
+        # Move data-api to the front for HF spot
+        selected_endpoints = ["https://data-api.binance.com", "https://api1.binance.com", "https://api2.binance.com", "https://api3.binance.com", "https://api.binance.com"]
+
+    last_error = None
+    for base_url in selected_endpoints:
         try:
-            client = await GlobalHTTPClient.get_client()
-            resp = await client.get(f"{base_url}{endpoint}", params=params)
-            if resp.status_code == 429:
-                cb.record_failure()
-                raise HTTPException(status_code=429, detail="Binance rate limit")
-            if resp.status_code >= 500:
-                cb.record_failure()
-                raise HTTPException(status_code=resp.status_code, detail="Binance server error")
+            cb = source_breakers["binance"]
+            if not cb.allow_request():
+                continue # Try next endpoint or fall through
+                
+            async def _do_fetch():
+                client = await GlobalHTTPClient.get_client()
+                resp = await client.get(f"{base_url}{endpoint}", params=params, timeout=10.0)
+                if resp.status_code == 451:
+                    logger.warning("[Binance] Endpoint %s blocked (451). Trying next...", base_url)
+                    raise RuntimeError("IP Blocked")
+                if resp.status_code == 429:
+                    raise HTTPException(status_code=429, detail="Binance rate limit")
+                resp.raise_for_status()
+                cb.record_success()
+                return resp.json()
+
+            payload = await _retry(_do_fetch)
+            # If successful, parse and return
+            parsed = [
+                {"time": int(k[0])//1000, "open": k[1], "high": k[2],
+                 "low": k[3], "close": k[4], "volume": k[5]}
+                for k in payload
+            ]
+            return _normalize_ohlcv(parsed, interval)[-limit:]
             
-            cb.record_success()
-            return resp.json()
         except Exception as ex:
-            cb.record_failure()
-            raise ex
-
-    payload = await _retry(_fetch)
-    parsed = [
-        {"time": int(k[0])//1000, "open": k[1], "high": k[2],
-         "low": k[3], "close": k[4], "volume": k[5]}
-        for k in payload
-    ]
-    return _normalize_ohlcv(parsed, interval)[-limit:]
+            last_error = ex
+            logger.error("[Binance] Failed with %s: %s", base_url, ex)
+            continue
+            
+    raise last_error or HTTPException(status_code=503, detail="Binance all endpoints failed")
 
 
 async def fetch_bybit(symbol: str, interval: str, limit: int) -> List[Dict[str, Any]]:
@@ -856,64 +875,69 @@ async def fetch_bybit(symbol: str, interval: str, limit: int) -> List[Dict[str,
     }
     logger.info("[Bybit] %s %s (cat=%s)", symbol, interval, bybit_cat)
 
-    async def _fetch():
-        cb = source_breakers["bybit"]
-        if not cb.allow_request():
-            raise HTTPException(status_code=503, detail="Bybit circuit is OPEN")
-            
+    # B-3: Endpoint Rotation for Bybit
+    bybit_endpoints = ["https://api.bybit.com", "https://api.bytick.com", "https://api.bybit.nl"]
+    if settings.is_hf:
+        # Prefer bytick on HF
+        bybit_endpoints = ["https://api.bytick.com", "https://api.bybit.com", "https://api.bybit.nl"]
+
+    last_error = None
+    for base_url in bybit_endpoints:
         try:
-            client = await GlobalHTTPClient.get_client()
-            url = "https://api.bybit.com/v5/market/kline"
-            
-            # Authenticate if keys are available
-            headers = {}
-            query_params = params.copy()
-            
-            if settings.bybit_api_key and settings.bybit_api_secret:
-                timestamp = str(int(time.time() * 1000))
-                recv_window = "5000"
-                # For GET, sort params alphabetically and join
-                sorted_params = "&".join([f"{k}={v}" for k, v in sorted(query_params.items())])
-                raw_str = timestamp + settings.bybit_api_key + recv_window + sorted_params
-                signature = hmac.new(settings.bybit_api_secret.encode('utf-8'), 
-                                    raw_str.encode('utf-8'), hashlib.sha256).hexdigest()
+            cb = source_breakers["bybit"]
+            if not cb.allow_request():
+                continue
                 
-                headers = {
-                    'X-BAPI-API-KEY': settings.bybit_api_key,
-                    'X-BAPI-TIMESTAMP': timestamp,
-                    'X-BAPI-SIGN-TYPE': '2',
-                    'X-BAPI-RECV-WINDOW': recv_window,
-                    'X-BAPI-SIGN': signature
-                }
+            async def _do_fetch():
+                client = await GlobalHTTPClient.get_client()
+                url = f"{base_url}/v5/market/kline"
+                
+                headers = {}
+                query_params = params.copy()
+                
+                if settings.bybit_api_key and settings.bybit_api_secret:
+                    timestamp = str(int(time.time() * 1000))
+                    recv_window = "5000"
+                    sorted_params = "&".join([f"{k}={v}" for k, v in sorted(query_params.items())])
+                    raw_str = timestamp + settings.bybit_api_key + recv_window + sorted_params
+                    signature = hmac.new(settings.bybit_api_secret.encode('utf-8'), 
+                                        raw_str.encode('utf-8'), hashlib.sha256).hexdigest()
+                    
+                    headers = {
+                        'X-BAPI-API-KEY': settings.bybit_api_key,
+                        'X-BAPI-TIMESTAMP': timestamp,
+                        'X-BAPI-SIGN-TYPE': '2',
+                        'X-BAPI-RECV-WINDOW': recv_window,
+                        'X-BAPI-SIGN': signature
+                    }
 
-            resp = await client.get(url, params=query_params, headers=headers)
+                resp = await client.get(url, params=query_params, headers=headers, timeout=10.0)
+                if resp.status_code == 403:
+                    logger.warning("[Bybit] Endpoint %s forbidden (403). Trying next...", base_url)
+                    raise RuntimeError("IP Blocked")
+                resp.raise_for_status()
+                cb.record_success()
+                return resp.json()
+
+            data = await _retry(_do_fetch)
+            if data.get("retCode", -1) != 0:
+                raise RuntimeError(f"Bybit Error: {data}")
+
+            rows = data.get("result", {}).get("list", [])
+            parsed = [
+                {"time": int(r[0])//1000, "open": r[1], "high": r[2],
+                 "low": r[3], "close": r[4], "volume": r[5]}
+                for r in rows
+            ]
+            parsed.reverse()
+            return _normalize_ohlcv(parsed, interval)[-limit:]
             
-            if resp.status_code == 429:
-                cb.record_failure()
-                raise HTTPException(status_code=429, detail="Bybit rate limit")
-            if resp.status_code >= 500:
-                cb.record_failure()
-                raise HTTPException(status_code=resp.status_code, detail="Bybit server error")
-            
-            cb.record_success()
-            return resp.json()
         except Exception as ex:
-            cb.record_failure()
-            raise ex
-
-    data = await _retry(_fetch)
-    if data.get("retCode", -1) != 0:
-        raise RuntimeError(f"Bybit: {data}")
+            last_error = ex
+            logger.error("[Bybit] Failed with %s: %s", base_url, ex)
+            continue
 
-    rows = data.get("result", {}).get("list", [])
-    # Bybit returns: [startTime, open, high, low, close, volume, turnover]
-    parsed = [
-        {"time": int(r[0])//1000, "open": r[1], "high": r[2],
-         "low": r[3], "close": r[4], "volume": r[5]}
-        for r in rows
-    ]
-    parsed.reverse()  # Bybit: newest first
-    return _normalize_ohlcv(parsed, interval)[-limit:]
+    raise last_error or HTTPException(status_code=503, detail="Bybit all endpoints failed")
 
 
 async def fetch_coingecko(symbol: str, interval: str, limit: int) -> List[Dict[str, Any]]:
@@ -2560,7 +2584,6 @@ async def lifespan(app: FastAPI):
     asyncio.create_task(ws_manager.heartbeat())
     asyncio.create_task(_background_cleanup())
     asyncio.create_task(_periodic_health_check())
-    asyncio.create_task(_prefetch_popular_symbols()) # D-3: Prefetcher
     
     # F-2: Rate limit cleanup task
     async def _ip_cleanup_loop():
@@ -2948,26 +2971,6 @@ async def rate_limit_middleware(request: Request, call_next):
     return await call_next(request)
 
 
-RECENT_SYMBOLS: List[str] = []
-
-async def _prefetch_popular_symbols():
-    """D-3: Prefetch data for major symbols and active session symbols (P2)."""
-    static_popular = ["XAUUSD", "BTCUSD", "ETHUSD", "DXY", "SP500"]
-    while True:
-        # Merge static favorites with recently viewed symbols
-        targets = list(dict.fromkeys(RECENT_SYMBOLS + static_popular))[:10]
-        
-        logger.info("[Prefetch] Refreshing symbols: %s", targets)
-        for sym in targets:
-            try:
-                # Fetch 1h and 1d to warm up both indicator contexts
-                await fetch_historical(sym, "1h", 300)
-                await asyncio.sleep(0.5) 
-                await fetch_historical(sym, "1d", 200)
-                await asyncio.sleep(1.0) # Gentle throttling
-            except Exception:
-                pass
-        await asyncio.sleep(300) # Every 5 mins
 
 async def _source_selftest():
     """Ping data sources at startup to confirm reachability (1 attempt each)."""
@@ -3062,49 +3065,51 @@ async def list_symbols(
 
 
 # ── Market Peers ──────────────────────────────────────────────────────────────
-@app.get("/api/market-peers")
-async def get_market_peers(symbol: str = Query("BTCUSD")) -> Dict[str, Any]:
-    # Fix: Ensure symbol is canonical to avoid category mismatches
-    symbol = _get_canonical_symbol(symbol.upper())
-    
-    if symbol not in SYMBOLS:
-        # Fallback to Crypto if unknown
-        return await crypto_market(top=10)
-    
-    cfg = SYMBOLS[symbol]
-    category = cfg.category
-    
-    # Find all peers in the same category
-    peers_list = [s for s in SYMBOLS.values() if s.category == category and s.symbol != symbol]
-    
-    # If too few peers in category, mix with others
-    if len(peers_list) < 3:
-        all_others = [s for s in SYMBOLS.values() if s.symbol != symbol]
-        peers_list.extend(all_others[:5])
+# @app.get("/api/market-peers")
+# async def get_market_peers(symbol: str = Query("BTCUSD")) -> Dict[str, Any]:
+#     # Fix: Ensure symbol is canonical to avoid category mismatches
+#     symbol = _get_canonical_symbol(symbol.upper())
+#     
+#     if symbol not in SYMBOLS:
+#         # Fallback to Crypto if unknown
+#         # return await crypto_market(top=10)
+#         return {"category": "Unknown", "peers": []}
+#     
+#     cfg = SYMBOLS[symbol]
+#     category = cfg.category
+#     
+#     # Find all peers in the same category
+#     peers_list = [s for s in SYMBOLS.values() if s.category == category and s.symbol != symbol]
+#     
+#     # If too few peers in category, mix with others
+#     if len(peers_list) < 3:
+#         all_others = [s for s in SYMBOLS.values() if s.symbol != symbol]
+#         peers_list.extend(all_others[:5])
+# 
+#     # Format result with actual price data
+#     result_peers = []
+#     # Batch fetch ticker data for peers
+#     for p in peers_list[:12]:
+#         try:
+#             ticker = await fetch_ticker(p.symbol)
+#             result_peers.append({
+#                 "symbol": p.symbol,
+#                 "label": p.label,
+#                 "category": p.category,
+#                 "price": ticker.get("price", 0),
+#                 "change_24h": ticker.get("change_pct", 0)
+#             })
+#         except:
+#             result_peers.append({
+#                 "symbol": p.symbol, "label": p.label, "category": p.category,
+#                 "price": 0, "change_24h": 0
+#             })
+#             
+#     return {
+#         "category": category,
+#         "peers": result_peers
+#     }
 
-    # Format result with actual price data
-    result_peers = []
-    # Batch fetch ticker data for peers
-    for p in peers_list[:12]:
-        try:
-            ticker = await fetch_ticker(p.symbol)
-            result_peers.append({
-                "symbol": p.symbol,
-                "label": p.label,
-                "category": p.category,
-                "price": ticker.get("price", 0),
-                "change_24h": ticker.get("change_pct", 0)
-            })
-        except:
-            result_peers.append({
-                "symbol": p.symbol, "label": p.label, "category": p.category,
-                "price": 0, "change_24h": 0
-            })
-            
-    return {
-        "category": category,
-        "peers": result_peers
-    }
 
 
 # ── Symbol search ──────���──────────────────────────────────────────────────────
@@ -3500,11 +3505,6 @@ async def switch_symbol_interval(body: SwitchRequest) -> Dict[str, Any]:
     h_cleared = historical_cache.delete_by_prefix(f"hist_{prefix}")
     f_cleared = forecast_cache.delete_by_prefix(f"forecast_{prefix}")
     
-    # P2: Track recent symbols for pre-fetching
-    global RECENT_SYMBOLS
-    if symbol not in RECENT_SYMBOLS:
-        RECENT_SYMBOLS.insert(0, symbol)
-        RECENT_SYMBOLS = RECENT_SYMBOLS[:5] # Keep top 5
     
     logger.info("[switch] %s %s → hist=%d forecast=%d", symbol, interval, h_cleared, f_cleared)