from __future__ import annotations import io import json import os from typing import Any DEFAULT_LOG_REPO_ID = "kyu823/silicon-sampling-logs" def hf_log_status() -> dict[str, Any]: repo_id = hf_log_repo_id() token = hf_log_token() return { "enabled": bool(repo_id and token), "repoId": repo_id, "tokenSet": bool(token), "private": hf_log_private(), } def upload_run_log_to_hf_dataset(run_document: dict[str, Any]) -> dict[str, Any]: repo_id = hf_log_repo_id() token = hf_log_token() if not repo_id: raise RuntimeError("HF_LOG_REPO_ID is required for Hugging Face Dataset logging") if not token: raise RuntimeError("HF_LOG_TOKEN or HF_TOKEN is required for Hugging Face Dataset logging") from huggingface_hub import HfApi api = HfApi(token=token) api.create_repo(repo_id, repo_type="dataset", private=hf_log_private(), exist_ok=True) path_in_repo = hf_log_path(run_document) data = json.dumps(run_document, ensure_ascii=False, indent=2, sort_keys=True).encode("utf-8") commit = api.upload_file( path_or_fileobj=io.BytesIO(data), path_in_repo=path_in_repo, repo_id=repo_id, repo_type="dataset", commit_message=f"Add silicon sampling log {run_document.get('runId', 'run')}", ) return { "enabled": True, "uploaded": True, "repoId": repo_id, "path": path_in_repo, "commitUrl": getattr(commit, "commit_url", None), } def hf_log_repo_id() -> str: return os.getenv("HF_LOG_REPO_ID", DEFAULT_LOG_REPO_ID).strip() def hf_log_token() -> str: return (os.getenv("HF_LOG_TOKEN") or os.getenv("HF_TOKEN") or "").strip() def hf_log_private() -> bool: return os.getenv("HF_LOG_REPO_PRIVATE", "true").strip().lower() not in {"0", "false", "no"} def hf_log_path(run_document: dict[str, Any]) -> str: created = str(run_document.get("createdAt") or "") day = created[:10] if len(created) >= 10 else "unknown-date" safe_created = created.replace(":", "-").replace("+", "Z") run_id = str(run_document.get("runId") or "run") return f"runs/{day}/{safe_created}_{run_id}.json"