scripts/sync_hf_publish_mirrors.py

#!/usr/bin/env python3
"""Sync repo publication files into the prepared Hugging Face bundles.

The upload step publishes ../hf_publish/{space,artifacts,model}; this helper
keeps those staging folders aligned with the same file groups checked by
validate_mirror_parity.py.
"""

from __future__ import annotations

import argparse
import importlib.util
import json
import shutil
from pathlib import Path


ROOT = Path(__file__).resolve().parents[1]
DEFAULT_HF_ROOT = ROOT.parent / "hf_publish"
PARITY_SCRIPT = ROOT / "scripts/validate_mirror_parity.py"
STALE_MIRROR_FILES = [
    "artifacts/scripts/omni/collect_qwen3_v4_publication_artifacts.py",
    "model/scripts/omni/collect_qwen3_v4_publication_artifacts.py",
]
GENERATED_REPORT_DATA_FILES = [
    # The parity validator rewrites this report, so it is synced after checks
    # rather than included in the self-referential hash parity file set.
    "mirror_parity.json",
]
ENHANCEMENT_MARKER = "docs/data/task_suite_enhancement_128.json"
ENHANCEMENT_CARD_BLOCK = """
## 128-Episode Enhancement Pack

The no-new-episode suite push is recorded in `TASK_SUITE_ENHANCEMENT_128.md`
and `docs/data/task_suite_enhancement_128.json`. It recommends
`multiscale_20s10_40s20_80s40`, hierarchical action/subtask targets,
label-normalized scoring, and compact raw-feature shards before adding more
episodes.
"""
TIER2_MARKER = "docs/data/task_suite_20.json"
TIER2_CARD_BLOCK = """
## Unified 20-Task Suite

The public-sample task surface is now one unified 20-task suite in
`TASK_SUITE_20.md` and `docs/data/task_suite_20.json`. Tasks 1-12 are the
original sample tasks; Tasks 13-20 reuse the same 20-frame windows, 5-frame
stride, feature manifest, chronological split, and minimal/neural head pattern.
The historical `tier2_task_suite` path is retained only for stable artifact
links to tasks 13-20. The unified radar chart is published as
`docs/assets/charts/unified_task_model_radar.svg` with values in
`docs/data/unified_task_model_radar.json`; the 9-method by 20-task completion
matrix is in `docs/data/task_method_20_result_matrix.json`, with the explicit
gap audit in `docs/data/task_method_20_gap_audit.json`. Split radars for
the one-episode baselines and selected 128-episode methods are published as
`docs/assets/charts/single_episode_task_model_radar.svg` and
`docs/assets/charts/episode128_task_model_radar.svg`.
"""
READER_MAP_MARKER = "docs/data/public_reader_map.json"
READER_MAP_ARTIFACT_ROW = (
    "| Choose the right public surface | `PUBLIC_READER_MAP.md`, "
    "`docs/data/public_reader_map.json` |"
)
READER_MAP_CARD_BLOCK = """
## Public Surface Map

Use `PUBLIC_READER_MAP.md` and `docs/data/public_reader_map.json` to choose
between the GitHub repo, GitHub Pages dashboard, HF Space, artifact dataset,
baseline model repo, Qwen3/Cosmos model repos, and release-health checks without
losing the full evidence trail.
"""
XPERIENCE128_MARKER = "docs/data/xperience10m_128_episode_feature_index.json"
XPERIENCE128_ARTIFACT_ROW = (
    "| Trace the 128-episode source and feature map | "
    "`XPERIENCE10M_128_EPISODE_FEATURE_INDEX.md`, "
    "`docs/data/xperience10m_128_episode_feature_index.json` |"
)
XPERIENCE128_CARD_BLOCK = """
## 128-Episode Source and Feature Index

The selected 128-episode split is linked back to the official gated
`ropedia-ai/xperience-10m` episode tree in
`XPERIENCE10M_128_EPISODE_FEATURE_INDEX.md` and
`docs/data/xperience10m_128_episode_feature_index.json`. The public mirrors
carry only public-safe processed artifacts: selection files, inspected
manifests, dense multiscale window rows, metadata feature matrices, and result
summaries.
"""
LANGUAGE_VERSIONS_MARKER = "docs/data/language_versions.json"
LANGUAGE_VERSIONS_ROW = (
    "| Read the project in 8 languages | `README.md`, `README.zh.md`, "
    "`README.es.md`, `README.fr.md`, `README.de.md`, `README.ja.md`, "
    "`README.ko.md`, `README.pt.md`, `docs/data/language_versions.json` |"
)
LANGUAGE_VERSIONS_CARD_BLOCK = """
## Multilingual Entry Points

The canonical repo README now has eight public reader entry points: English,
Chinese, Spanish, French, German, Japanese, Korean, and Portuguese. The
machine-readable language map is in `docs/data/language_versions.json`; each
Hugging Face mirror carries the same translated README files so readers can
move between GitHub, the dashboard, the Space, the artifact dataset, and model
cards without losing the evidence trail.
"""
QWEN_COMPARISON_MARKER = "docs/data/qwen3_v5_v6_comparison.json"
QWEN_COMPARISON_ROW = (
    "| Compare Qwen3 v5/v6 diagnostic branches | "
    "`docs/data/qwen3_v5_v6_comparison.json` |"
)
QWEN_ARTIFACT_OLD_BULLET = """- A current verified Qwen3-Omni strict-label v3 held-out package for the
  selected 96/16/16 episode split, with 100.00% JSON validity and weak
  action/subtask quality documented as the next error-analysis target."""
QWEN_ARTIFACT_CURRENT_BULLET = """- The latest verified Qwen3-Omni LoRA v6 diagnostic package for the selected
  96/16/16 episode split includes 34,269 exported windows, 4,032 held-out test
  predictions, 99.90% JSON validity, and public-safe metrics/predictions."""
README_QWEN_OLD_PARAGRAPH = """The current verified diagnostic package uses the same selected split and 8-GPU
training path, records validation loss over 512 validation windows, and keeps
the held-out test split sealed for final evaluation. The next pass should keep
this package contract while tightening JSON decoding, target formatting, and
action/subtask error analysis."""
README_QWEN_CURRENT_PARAGRAPH = """The latest verified diagnostic package uses the same selected split and 8-GPU
training path, includes the full held-out evaluation with 4,032 predictions and
99.90% JSON validity, and keeps raw data plus full Qwen weights out of the
public repos. The next pass should keep this package contract while improving
action/subtask target quality and error analysis."""


def load_parity_module():
    spec = importlib.util.spec_from_file_location("validate_mirror_parity", PARITY_SCRIPT)
    if spec is None or spec.loader is None:
        raise SystemExit(f"Could not load {PARITY_SCRIPT}")
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


def copy_file(src: Path, destinations: list[Path], *, dry_run: bool) -> list[dict]:
    records = []
    if not src.is_file():
        raise SystemExit(f"Missing source file: {src}")
    for dst in destinations:
        records.append({"source": src.relative_to(ROOT).as_posix(), "dest": dst.as_posix()})
        if dry_run:
            continue
        dst.parent.mkdir(parents=True, exist_ok=True)
        shutil.copy2(src, dst)
    return records


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--hf-root", type=Path, default=DEFAULT_HF_ROOT)
    parser.add_argument("--dry-run", action="store_true")
    parser.add_argument("--json", action="store_true", help="print machine-readable copy records")
    return parser.parse_args()


def prune_stale_files(hf_root: Path, *, dry_run: bool) -> list[str]:
    removed = []
    for relative_path in STALE_MIRROR_FILES:
        path = hf_root / relative_path
        if not path.exists():
            continue
        removed.append(path.as_posix())
        if not dry_run:
            path.unlink()
    return removed


def ensure_enhancement_card_links(hf_root: Path, *, dry_run: bool) -> list[str]:
    updated = []
    for relative_path in ("artifacts/README.md", "model/README.md"):
        path = hf_root / relative_path
        if not path.exists():
            continue
        text = path.read_text(encoding="utf-8")
        if ENHANCEMENT_MARKER in text:
            continue
        insert_before = "\n## Dataset Boundary" if relative_path.startswith("artifacts/") else "\n## Start Here"
        if insert_before in text:
            text = text.replace(insert_before, ENHANCEMENT_CARD_BLOCK + insert_before, 1)
        else:
            text = text.rstrip() + "\n" + ENHANCEMENT_CARD_BLOCK
        updated.append(relative_path)
        if not dry_run:
            path.write_text(text, encoding="utf-8")
    return updated


def ensure_tier2_card_links(hf_root: Path, *, dry_run: bool) -> list[str]:
    updated = []
    for relative_path in ("space/README.md", "artifacts/README.md", "model/README.md"):
        path = hf_root / relative_path
        if not path.exists():
            continue
        text = path.read_text(encoding="utf-8")
        text = text.replace(
            "original sample tasks; tasks 13-20 reuse",
            "original sample tasks; Tasks 13-20 reuse",
        )
        if "docs/data/unified_task_model_radar.json" not in text:
            text = text.replace(
                "links to tasks 13-20.\n",
                "links to tasks 13-20. The unified radar chart is published as\n"
                "`docs/assets/charts/unified_task_model_radar.svg` with values in\n"
                "`docs/data/unified_task_model_radar.json`; the 9-method by\n"
                "20-task completion matrix is in\n"
                "`docs/data/task_method_20_result_matrix.json`, with the explicit\n"
                "gap audit in `docs/data/task_method_20_gap_audit.json`. Split radars are in\n"
                "`docs/assets/charts/single_episode_task_model_radar.svg` and\n"
                "`docs/assets/charts/episode128_task_model_radar.svg`.\n",
            )
        if (
            "docs/data/unified_task_model_radar.json" in text
            and "docs/data/task_method_20_result_matrix.json" not in text
        ):
            text = text.replace(
                "`docs/data/unified_task_model_radar.json`.",
                "`docs/data/unified_task_model_radar.json`; the 9-method by 20-task\n"
                "completion matrix is in `docs/data/task_method_20_result_matrix.json`,\n"
                "with the explicit gap audit in `docs/data/task_method_20_gap_audit.json`.",
            )
        if (
            "docs/data/task_method_20_result_matrix.json" in text
            and "docs/data/task_method_20_gap_audit.json" not in text
        ):
            text = text.replace(
                "`docs/data/task_method_20_result_matrix.json`.",
                "`docs/data/task_method_20_result_matrix.json`, with the explicit\n"
                "gap audit in `docs/data/task_method_20_gap_audit.json`.",
            )
        if (
            "docs/data/task_method_20_result_matrix.json" in text
            and "docs/assets/charts/single_episode_task_model_radar.svg" not in text
        ):
            text = text.replace(
                "`docs/data/task_method_20_result_matrix.json`.",
                "`docs/data/task_method_20_result_matrix.json`. Split radars are in\n"
                "`docs/assets/charts/single_episode_task_model_radar.svg` and\n"
                "`docs/assets/charts/episode128_task_model_radar.svg`.",
            )
        if TIER2_MARKER in text:
            if not dry_run:
                path.write_text(text, encoding="utf-8")
            continue
        insert_before = "\n## Dataset Boundary" if relative_path.startswith("artifacts/") else "\n## Start Here"
        if insert_before in text:
            text = text.replace(insert_before, TIER2_CARD_BLOCK + insert_before, 1)
        else:
            text = text.rstrip() + "\n" + TIER2_CARD_BLOCK
        updated.append(relative_path)
        if not dry_run:
            path.write_text(text, encoding="utf-8")
    return updated


def ensure_reader_map_card_links(hf_root: Path, *, dry_run: bool) -> list[str]:
    updated = []
    artifacts_readme = hf_root / "artifacts/README.md"
    if not artifacts_readme.exists():
        return updated
    text = artifacts_readme.read_text(encoding="utf-8")
    original = text
    if READER_MAP_ARTIFACT_ROW not in text:
        table_anchor = "| Reader goal | Artifact |\n| --- | --- |\n"
        if table_anchor in text:
            text = text.replace(table_anchor, table_anchor + READER_MAP_ARTIFACT_ROW + "\n", 1)
        else:
            text = text.rstrip() + "\n\n" + READER_MAP_ARTIFACT_ROW + "\n"
    if "## Public Surface Map" not in text:
        insert_before = "\n## Dataset Boundary"
        if insert_before in text:
            text = text.replace(insert_before, READER_MAP_CARD_BLOCK + insert_before, 1)
        else:
            text = text.rstrip() + "\n" + READER_MAP_CARD_BLOCK
    if text != original:
        updated.append("artifacts/README.md")
        if not dry_run:
            artifacts_readme.write_text(text, encoding="utf-8")
    return updated


def ensure_xperience128_card_links(hf_root: Path, *, dry_run: bool) -> list[str]:
    updated = []
    artifacts_readme = hf_root / "artifacts/README.md"
    if not artifacts_readme.exists():
        return updated
    text = artifacts_readme.read_text(encoding="utf-8")
    original = text
    if XPERIENCE128_ARTIFACT_ROW not in text:
        table_anchor = "| Reader goal | Artifact |\n| --- | --- |\n"
        if table_anchor in text:
            text = text.replace(table_anchor, table_anchor + XPERIENCE128_ARTIFACT_ROW + "\n", 1)
        else:
            text = text.rstrip() + "\n\n" + XPERIENCE128_ARTIFACT_ROW + "\n"
    if XPERIENCE128_MARKER not in text:
        insert_before = "\n## Dataset Boundary"
        if insert_before in text:
            text = text.replace(insert_before, XPERIENCE128_CARD_BLOCK + insert_before, 1)
        else:
            text = text.rstrip() + "\n" + XPERIENCE128_CARD_BLOCK
    if text != original:
        updated.append("artifacts/README.md")
        if not dry_run:
            artifacts_readme.write_text(text, encoding="utf-8")
    return updated


def ensure_language_card_links(hf_root: Path, *, dry_run: bool) -> list[str]:
    updated = []
    artifacts_readme = hf_root / "artifacts/README.md"
    if not artifacts_readme.exists():
        return updated
    text = artifacts_readme.read_text(encoding="utf-8")
    original = text
    if LANGUAGE_VERSIONS_ROW not in text:
        table_anchor = "| Reader goal | Artifact |\n| --- | --- |\n"
        if table_anchor in text:
            text = text.replace(table_anchor, table_anchor + LANGUAGE_VERSIONS_ROW + "\n", 1)
        else:
            text = text.rstrip() + "\n\n" + LANGUAGE_VERSIONS_ROW + "\n"
    if "## Multilingual Entry Points" not in text:
        insert_before = "\n## Dataset Boundary"
        if insert_before in text:
            text = text.replace(insert_before, LANGUAGE_VERSIONS_CARD_BLOCK + insert_before, 1)
        else:
            text = text.rstrip() + "\n" + LANGUAGE_VERSIONS_CARD_BLOCK
    if text != original:
        updated.append("artifacts/README.md")
        if not dry_run:
            artifacts_readme.write_text(text, encoding="utf-8")
    return updated


def split_hf_frontmatter(text: str) -> tuple[str, str]:
    lines = text.splitlines()
    if not lines or lines[0].strip() != "---":
        return "", text
    for idx in range(1, len(lines)):
        if lines[idx].strip() == "---":
            frontmatter = "\n".join(lines[: idx + 1]).rstrip() + "\n\n"
            body = "\n".join(lines[idx + 1 :]).lstrip()
            return frontmatter, body
    return "", text


def refresh_project_readme_cards(hf_root: Path, *, dry_run: bool) -> list[str]:
    """Keep HF project cards aligned with the current repo README body."""

    project_readme = (ROOT / "README.md").read_text(encoding="utf-8").rstrip() + "\n"
    updated = []
    for relative_path in ("space/PROJECT_README.md", "artifacts/PROJECT_README.md", "model/PROJECT_README.md"):
        path = hf_root / relative_path
        if not path.exists():
            continue
        original = path.read_text(encoding="utf-8")
        if original == project_readme:
            continue
        updated.append(relative_path)
        if not dry_run:
            path.write_text(project_readme, encoding="utf-8")

    for relative_path in ("space/README.md", "model/README.md"):
        path = hf_root / relative_path
        if not path.exists():
            continue
        original = path.read_text(encoding="utf-8")
        frontmatter, _body = split_hf_frontmatter(original)
        refreshed = frontmatter + project_readme
        if original == refreshed:
            continue
        updated.append(relative_path)
        if not dry_run:
            path.write_text(refreshed, encoding="utf-8")
    return updated


def read_current_scaleup_line() -> str | None:
    """Return the legacy Markdown scale-up row when the README still has one.

    The current public README uses an HTML table for the research overview, so
    mirrored full project cards no longer need a standalone Markdown row. Keep
    this compatibility hook for older compact cards only.
    """

    for line in (ROOT / "README.md").read_text(encoding="utf-8").splitlines():
        if line.startswith("| Scale-up |"):
            return line
    return None


def ensure_current_qwen_card_links(hf_root: Path, *, dry_run: bool) -> list[str]:
    updated = []

    artifacts_readme = hf_root / "artifacts/README.md"
    if artifacts_readme.exists():
        text = artifacts_readme.read_text(encoding="utf-8")
        original = text
        if QWEN_COMPARISON_ROW not in text:
            anchor = "| Compare current versions and model groups | `docs/data/omni_model_comparison.json` |"
            text = text.replace(anchor, anchor + "\n" + QWEN_COMPARISON_ROW, 1)
        if QWEN_ARTIFACT_OLD_BULLET in text:
            text = text.replace(QWEN_ARTIFACT_OLD_BULLET, QWEN_ARTIFACT_CURRENT_BULLET, 1)
        if "99.90% JSON validity" not in text:
            text = text.rstrip() + "\n\n" + QWEN_ARTIFACT_CURRENT_BULLET + "\n"
        if QWEN_COMPARISON_MARKER not in text:
            text = text.rstrip() + f"\n\nQwen v5/v6 comparison: `{QWEN_COMPARISON_MARKER}`.\n"
        if text != original:
            updated.append("artifacts/README.md")
            if not dry_run:
                artifacts_readme.write_text(text, encoding="utf-8")

    scaleup_line = read_current_scaleup_line()
    for relative_path in ("model/README.md", "model/PROJECT_README.md"):
        path = hf_root / relative_path
        if not path.exists():
            continue
        original = path.read_text(encoding="utf-8")
        text = original
        if README_QWEN_OLD_PARAGRAPH in text:
            text = text.replace(README_QWEN_OLD_PARAGRAPH, README_QWEN_CURRENT_PARAGRAPH, 1)
        changed = text != original
        lines = text.splitlines()
        if scaleup_line:
            for idx, line in enumerate(lines):
                if line.startswith("| Scale-up |"):
                    if line != scaleup_line:
                        lines[idx] = scaleup_line
                        changed = True
                    break
        if changed:
            updated.append(relative_path)
            if not dry_run:
                path.write_text("\n".join(lines) + "\n", encoding="utf-8")

    return updated


def main() -> int:
    args = parse_args()
    hf_root = args.hf_root.expanduser().resolve()
    parity = load_parity_module()

    removed = prune_stale_files(hf_root, dry_run=args.dry_run)
    copied: list[dict] = []
    for filename in parity.DATA_FILES:
        src = ROOT / "docs/data" / filename
        copied += copy_file(
            src,
            [
                hf_root / "space/data" / filename,
                hf_root / "artifacts/data" / filename,
                hf_root / "artifacts/docs/data" / filename,
                hf_root / "model/data" / filename,
                hf_root / "model/docs/data" / filename,
                hf_root / "model/metrics" / filename,
            ],
            dry_run=args.dry_run,
        )

    for filename in GENERATED_REPORT_DATA_FILES:
        src = ROOT / "docs/data" / filename
        copied += copy_file(
            src,
            [
                hf_root / "space/data" / filename,
                hf_root / "artifacts/data" / filename,
                hf_root / "artifacts/docs/data" / filename,
                hf_root / "model/data" / filename,
                hf_root / "model/docs/data" / filename,
                hf_root / "model/metrics" / filename,
            ],
            dry_run=args.dry_run,
        )

    for filename in parity.ASSET_FILES:
        src = ROOT / "docs/assets" / filename
        copied += copy_file(
            src,
            [
                hf_root / "space/assets" / filename,
                hf_root / "artifacts/docs/assets" / filename,
                hf_root / "artifacts/assets" / filename,
                hf_root / "model/assets" / filename,
            ],
            dry_run=args.dry_run,
        )

    for filename in parity.SCRIPT_FILES:
        src = ROOT / "scripts" / filename
        copied += copy_file(
            src,
            [
                hf_root / "artifacts/scripts" / filename,
                hf_root / "model/scripts" / filename,
            ],
            dry_run=args.dry_run,
        )

    for filename in parity.WEBSITE_FILES:
        src = ROOT / "docs" / filename
        copied += copy_file(
            src,
            [
                hf_root / "space" / filename,
                hf_root / "artifacts" / filename,
                hf_root / "artifacts/docs" / filename,
                hf_root / "model" / filename,
                hf_root / "model/docs" / filename,
            ],
            dry_run=args.dry_run,
        )

    result_files = sorted(
        set(parity.RESULT_FILES)
        | set(parity.verified_public_result_files())
        | set(parity.tier2_result_files())
        | set(parity.a100_128_metadata_result_files())
        | set(parity.a100_128_raw20_result_files())
        | set(parity.xperience10m_128_data_feature_files())
        | set(parity.model_output_task_probe_result_files())
        | set(parity.qwen3_future_task_probe_result_files())
    )
    for filename in result_files:
        src = ROOT / "results" / filename
        copied += copy_file(
            src,
            [
                hf_root / "space/results" / filename,
                hf_root / "artifacts/results" / filename,
                hf_root / "model/results" / filename,
            ],
            dry_run=args.dry_run,
        )

    for filename in parity.DOC_FILES:
        src = ROOT / filename
        copied += copy_file(
            src,
            [
                hf_root / "space" / filename,
                hf_root / "artifacts" / filename,
                hf_root / "model" / filename,
            ],
            dry_run=args.dry_run,
        )

    card_updates = refresh_project_readme_cards(hf_root, dry_run=args.dry_run)
    card_updates += ensure_language_card_links(hf_root, dry_run=args.dry_run)
    card_updates += ensure_reader_map_card_links(hf_root, dry_run=args.dry_run)
    card_updates += ensure_xperience128_card_links(hf_root, dry_run=args.dry_run)
    card_updates += ensure_enhancement_card_links(hf_root, dry_run=args.dry_run)
    card_updates += ensure_tier2_card_links(hf_root, dry_run=args.dry_run)
    card_updates += ensure_current_qwen_card_links(hf_root, dry_run=args.dry_run)
    summary = {
        "status": "dry_run" if args.dry_run else "synced",
        "hf_root": hf_root.as_posix(),
        "copy_count": len(copied),
        "removed_stale_count": len(removed),
        "removed_stale": removed,
        "card_updates": card_updates,
        "records": copied,
    }
    if args.json:
        print(json.dumps(summary, indent=2))
    else:
        print(
            f"{summary['status'].upper()}: copied {summary['copy_count']} files into {hf_root}; "
            f"removed {summary['removed_stale_count']} stale files"
        )
    return 0


if __name__ == "__main__":
    raise SystemExit(main())