Instructions to use prometheus04/qwen3-4b-thinking-microagent with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use prometheus04/qwen3-4b-thinking-microagent with PEFT:
Task type is invalid.
- Notebooks
- Google Colab
- Kaggle
| """V2 pipeline: drop math, use convert_code_v2 for code.parquet, original convert for the rest. | |
| Sources (math.parquet removed): | |
| - 3 streaming configs: skill_based_easy, skill_based_medium, skill_based_mixed | |
| - 2 adapter files: dataset_adapters/{code,swe}.parquet | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import os | |
| import sys | |
| from collections import Counter | |
| from pathlib import Path | |
| os.environ.setdefault("HF_HUB_DISABLE_SYMLINKS_WARNING", "1") | |
| sys.path.insert(0, str(Path(__file__).parent)) | |
| from convert import convert_trajectory, RejectStats | |
| from convert_code_v2 import convert_code_v2, CodeRejectStats | |
| REPO = "nvidia/Nemotron-Terminal-Corpus" | |
| STREAMING_CONFIGS = ["skill_based_easy", "skill_based_medium", "skill_based_mixed"] | |
| ADAPTER_FILES = [ | |
| "dataset_adapters/code.parquet", | |
| "dataset_adapters/swe.parquet", | |
| ] | |
| ADAPTER_BATCH_SIZE = 64 | |
| CODE_FILE = "dataset_adapters/code.parquet" | |
| def iter_streaming_config(cfg, limit=None): | |
| from datasets import load_dataset | |
| ds = load_dataset(REPO, cfg, streaming=True) | |
| split = list(ds.keys())[0] | |
| n = 0 | |
| for row in ds[split]: | |
| if limit is not None and n >= limit: | |
| break | |
| yield row | |
| n += 1 | |
| def iter_adapter_file(remote_path, limit=None): | |
| from huggingface_hub import hf_hub_download | |
| import pyarrow.parquet as pq | |
| print(f" downloading {remote_path} (cached if present)...", flush=True) | |
| local = hf_hub_download(repo_id=REPO, repo_type="dataset", filename=remote_path) | |
| pf = pq.ParquetFile(local) | |
| n = 0 | |
| for batch in pf.iter_batches(batch_size=ADAPTER_BATCH_SIZE): | |
| for row in batch.to_pylist(): | |
| if limit is not None and n >= limit: | |
| return | |
| yield row | |
| n += 1 | |
| def run(out_path: Path, sample_per_source: int | None): | |
| out_path.parent.mkdir(parents=True, exist_ok=True) | |
| legacy_stats = RejectStats() | |
| code_stats = CodeRejectStats() | |
| per_source_scanned = Counter() | |
| per_source_accepted = Counter() | |
| sources = [] | |
| for cfg in STREAMING_CONFIGS: | |
| sources.append(("streaming", cfg, lambda c=cfg: iter_streaming_config(c, sample_per_source))) | |
| for f in ADAPTER_FILES: | |
| sources.append(("adapter", f, lambda f=f: iter_adapter_file(f, sample_per_source))) | |
| with open(out_path, "w", encoding="utf-8") as out: | |
| for kind, name, factory in sources: | |
| print(f"\n--- {kind}: {name} ---", flush=True) | |
| try: | |
| for row in factory(): | |
| per_source_scanned[name] += 1 | |
| try: | |
| if name == CODE_FILE: | |
| r = convert_code_v2(row, code_stats) | |
| else: | |
| r = convert_trajectory(row, name, legacy_stats) | |
| except Exception: | |
| legacy_stats.other += 1 | |
| continue | |
| if r is not None: | |
| out.write(json.dumps(r, ensure_ascii=False) + "\n") | |
| per_source_accepted[name] += 1 | |
| if per_source_scanned[name] % 2000 == 0: | |
| rate = per_source_accepted[name] / max(per_source_scanned[name], 1) * 100 | |
| print(f" {name}: scanned={per_source_scanned[name]:6d} " | |
| f"accepted={per_source_accepted[name]:6d} ({rate:.1f}%)", | |
| flush=True) | |
| except Exception as e: | |
| print(f" ERROR on {name}: {e}", flush=True) | |
| import traceback; traceback.print_exc() | |
| continue | |
| rate = per_source_accepted[name] / max(per_source_scanned[name], 1) * 100 | |
| print(f" {name}: DONE scanned={per_source_scanned[name]} " | |
| f"accepted={per_source_accepted[name]} ({rate:.1f}%)", flush=True) | |
| print("\n\n========== FINAL ==========") | |
| print(f"Output: {out_path}") | |
| total_scanned = sum(per_source_scanned.values()) | |
| total_accepted = sum(per_source_accepted.values()) | |
| print(f"Total scanned : {total_scanned}") | |
| print(f"Total accepted: {total_accepted} ({100*total_accepted/max(total_scanned,1):.1f}%)") | |
| print("\nPer-source accepted:") | |
| for k, v in per_source_accepted.most_common(): | |
| sc = per_source_scanned[k] | |
| rate = v / max(sc, 1) * 100 | |
| print(f" {k:50s} {v:6d} / {sc:6d} ({rate:5.1f}%)") | |
| print(f"\nCode (v2) breakdown:") | |
| print(f" success/<finish>: {code_stats.accepted_success}") | |
| print(f" give_up: {code_stats.accepted_giveup}") | |
| print(f" rejected reasons: too_few={code_stats.rejected_too_few_turns} " | |
| f"too_many={code_stats.rejected_too_many_turns} " | |
| f"early_parse_err={code_stats.rejected_early_parse_err} " | |
| f"json_unrecoverable={code_stats.rejected_json_unrecoverable} " | |
| f"no_real_attempt={code_stats.rejected_no_real_attempt_failure} " | |
| f"per_task_cap={code_stats.rejected_per_task_cap}") | |
| def main(): | |
| p = argparse.ArgumentParser() | |
| p.add_argument("--sample", type=int, default=None) | |
| p.add_argument("--out", type=str, default="data/microagent_train_v2.jsonl") | |
| args = p.parse_args() | |
| run(Path(args.out), args.sample) | |
| if __name__ == "__main__": | |
| main() | |