Upload folder using huggingface_hub

20260628-qwen-small-logits-orchestrator-full-gpu-l40s-plan.json

@@ -0,0 +1,41 @@
+{
+  "can_train_here": true,
+  "config": {
+    "backend": "transformers",
+    "base_model": "Qwen/Qwen2.5-0.5B-Instruct",
+    "batch_size": 4,
+    "device": "cuda",
+    "epochs": 40,
+    "learning_rate": 0.0002,
+    "lora_rank": 0,
+    "max_length": 1536,
+    "seed": 7,
+    "train_backbone": false
+  },
+  "dependency_status": {
+    "mlx": false,
+    "mlx_lm": false,
+    "torch": true,
+    "transformers": true
+  },
+  "prepared_rows": "research/datasets/20260628-qwen-small-logits-orchestrator-split-train.jsonl",
+  "record_count": 53,
+  "rows_output": "research/datasets/20260628-qwen-small-logits-orchestrator-split-train.jsonl",
+  "schema_version": "mempool.qwen_logits_orchestrator_plan.v1",
+  "training_order": [
+    "freeze Qwen-small backbone",
+    "train worker/workflow/verifier/abstain heads on measured soft targets",
+    "compare held-out routing against the linear multi-head baseline",
+    "only enable LoRA/backbone updates after the heads beat the baseline"
+  ],
+  "worker_ids": [
+    "ollama-cloud-deepseek-v4-pro",
+    "ollama-cloud-glm-5.2",
+    "ollama-cloud-kimi-k2.7-code",
+    "ollama-cloud-qwen3-coder-480b"
+  ],
+  "workflow_labels": [
+    "direct",
+    "verify_then_fallback"
+  ]
+}

20260628-qwen-small-logits-orchestrator-split-manifest.json

@@ -0,0 +1,26 @@
+{
+  "heldout_count": 13,
+  "heldout_fraction": 0.2,
+  "heldout_output": "research/datasets/20260628-qwen-small-logits-orchestrator-split-heldout.jsonl",
+  "heldout_task_ids": [
+    "bigcodebench-hard-BigCodeBench-162",
+    "bigcodebench-hard-BigCodeBench-1006",
+    "bigcodebench-hard-BigCodeBench-322",
+    "bigcodebench-hard-BigCodeBench-123",
+    "bigcodebench-hard-BigCodeBench-760",
+    "bigcodebench-hard-BigCodeBench-897",
+    "bigcodebench-hard-BigCodeBench-161",
+    "bigcodebench-hard-BigCodeBench-963",
+    "bigcodebench-hard-BigCodeBench-928",
+    "bigcodebench-hard-BigCodeBench-678",
+    "bigcodebench-hard-BigCodeBench-1022",
+    "bigcodebench-hard-BigCodeBench-139",
+    "bigcodebench-hard-BigCodeBench-955"
+  ],
+  "record_count": 66,
+  "schema_version": "mempool.qwen_logits_split.v1",
+  "seed": 7,
+  "source": "research/datasets/20260628-qwen-small-logits-orchestrator-smoke-rows.jsonl",
+  "train_count": 53,
+  "train_output": "research/datasets/20260628-qwen-small-logits-orchestrator-split-train.jsonl"
+}

README.md

@@ -20,7 +20,7 @@ model weights. Load the base model separately and attach the heads.
 
 - Checkpoint: `qwen_logits_heads.pt`
 - Training rows: `53`
-- Final training loss: `2.8251984119415283`
+- Final training loss: `2.1550024236951555`
 
 Worker labels:
 
@@ -31,17 +31,17 @@ Worker labels:
 
 Train-row evaluation:
 
-- Worker accuracy: `0.6037735849056604`
-- Workflow accuracy: `0.8867924528301887`
-- Mean worker loss: `1.2023728118752532`
-- Mean workflow loss: `0.3021884743686555`
+- Worker accuracy: `0.7358490566037735`
+- Workflow accuracy: `0.9433962264150944`
+- Mean worker loss: `1.124755915605797`
+- Mean workflow loss: `0.142171498247475`
 
 Held-out evaluation:
 
-- Worker accuracy: `0.38461538461538464`
+- Worker accuracy: `0.5384615384615384`
 - Workflow accuracy: `0.7692307692307693`
-- Mean worker loss: `1.3285022240418654`
-- Mean workflow loss: `1.0098582073473013`
+- Mean worker loss: `1.3366980185875525`
+- Mean workflow loss: `0.672384113073349`
 
 A sample router prediction is included at `sample_prediction.json`.
 

heldout_eval_report.json

@@ -1,8 +1,8 @@
 {
-  "checkpoint": "research/models/20260628-qwen-small-logits-orchestrator-full-local/qwen_logits_heads.pt",
-  "device": "mps",
-  "mean_worker_loss": 1.3285022240418654,
-  "mean_workflow_loss": 1.0098582073473013,
+  "checkpoint": "research/models/20260628-qwen-small-logits-orchestrator-full-gpu-l40s/qwen_logits_heads.pt",
+  "device": "cuda",
+  "mean_worker_loss": 1.3366980185875525,
+  "mean_workflow_loss": 0.672384113073349,
   "prediction_sample": [
     {
       "predicted_worker_id": "ollama-cloud-qwen3-coder-480b",
@@ -17,7 +17,7 @@
       "target_workflow": "direct"
     },
     {
-      "predicted_worker_id": "ollama-cloud-deepseek-v4-pro",
+      "predicted_worker_id": "ollama-cloud-qwen3-coder-480b",
       "predicted_workflow": "direct",
       "target_worker_id": "ollama-cloud-qwen3-coder-480b",
       "target_workflow": "verify_then_fallback"
@@ -47,7 +47,7 @@
       "target_workflow": "direct"
     },
     {
-      "predicted_worker_id": "ollama-cloud-kimi-k2.7-code",
+      "predicted_worker_id": "ollama-cloud-qwen3-coder-480b",
       "predicted_workflow": "direct",
       "target_worker_id": "ollama-cloud-glm-5.2",
       "target_workflow": "direct"
@@ -66,9 +66,9 @@
     }
   ],
   "record_count": 13,
-  "rows": "/Users/paras/projects/mempool/research/datasets/20260628-qwen-small-logits-orchestrator-split-heldout.jsonl",
+  "rows": "research/datasets/20260628-qwen-small-logits-orchestrator-split-heldout.jsonl",
   "schema_version": "mempool.qwen_logits_orchestrator_eval_report.v1",
-  "worker_accuracy": 0.38461538461538464,
+  "worker_accuracy": 0.5384615384615384,
   "worker_ids": [
     "ollama-cloud-deepseek-v4-pro",
     "ollama-cloud-glm-5.2",

qwen_logits_heads.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8c7e6fcbab8a6b2947ad921f95c726bce01675786443e1431d2069fdd23566d3
-size 33305
+oid sha256:f13153709a439b6f4d4066f05afd739085739d9fe70a7e0866aece7893bec8f6
+size 33689

sample_prediction.json

@@ -1,22 +1,22 @@
 {
-  "checkpoint": "research/models/20260628-qwen-small-logits-orchestrator-full-local/qwen_logits_heads.pt",
+  "checkpoint": "research/models/20260628-qwen-small-logits-orchestrator-full-gpu-l40s/qwen_logits_heads.pt",
   "device": "mps",
   "predictions": [
     {
-      "abstain_probability": 0.4838430881500244,
+      "abstain_probability": 0.5303195118904114,
       "predicted_worker_id": "ollama-cloud-deepseek-v4-pro",
       "predicted_workflow": "direct",
       "text": "Write Python code to scrape the first HTML table from a URL into a pandas DataFrame, handling HTTP errors and empty tables.",
-      "verifier_probability": 0.9267762899398804,
+      "verifier_probability": 0.9339272975921631,
       "worker_distribution": {
-        "ollama-cloud-deepseek-v4-pro": 0.9986918568611145,
-        "ollama-cloud-glm-5.2": 0.0009675596375018358,
-        "ollama-cloud-kimi-k2.7-code": 0.0002335379394935444,
-        "ollama-cloud-qwen3-coder-480b": 0.00010703552106861025
+        "ollama-cloud-deepseek-v4-pro": 0.9990161657333374,
+        "ollama-cloud-glm-5.2": 0.0008307626703754067,
+        "ollama-cloud-kimi-k2.7-code": 9.308168955612928e-05,
+        "ollama-cloud-qwen3-coder-480b": 5.9906586102442816e-05
       },
       "workflow_distribution": {
-        "direct": 0.9999382495880127,
-        "verify_then_fallback": 6.179204501677305e-05
+        "direct": 0.9992326498031616,
+        "verify_then_fallback": 0.0007673977524973452
       }
     }
   ],

train_eval_report.json

@@ -1,8 +1,8 @@
 {
-  "checkpoint": "research/models/20260628-qwen-small-logits-orchestrator-full-local/qwen_logits_heads.pt",
-  "device": "mps",
-  "mean_worker_loss": 1.2023728118752532,
-  "mean_workflow_loss": 0.3021884743686555,
+  "checkpoint": "research/models/20260628-qwen-small-logits-orchestrator-full-gpu-l40s/qwen_logits_heads.pt",
+  "device": "cuda",
+  "mean_worker_loss": 1.124755915605797,
+  "mean_workflow_loss": 0.142171498247475,
   "prediction_sample": [
     {
       "predicted_worker_id": "ollama-cloud-qwen3-coder-480b",
@@ -41,7 +41,7 @@
       "target_workflow": "direct"
     },
     {
-      "predicted_worker_id": "ollama-cloud-deepseek-v4-pro",
+      "predicted_worker_id": "ollama-cloud-qwen3-coder-480b",
       "predicted_workflow": "direct",
       "target_worker_id": "ollama-cloud-qwen3-coder-480b",
       "target_workflow": "direct"
@@ -66,16 +66,16 @@
     }
   ],
   "record_count": 53,
-  "rows": "/Users/paras/projects/mempool/research/datasets/20260628-qwen-small-logits-orchestrator-split-train.jsonl",
+  "rows": "research/datasets/20260628-qwen-small-logits-orchestrator-split-train.jsonl",
   "schema_version": "mempool.qwen_logits_orchestrator_eval_report.v1",
-  "worker_accuracy": 0.6037735849056604,
+  "worker_accuracy": 0.7358490566037735,
   "worker_ids": [
     "ollama-cloud-deepseek-v4-pro",
     "ollama-cloud-glm-5.2",
     "ollama-cloud-kimi-k2.7-code",
     "ollama-cloud-qwen3-coder-480b"
   ],
-  "workflow_accuracy": 0.8867924528301887,
+  "workflow_accuracy": 0.9433962264150944,
   "workflow_labels": [
     "direct",
     "verify_then_fallback"

train_report.json

@@ -1,88 +1,168 @@
 {
-  "device": "mps",
+  "device": "cuda",
   "history": [
     {
       "epoch": 0,
-      "loss": 4.6159288712910245
+      "loss": 4.668870312826974
     },
     {
       "epoch": 1,
-      "loss": 3.534461634499686
+      "loss": 3.560512134007045
     },
     {
       "epoch": 2,
-      "loss": 3.2642898644719804
+      "loss": 3.273119338921138
     },
     {
       "epoch": 3,
-      "loss": 3.095064878463745
+      "loss": 3.096318347113473
     },
     {
       "epoch": 4,
-      "loss": 2.8912287099020824
+      "loss": 2.871755446706499
     },
     {
       "epoch": 5,
-      "loss": 3.0201558896473477
+      "loss": 2.9902492335864475
     },
     {
       "epoch": 6,
-      "loss": 2.9031057953834534
+      "loss": 2.886969430106027
     },
     {
       "epoch": 7,
-      "loss": 2.837797522544861
+      "loss": 2.831721442086356
     },
     {
       "epoch": 8,
-      "loss": 2.824960572378976
+      "loss": 2.845917650631496
     },
     {
       "epoch": 9,
-      "loss": 2.7867482985769
+      "loss": 2.7882944686072215
     },
     {
       "epoch": 10,
-      "loss": 2.7380643912724087
+      "loss": 2.73999228647777
     },
     {
       "epoch": 11,
-      "loss": 2.7709683179855347
+      "loss": 2.7610760246004378
     },
     {
       "epoch": 12,
-      "loss": 2.8764453445162093
+      "loss": 2.8626872641699657
     },
     {
       "epoch": 13,
-      "loss": 2.5639974730355397
+      "loss": 2.54475006035396
     },
     {
       "epoch": 14,
-      "loss": 2.5871658495494296
+      "loss": 2.577164743627821
     },
     {
       "epoch": 15,
-      "loss": 2.4336864948272705
+      "loss": 2.4409775733947754
     },
     {
       "epoch": 16,
-      "loss": 2.5757188967296054
+      "loss": 2.575380103928702
     },
     {
       "epoch": 17,
-      "loss": 2.670148457799639
+      "loss": 2.6568912523133412
     },
     {
       "epoch": 18,
-      "loss": 2.7170623200280324
+      "loss": 2.7085645369121005
     },
     {
       "epoch": 19,
-      "loss": 2.8251984119415283
+      "loss": 2.8356578860964095
+    },
+    {
+      "epoch": 20,
+      "loss": 2.4540705425398692
+    },
+    {
+      "epoch": 21,
+      "loss": 2.572358787059784
+    },
+    {
+      "epoch": 22,
+      "loss": 2.3995534351893832
+    },
+    {
+      "epoch": 23,
+      "loss": 2.540985039302281
+    },
+    {
+      "epoch": 24,
+      "loss": 2.26006087235042
+    },
+    {
+      "epoch": 25,
+      "loss": 2.5964779428073337
+    },
+    {
+      "epoch": 26,
+      "loss": 2.4571213892527988
+    },
+    {
+      "epoch": 27,
+      "loss": 2.3653977683612277
+    },
+    {
+      "epoch": 28,
+      "loss": 2.2759930406297957
+    },
+    {
+      "epoch": 29,
+      "loss": 2.273615445409502
+    },
+    {
+      "epoch": 30,
+      "loss": 2.1414122751780917
+    },
+    {
+      "epoch": 31,
+      "loss": 2.3408056667872836
+    },
+    {
+      "epoch": 32,
+      "loss": 2.3234308872904097
+    },
+    {
+      "epoch": 33,
+      "loss": 2.2955648558480397
+    },
+    {
+      "epoch": 34,
+      "loss": 2.191408165863582
+    },
+    {
+      "epoch": 35,
+      "loss": 2.2341184190341403
+    },
+    {
+      "epoch": 36,
+      "loss": 2.3798889943531583
+    },
+    {
+      "epoch": 37,
+      "loss": 2.2818061879702976
+    },
+    {
+      "epoch": 38,
+      "loss": 2.1292204516274587
+    },
+    {
+      "epoch": 39,
+      "loss": 2.1550024236951555
     }
   ],
-  "output_dir": "research/models/20260628-qwen-small-logits-orchestrator-full-local",
+  "output_dir": "research/models/20260628-qwen-small-logits-orchestrator-full-gpu-l40s",
   "record_count": 53,
   "schema_version": "mempool.qwen_logits_orchestrator_train_report.v1",
   "worker_ids": [