| { |
| "title": "Qwen3-Omni Full-Parameter Feasibility Gates", |
| "generated_at_utc": "2026-06-18T12:53:13+00:00", |
| "status": "pass", |
| "decision": "full_parameter_feasible_for_guarded_short_runs_not_promoted", |
| "interpretation": "The full-parameter gates prove that Qwen3-Omni full-parameter FSDP can load, prepare, run backward/optimizer steps, and complete guarded pilots up to 256 optimizer steps on an 8-GPU remote worker. They do not prove a production full-parameter fine-tune, and they intentionally save no full checkpoints or public weights.", |
| "aggregate": { |
| "run_count": 7, |
| "passed_run_count": 6, |
| "preempted_run_count": 1, |
| "review_or_missing_run_count": 0, |
| "completed_full_parameter_train_steps": 489, |
| "longest_passed_run_id": "xperience10m_qwen3_omni_128ep_fullparam_pilot256_after_qwen_v6_preemptible_8gpu_20260611", |
| "longest_passed_steps": 256, |
| "num_processes": [ |
| 8 |
| ], |
| "checkpoint_saved": false |
| }, |
| "runs": [ |
| { |
| "id": "fullparam_smoke_1step", |
| "title": "Full-Parameter 1-Step Feasibility Smoke", |
| "status": "passed", |
| "scope": "1 optimizer step over 8 train samples", |
| "summary_path": "results/omni_finetune/xperience10m_qwen3_omni_128ep_fullparam_smoke_preemptible_8gpu_20260609/fullparam_feasibility_summary.json", |
| "run_id": "xperience10m_qwen3_omni_128ep_fullparam_smoke_preemptible_8gpu_20260609", |
| "purpose": "Full-parameter Qwen3-Omni feasibility gate: load, FSDP prepare, backward/optimizer step, and no checkpoint save.", |
| "tuning_mode": "full", |
| "training_objective": null, |
| "num_processes": 8, |
| "num_train_samples": 8, |
| "configured_max_train_steps": 1, |
| "observed_train_steps": 1, |
| "first_step_loss": null, |
| "final_step_loss": 1.2726006507873535, |
| "epoch_train_loss": 1.2726006507873535, |
| "min_step_loss": null, |
| "max_step_loss": null, |
| "model_load_seconds": null, |
| "accelerator_prepare_seconds": null, |
| "train_loop_seconds": null, |
| "save_mode": "none", |
| "checkpoint_saved": false, |
| "checkpoint_policy": "no full-parameter checkpoint or public weights; save_mode=none", |
| "preempt_event": null, |
| "parent_resume_event": null |
| }, |
| { |
| "id": "fullparam_shorttrain8", |
| "title": "Full-Parameter 8-Step Short Train", |
| "status": "passed", |
| "scope": "8 optimizer steps over 64 train samples", |
| "summary_path": "results/omni_finetune/xperience10m_qwen3_omni_128ep_fullparam_shorttrain8_preemptible_8gpu_20260609/fullparam_shorttrain8_summary.json", |
| "run_id": "xperience10m_qwen3_omni_128ep_fullparam_shorttrain8_preemptible_8gpu_20260609", |
| "purpose": "guarded full-parameter Qwen3-Omni short train using all 8 remote GPUs while Qwen v5 export was CPU-bound", |
| "tuning_mode": "full", |
| "training_objective": "structured_episode_understanding_json_qa", |
| "num_processes": 8, |
| "num_train_samples": 64, |
| "configured_max_train_steps": 8, |
| "observed_train_steps": 8, |
| "first_step_loss": 1.2701153755187988, |
| "final_step_loss": 1.180522084236145, |
| "epoch_train_loss": 1.2190196067094803, |
| "min_step_loss": 1.1532728672027588, |
| "max_step_loss": 1.2701153755187988, |
| "model_load_seconds": 111.21318578720093, |
| "accelerator_prepare_seconds": null, |
| "train_loop_seconds": null, |
| "save_mode": "none", |
| "checkpoint_saved": false, |
| "checkpoint_policy": "no full-parameter checkpoint or public weights; save_mode=none", |
| "preempt_event": null, |
| "parent_resume_event": null |
| }, |
| { |
| "id": "fullparam_pilot32", |
| "title": "Full-Parameter 32-Step Pilot", |
| "status": "passed", |
| "scope": "32 optimizer steps over 256 train samples", |
| "summary_path": "results/omni_finetune/xperience10m_qwen3_omni_128ep_fullparam_pilot32_preemptible_8gpu_20260609/fullparam_pilot32_summary.json", |
| "run_id": "xperience10m_qwen3_omni_128ep_fullparam_pilot32_preemptible_8gpu_20260609", |
| "purpose": "guarded full-parameter Qwen3-Omni 32-step pilot using all 8 remote GPUs while Qwen v5 export was CPU-bound", |
| "tuning_mode": "full", |
| "training_objective": "structured_episode_understanding_json_qa", |
| "num_processes": 8, |
| "num_train_samples": 256, |
| "configured_max_train_steps": 32, |
| "observed_train_steps": 32, |
| "first_step_loss": 1.2583622932434082, |
| "final_step_loss": 0.2206273376941681, |
| "epoch_train_loss": 0.8451133379712701, |
| "min_step_loss": 0.2206273376941681, |
| "max_step_loss": 1.26246976852417, |
| "model_load_seconds": 116.43117642402649, |
| "accelerator_prepare_seconds": 1122.333820104599, |
| "train_loop_seconds": 440.2827479839325, |
| "save_mode": "none", |
| "checkpoint_saved": false, |
| "checkpoint_policy": "no full-parameter checkpoint or public weights; save_mode=none", |
| "preempt_event": null, |
| "parent_resume_event": null |
| }, |
| { |
| "id": "fullparam_pilot64", |
| "title": "Full-Parameter 64-Step Pilot", |
| "status": "passed", |
| "scope": "64 optimizer steps over 512 train samples", |
| "summary_path": "results/omni_finetune/xperience10m_qwen3_omni_128ep_fullparam_pilot64_preemptible_8gpu_20260609/fullparam_pilot64_summary.json", |
| "run_id": "xperience10m_qwen3_omni_128ep_fullparam_pilot64_preemptible_8gpu_20260609", |
| "purpose": "guarded full-parameter Qwen3-Omni 64-step pilot using all 8 remote GPUs while Qwen v5 long multiscale export was CPU-bound", |
| "tuning_mode": "full", |
| "training_objective": "structured_episode_understanding_json_qa", |
| "num_processes": 8, |
| "num_train_samples": 512, |
| "configured_max_train_steps": 64, |
| "observed_train_steps": 64, |
| "first_step_loss": 1.2597532272338867, |
| "final_step_loss": 0.011219973675906658, |
| "epoch_train_loss": 0.4434075650788145, |
| "min_step_loss": 0.011219973675906658, |
| "max_step_loss": 1.2597532272338867, |
| "model_load_seconds": 80.75278043746948, |
| "accelerator_prepare_seconds": 811.6013672351837, |
| "train_loop_seconds": 903.8602497577667, |
| "save_mode": "none", |
| "checkpoint_saved": false, |
| "checkpoint_policy": "no full-parameter checkpoint or public weights; save_mode=none", |
| "preempt_event": null, |
| "parent_resume_event": null |
| }, |
| { |
| "id": "fullparam_pilot128_preempted", |
| "title": "Full-Parameter 128-Step Opportunistic Pilot", |
| "status": "preempted_for_qwen_v5_handoff", |
| "scope": "planned 128 optimizer steps over 1024 train samples; preempted for Qwen v5 handoff", |
| "summary_path": "results/omni_finetune/xperience10m_qwen3_omni_128ep_fullparam_pilot128_preemptible_8gpu_20260609/fullparam_pilot128_summary.json", |
| "run_id": "xperience10m_qwen3_omni_128ep_fullparam_pilot128_preemptible_8gpu_20260609", |
| "purpose": "opportunistic guarded full-parameter Qwen3-Omni 128-step pilot launched only while Qwen v5 export was CPU-bound; preempted for main Qwen v5 handoff", |
| "tuning_mode": "full", |
| "training_objective": "structured_episode_understanding_json_qa", |
| "num_processes": 8, |
| "num_train_samples": 1024, |
| "configured_max_train_steps": 128, |
| "observed_train_steps": 0, |
| "first_step_loss": null, |
| "final_step_loss": null, |
| "epoch_train_loss": null, |
| "min_step_loss": null, |
| "max_step_loss": null, |
| "model_load_seconds": 65.33229899406433, |
| "accelerator_prepare_seconds": null, |
| "train_loop_seconds": null, |
| "save_mode": "none", |
| "checkpoint_saved": false, |
| "checkpoint_policy": "no full-parameter checkpoint or public weights; save_mode=none", |
| "preempt_event": { |
| "event": "fullparam_pilot128_preempt_for_qwen_v5_handoff", |
| "guard_pid": 3757690, |
| "kind": "fullparam_pilot128_handoff_guard", |
| "time": 1780954910, |
| "watched_pid": 3753782 |
| }, |
| "parent_resume_event": { |
| "event": "fullparam_pilot128_resumed_parent_after_preempt", |
| "guard_pid": 3757690, |
| "kind": "fullparam_pilot128_handoff_guard", |
| "time": 1780954925, |
| "watched_pid": 3753782 |
| } |
| }, |
| { |
| "id": "fullparam_pilot128_after_qwen_v5", |
| "title": "Full-Parameter 128-Step Post-Qwen-v5 Pilot", |
| "status": "passed", |
| "scope": "128 optimizer steps over 1024 train samples after verified Qwen v5 handoff", |
| "summary_path": "results/omni_finetune/xperience10m_qwen3_omni_128ep_fullparam_pilot128_after_qwen_v5_preemptible_8gpu_20260609/training_metadata.json", |
| "progress_path": "results/omni_finetune/xperience10m_qwen3_omni_128ep_fullparam_pilot128_after_qwen_v5_preemptible_8gpu_20260609/progress.jsonl", |
| "run_id": "xperience10m_qwen3_omni_128ep_fullparam_pilot128_after_qwen_v5_preemptible_8gpu_20260609", |
| "purpose": "post_verified_qwen_v5_full_parameter_feasibility_pilot", |
| "tuning_mode": "full", |
| "training_objective": "structured_episode_understanding_json_qa", |
| "num_processes": 8, |
| "num_train_samples": 1024, |
| "configured_max_train_steps": 128, |
| "observed_train_steps": 128, |
| "first_step_loss": 1.2273844480514526, |
| "final_step_loss": 0.0136940386146307, |
| "epoch_train_loss": 0.21579630990163423, |
| "min_step_loss": 0.004702376667410135, |
| "max_step_loss": 1.2273844480514526, |
| "model_load_seconds": null, |
| "accelerator_prepare_seconds": null, |
| "train_loop_seconds": null, |
| "save_mode": "none", |
| "checkpoint_saved": false, |
| "checkpoint_policy": "no full-parameter checkpoint or public weights; save_mode=none", |
| "preempt_event": null, |
| "parent_resume_event": null, |
| "progress_events": { |
| "max_steps_reached": true, |
| "save_skipped": true, |
| "complete": true |
| } |
| }, |
| { |
| "id": "fullparam_pilot256_after_qwen_v6", |
| "title": "Full-Parameter 256-Step Post-Qwen-v6 Pilot", |
| "status": "passed", |
| "scope": "256 optimizer steps over 2048 train samples after verified Qwen v6 handoff", |
| "summary_path": "results/omni_finetune/xperience10m_qwen3_omni_128ep_fullparam_pilot256_after_qwen_v6_preemptible_8gpu_20260611/training_metadata.json", |
| "progress_path": "results/omni_finetune/xperience10m_qwen3_omni_128ep_fullparam_pilot256_after_qwen_v6_preemptible_8gpu_20260611/progress.jsonl", |
| "run_id": "xperience10m_qwen3_omni_128ep_fullparam_pilot256_after_qwen_v6_preemptible_8gpu_20260611", |
| "purpose": "post_verified_qwen_v6_full_parameter_feasibility_pilot", |
| "tuning_mode": "full", |
| "training_objective": "structured_episode_understanding_json_qa", |
| "num_processes": 8, |
| "num_train_samples": 2048, |
| "configured_max_train_steps": 256, |
| "observed_train_steps": 256, |
| "first_step_loss": 1.2249484062194824, |
| "final_step_loss": 0.009560374543070793, |
| "epoch_train_loss": 0.1157912792496063, |
| "min_step_loss": 0.002165592508390546, |
| "max_step_loss": 1.2249484062194824, |
| "model_load_seconds": null, |
| "accelerator_prepare_seconds": null, |
| "train_loop_seconds": null, |
| "save_mode": "none", |
| "checkpoint_saved": false, |
| "checkpoint_policy": "no full-parameter checkpoint or public weights; save_mode=none", |
| "preempt_event": null, |
| "parent_resume_event": null, |
| "progress_events": { |
| "max_steps_reached": true, |
| "save_skipped": true, |
| "complete": true |
| } |
| } |
| ], |
| "publication_policy": { |
| "public_summary_allowed": true, |
| "publish_full_parameter_weights": false, |
| "publish_full_checkpoints": false, |
| "reason": "All completed full-parameter gate runs used save_mode=none; the preempted pilot saved nothing. These are feasibility evidence only." |
| }, |
| "next_steps": [ |
| "Keep the verified Qwen3-Omni LoRA adapter as the published production result for the 128-episode suite.", |
| "For a production full-parameter run, add a sharded checkpoint/resume plan before any long training launch.", |
| "Run a separate checkpointed full-parameter pilot only when GPUs are not needed by verified LoRA evaluation/publication work." |
| ] |
| } |
|
|
