| { |
| "dataset": "data/rmbench_lerobot_full/battery_try", |
| "checkpoint_dir": "training/models/Wan2.2-TI2V-5B", |
| "action_dit_pretrained_path": "none", |
| "prompt_cache": null, |
| "output_dir": "training/experiments/rmbench_kairos_wam_v15_gist4_gated_r4_pointwise_full_kv_d640_battery_try_nextstate_1x8_50k_run1/outputs", |
| "resume": "none", |
| "required_cameras": "mosaic", |
| "device": "cuda", |
| "dtype": "bfloat16", |
| "mixed_precision": "bf16", |
| "batch_size": 1, |
| "gradient_accumulation_steps": 1, |
| "num_workers": 4, |
| "learning_rate": 0.0002, |
| "weight_decay": 0.01, |
| "beta1": 0.9, |
| "beta2": 0.95, |
| "optimizer_fused": false, |
| "optimizer_foreach": "auto", |
| "max_grad_norm": 1.0, |
| "num_epochs": 1000, |
| "max_steps": 50000, |
| "warmup_ratio": 0.05, |
| "log_every": 10, |
| "save_every": 1000, |
| "keep_last": 10, |
| "skip_final_checkpoint": false, |
| "skip_trainer_state_checkpoint": false, |
| "profile_timings": false, |
| "profile_microbatches": false, |
| "profile_all_ranks": false, |
| "profile_warmup_steps": 3, |
| "profile_log_every": 1, |
| "torch_profile_dir": null, |
| "torch_profile_wait": 1, |
| "torch_profile_warmup": 1, |
| "torch_profile_active": 3, |
| "torch_profile_repeat": 1, |
| "torch_profile_record_shapes": false, |
| "torch_profile_memory": false, |
| "torch_profile_with_stack": false, |
| "seed": 42, |
| "tokenizer_max_len": 128, |
| "keep_text_encoder_loaded": false, |
| "image_size": "160,192", |
| "disable_mot_checkpoint_mixed_attn": false, |
| "noise_beta_alpha": 0.0, |
| "noise_beta_beta": 1.0, |
| "proprio_mode": "native", |
| "action_space": "aloha_agilex_absolute", |
| "model_variant": "wam", |
| "video_backbone": "kairos", |
| "kairos_video_pretrained": "training/kairos/step-4500.safetensors", |
| "kairos_action_pretrained": "training/checkpoints/KairosActionDiT_from_video_step4500_alphascale_14dim_d640_ffn2560_attn2560.safetensors", |
| "kairos_action_model_dim": 640, |
| "kairos_action_ffn_dim": 2560, |
| "kairos_action_attn_hidden_dim": 2560, |
| "allow_random_kairos_action_init": false, |
| "kairos_vae_pretrained": "training/pretrained/Wan-AI/Wan2.1-T2V-1.3B/Wan2.1_VAE.pth", |
| "kairos_text_encoder_path": "training/pretrained/Qwen/Qwen2.5-VL-7B-Instruct", |
| "wandb": false, |
| "wandb_project": "little-wam", |
| "wandb_entity": null, |
| "wandb_name": null, |
| "wandb_group": null, |
| "eval_every": 0, |
| "eval_num_inference_steps": 10, |
| "faithfulness": { |
| "spec": "memorywam_kairos", |
| "spec_doc": "docs/memorywam-kairos-design.md#7c-faithfulness-deviation-ledger", |
| "contract_lineage": "v15", |
| "contract_version": "v15_gist4_gated_episode_d640_0init_4recent_pointwise_adaptive_full", |
| "memory_bootstrap_opt_in": true, |
| "runtime_debug": { |
| "zero_optimizer_offload_device": "none", |
| "speed_comparable_to_standard_gpu_training": true |
| }, |
| "backbone_provenance": "Kairos 3.1-pre WAM/Libero config (2560/32/20), not the G1-robot 1024/28/484 config", |
| "train_infer_fix": "absolute_video_rope_with_full_video3d_action_rope", |
| "train_infer_fix_components": [ |
| "per_query_role_routing", |
| "absolute_pinned_3d_rope_video_cache", |
| "full_video3d_temporal_fraction_marker_action_rope" |
| ], |
| "rope_basis": "video3d_absolute_pinned", |
| "action_rope": "video3d_temporal_fraction_marker", |
| "rope_cache_append_only": false, |
| "surviving_kv_immutable": true, |
| "cache_update": "append_new_delete_expired_full_no_recompute", |
| "geometry": { |
| "video_chunk_frames": 1, |
| "latent_frames_per_chunk": 1, |
| "action_horizon": 16, |
| "chunk_stride": 16, |
| "video_objective": "per_frame_autoregressive_F1" |
| }, |
| "memory_policy": { |
| "n_init": 0, |
| "n_recent": 4, |
| "mv": 4, |
| "gist_tokens": 4, |
| "min_target_chunk": 0, |
| "clean_aug_prob": 1.0, |
| "max_prefix_frames": 128, |
| "compressed_repr": "gist_only", |
| "gated_memory": "episode", |
| "router_mode": "delta_pointwise_threshold", |
| "delta_threshold": 1.5, |
| "physical_evict_full": true, |
| "physical_memory_unit": "observed_latent_frame", |
| "latent_gid": "chunk_idx" |
| }, |
| "compressed_memory": { |
| "representation": "gist_only", |
| "gist_tokens": 4, |
| "materialized_levels": [ |
| "gist" |
| ], |
| "router": { |
| "mode": "delta_pointwise_threshold", |
| "enabled": true, |
| "selection_signal": "positive_raw_dt", |
| "token_filter": "clean_committed_full_only", |
| "patch_pool": "mean", |
| "head_reduce": "raw_l2", |
| "threshold": 1.5, |
| "pass1": "no_grad_streaming", |
| "pass2": "monolithic_frozen_route_no_delta_capture", |
| "dynamic_anchor_cap": null, |
| "selected_anchor_demotion": false, |
| "decision_rule": "frame_score_gte_threshold", |
| "score_scope": "single_expiring_frame", |
| "accumulator": false, |
| "pending_capacity": 0, |
| "delta_layer_resolution": "pending_model_init" |
| }, |
| "physical_evict_full": true |
| }, |
| "action_representation": { |
| "status": "fixed", |
| "ours": "absolute_joint_14d", |
| "paper": "absolute_joint_14d", |
| "action_space_name": "aloha_agilex_absolute", |
| "action_space_tag": "aloha_agilex_absolute_joint_14d", |
| "fail_loud_on_resume": true, |
| "note": "ACTION_SPACE=aloha_agilex_absolute is the canonical paper-faithful RMBench 14D joint action representation; ACTION_SPACE=aloha_agilex remains a legacy chunk-delta ablation." |
| }, |
| "memory_window": { |
| "status": "fixed", |
| "ours": "no permanent anchors; recent window has 4 FULL chunks including current; older frames use learned gist tokens only; video GATED state persists across clean commits and noisy targets read clock-correct state without committing", |
| "paper": "V15-baseline-derived GIST-only static retention", |
| "n_init": 0, |
| "n_recent": 4, |
| "clean_full_attended": 4, |
| "wam_only": true, |
| "note": "V15 GIST4/GATED-R4 has n_init=0 and n_recent=4 by default. The exact resolved values are recorded here and in checkpoint metadata; streaming physically removes expired FULL KV after verifying that its GIST KV exists. No L1 representation is constructed or retained." |
| }, |
| "kairos_forced_deviations": [ |
| "8 GATED (linear-attn) layers in the 32-block schedule (Kairos arch, not paper's 30 uniform softmax)", |
| "decomposed video/action two-tower forward (consequence of the 8 GATED layers)", |
| "Wan2.1 16-ch VAE vs paper Wan2.2 48-ch latent", |
| "Qwen2.5-VL text encoder vs paper T5" |
| ], |
| "noise_schedule": { |
| "objective": "continuous flow-matching, 1000 steps (paper-faithful)", |
| "timestep_sampling": { |
| "status": "fixed", |
| "ours": "shifted logit-normal (SD3-style: u~N(0,1) -> sigmoid -> phi-shift)", |
| "paper": "shifted logit-normal", |
| "wam_only": true, |
| "note": "memwam loss only; idm/joint/base keep uniform+phi-shift (untouched)" |
| }, |
| "video_flow_shift": { |
| "status": "fixed", |
| "ours": 5.0, |
| "paper": 5.0, |
| "wam_only": true, |
| "fail_loud_on_resume": true, |
| "note": "was dynamic kairos_shift_from_shape ~1.67 at 480-token F=1 latent; now fixed (resolved from KAIROS_WAM_VIDEO_SHIFT, default 5.0)" |
| }, |
| "action_flow_shift": { |
| "status": "fixed", |
| "ours": 1.0, |
| "paper": 1.0, |
| "fail_loud_on_resume": true, |
| "note": "already 1.0; kept and now sampled via the logit-normal schedule (resolved from KAIROS_WAM_ACTION_SHIFT, default 1.0)" |
| }, |
| "loss_reweight": { |
| "status": "fixed", |
| "ours": "unit weight (1.0); emphasis carried once by logit-normal sampling (SD3 single-application)", |
| "paper": "\u00a74.1 'reweighted by the scheduler's logit-normal training weight' (prose names the schedule = the sampling; no explicit weighting equation)", |
| "wam_only": true, |
| "metadata_value": "scheduler_logit_normal_single", |
| "flag": "KAIROS_WAM_LOSS_REWEIGHT={scheduler_logit_normal|unit} (default scheduler_logit_normal; unit = compat/debug, NOT paper-faithful)", |
| "note": "PAPER AMBIGUITY (no equation; \u00a74.1 only NAMES the logit-normal training weight): we follow SD3 single-application. \u00a74.1 'reweighted by the scheduler's logit-normal weight' and 'unit weight' are the SAME single application \u2014 the emphasis lives in the logit-normal SAMPLING, not a SECOND density multiply. The blueprint STEP-6 'non-constant training_weight ON TOP of logit-normal sampling' would DOUBLE-apply the emphasis (\u221d \u03c0_ln\u00b2, mean \u22481.29) \u2014 the v8 bug G1 fixed \u2014 so we keep single-application (unit weight). idm/joint/base keep the centred Gaussian bump (untouched)." |
| } |
| }, |
| "observation_window": { |
| "status": "fixed", |
| "ours": "4-raw-frame window {3,7,11,15} of the PREVIOUS segment (anchor + {-12,-8,-4,0}) VAE-encoded into one F=1 latent; chunk 0 seeded from the repeated initial mosaic", |
| "paper": "appendix A.1 4-raw-frame window {3,7,11,15} aggregated into one latent frame", |
| "causal_alignment": "train_obs(chunk_i) == deploy_obs(chunk_i); conditioned on frames <= start_i (offset 0 = anchor, the pre-action obs)", |
| "ledger": "docs/memorywam-kairos-design.md \u00a77c C (Latent temporal window) + \u00a77c F1", |
| "gate": "tests/test_kairos_obs_window_causal_alignment.py (train obs == deploy obs per chunk, no future leak)", |
| "load_fail_loud_keys": [ |
| "action_horizon", |
| "obs_window_mode", |
| "obs_window_substep_offsets", |
| "obs_window_prev_segment", |
| "obs_window_prev_window_offsets" |
| ], |
| "wam_only": true, |
| "note": "supersedes the earlier 'single conditioning latent frame' deviation AND the earlier same-chunk {3,7,11,15} leak (v5); requires the v6 re-train (v5 trained the leaky alignment)" |
| }, |
| "sim_deploy_denoise_steps": { |
| "status": "fixed", |
| "ours_realworld_default": 10, |
| "ours_sim": 50, |
| "paper_sim": 50, |
| "paper_realworld": 10, |
| "control": "robot.deployment.policy_server --num-inference-steps/--sim-denoise-steps (per-request params.num_inference_steps overrides)", |
| "note": "sim closed-loop eval pins the SERVER's resolved default to 50 via the deploy recipe (03_deploy_server.sh --num-inference-steps 50, SIM_NUM_INFERENCE_STEPS), not relying solely on the RMBench client's per-request value; per-request params.num_inference_steps still overrides" |
| }, |
| "open_deviations": [], |
| "resolved_deviations": [ |
| { |
| "aspect": "action_rope_basis", |
| "ours": "full video 3-D temporal-fraction action RoPE: action tokens use f=c+(p+1)/(T+1) and one constant non-pixel h/w marker from the video's f/h/w RoPE basis", |
| "paper": "action queries share the video's full 3-D (f,h,w) RoPE basis", |
| "deviation": "none; temporal fractional action slots and the non-pixel h/w marker follow the LingBot-VA released-code convention for action coordinates", |
| "action_video_spatial_cross_term": "R_f(c+frac_p-gid) * R_h(marker_h-i) * R_w(marker_w-j)", |
| "train_infer_consistent": true, |
| "class": "F", |
| "ledger": "docs/memorywam-kairos-design.md#7c-faithfulness-deviation-ledger (C + F3)" |
| } |
| ] |
| } |
| } |
