Upload analysis__disc1000_panel_row_audit.json with huggingface_hub

analysis__disc1000_panel_row_audit.json

@@ -0,0 +1,201 @@
+{
+ "npz": "reports/eval__grasp_sft_v2_demosonly_1gpu_disc__step_001000__panel_v2_k4l2_euler10_draws1_stable.npz",
+ "checkpoint": "/home/ubuntu/checkpoints/finetune/grasp_sft_v2_demosonly_1gpu_disc/step_001000",
+ "wear_facts": {
+  "leg_molmo_norm": "checkpoint",
+  "flow_normalization_tag": "q01q99",
+  "train_per_dataset_flow_norm": false,
+  "per_dataset_stats_keys": [
+   "grasp_demos_v2/merged"
+  ],
+  "stats_note": "global stats RECOMPUTED at launch: exact pooled quantiles over 1750071 train-split frames of ['grasp_demos_v2/merged'], oriented like the source table; the original table is the source checkpoint's (/home/ubuntu/checkpoints/molmoact2-so101-released)",
+  "worn_table": {
+   "mean": [
+    0.09233876317739487,
+    -7.418671607971191,
+    22.1364803314209,
+    51.24595642089844,
+    82.55037689208984,
+    12.741997718811035
+   ],
+   "std": [
+    27.988689422607422,
+    40.790802001953125,
+    31.966609954833984,
+    20.678668975830078,
+    77.26783752441406,
+    19.013696670532227
+   ],
+   "q01": [
+    -54.11045837402344,
+    55.71287155151367,
+    -65.48938751220703,
+    11.65783405303955,
+    -157.21087646484375,
+    0.0
+   ],
+   "q99": [
+    50.82916259765625,
+    -113.50212097167969,
+    97.0,
+    95.00440979003906,
+    157.21087646484375,
+    41.692020416259766
+   ]
+  },
+  "verdict": "merged scheme: flow_denormalize_chunk(per_dataset=False) \u2014 every item wore the recomputed-at-launch demos-only global table; per_dataset_stats is never consulted at eval"
+ },
+ "caveats": [
+  "re-wear counterfactuals re-express the OUTPUT side only: the state input was normalized/binned through the worn demos table when the model ran, so state-side OOD is not undone \u2014 they bound the wear share, they do not re-run the model",
+  "per-repo rows are fit on the panel's own truth (np.quantile linear over valid chunk elements, duplicate-weighted by window overlap) \u2014 oracle-ish rows, optimistic vs a deployment row estimate"
+ ],
+ "anchors_reproduced": {
+  "bijou@1000": 58.143384213383385,
+  "state-copy": 8.367757873526733
+ },
+ "box_audit": {
+  "truth_outside_frac_per_joint": {
+   "shoulder_pan": 0.07369556616706689,
+   "shoulder_lift": 0.7296525803453955,
+   "elbow_flex": 0.5511537718402385,
+   "wrist_flex": 0.19014923587917598,
+   "wrist_roll": 0.027241068389043766,
+   "gripper": 0.26894430796691415
+  },
+  "truth_any_joint_outside_frac": 0.8583248226558834,
+  "floor_chunk_mae": 14.401469230651855,
+  "floor_per_joint": {
+   "shoulder_pan": 1.4592567682266235,
+   "shoulder_lift": 52.04825210571289,
+   "elbow_flex": 24.017120361328125,
+   "wrist_flex": 6.615574359893799,
+   "wrist_roll": 0.5258612036705017,
+   "gripper": 1.74805748462677
+  },
+  "pred_edge_saturation_per_joint": {
+   "shoulder_pan": 0.000588846872211028,
+   "shoulder_lift": 0.0014905186452841646,
+   "elbow_flex": 0.00014491153495818266,
+   "wrist_flex": 0.002872928685123336,
+   "wrist_roll": 0.0016561318280935162,
+   "gripper": 0.11585562210751976
+  }
+ },
+ "actual_per_joint_mae": {
+  "shoulder_pan": 23.462173461914062,
+  "shoulder_lift": 104.43495178222656,
+  "elbow_flex": 99.3005599975586,
+  "wrist_flex": 27.106822967529297,
+  "wrist_roll": 71.17012023925781,
+  "gripper": 23.381319046020508
+ },
+ "rewear": {
+  "round_trip_worst_abs_deg": 1.52587890625e-05,
+  "repo_rows_chunk_mae": 27.395334001919885,
+  "repo_rows_per_joint": {
+   "shoulder_pan": 21.10439109802246,
+   "shoulder_lift": 41.8028678894043,
+   "elbow_flex": 36.7511100769043,
+   "wrist_flex": 22.337148666381836,
+   "wrist_roll": 22.218639373779297,
+   "gripper": 20.153600692749023
+  },
+  "released_table_chunk_mae": 54.399765688015435,
+  "released_table_per_joint": {
+   "shoulder_pan": 24.25238609313965,
+   "shoulder_lift": 105.2990951538086,
+   "elbow_flex": 98.5770263671875,
+   "wrist_flex": 27.263837814331055,
+   "wrist_roll": 46.14845275878906,
+   "gripper": 24.851577758789062
+  },
+  "null_repo_midpoint_chunk_mae": 25.154475696265422,
+  "n_repo_rows": 838
+ },
+ "demos_prior_collapse": {
+  "pred_vs_demos_mean_mae": 22.636736684240066,
+  "truth_vs_demos_mean_mae": 58.19400664292878
+ },
+ "per_repo_worst10": [
+  {
+   "repo": "Cidoyi/so100_all_notes",
+   "elements": 215,
+   "mae": 121.31421661376953,
+   "floor_mae": 42.90715408325195,
+   "rewear_repo_mae": 3.015285015106201,
+   "truth_outside_frac": 0.8333333333333334
+  },
+  {
+   "repo": "Cidoyi/so100_all_notes_3",
+   "elements": 354,
+   "mae": 115.09871673583984,
+   "floor_mae": 38.250736236572266,
+   "rewear_repo_mae": 8.197112083435059,
+   "truth_outside_frac": 0.8333333333333334
+  },
+  {
+   "repo": "roboticshack/team11_pianobot",
+   "elements": 452,
+   "mae": 107.51007080078125,
+   "floor_mae": 35.70561981201172,
+   "rewear_repo_mae": 15.485651016235352,
+   "truth_outside_frac": 0.7901917404129793
+  },
+  {
+   "repo": "Cidoyi/so100_all_notes_1",
+   "elements": 389,
+   "mae": 107.12379455566406,
+   "floor_mae": 34.05473327636719,
+   "rewear_repo_mae": 15.588462829589844,
+   "truth_outside_frac": 0.786203941730934
+  },
+  {
+   "repo": "kantine/flip_A2",
+   "elements": 120,
+   "mae": 101.39452362060547,
+   "floor_mae": 55.31953048706055,
+   "rewear_repo_mae": 26.316650390625,
+   "truth_outside_frac": 0.8333333333333334
+  },
+  {
+   "repo": "siqiLi/so100_test16",
+   "elements": 548,
+   "mae": 96.68895721435547,
+   "floor_mae": 32.16460418701172,
+   "rewear_repo_mae": 41.742645263671875,
+   "truth_outside_frac": 0.40206812652068125
+  },
+  {
+   "repo": "juni3227/so100_test_cap02_sideway",
+   "elements": 720,
+   "mae": 96.2467269897461,
+   "floor_mae": 28.773996353149414,
+   "rewear_repo_mae": 33.46946716308594,
+   "truth_outside_frac": 0.47824074074074074
+  },
+  {
+   "repo": "shuohsuan/grasp_data_tg_10",
+   "elements": 115,
+   "mae": 95.52738952636719,
+   "floor_mae": 29.273643493652344,
+   "rewear_repo_mae": 16.6530704498291,
+   "truth_outside_frac": 0.3289855072463768
+  },
+  {
+   "repo": "kantine/flip_A3",
+   "elements": 120,
+   "mae": 93.62525939941406,
+   "floor_mae": 41.843135833740234,
+   "rewear_repo_mae": 25.944631576538086,
+   "truth_outside_frac": 0.7041666666666667
+  },
+  {
+   "repo": "bjb7/so101_pen_mug_10_13",
+   "elements": 120,
+   "mae": 93.45197296142578,
+   "floor_mae": 27.867626190185547,
+   "rewear_repo_mae": 17.589902877807617,
+   "truth_outside_frac": 0.4777777777777778
+  }
+ ]
+}