Upload two_stream_attn_v1_finetune_20260513T160347Z

README.md

@@ -17,7 +17,7 @@ metrics:
 - accuracy
 - f1
 model-index:
-- name: two_stream_attn_v1_finetune_20260513T093937Z
+- name: two_stream_attn_v1_finetune_20260513T160347Z
   results:
   - task:
       type: gesture-recognition
@@ -26,15 +26,15 @@ model-index:
       type: IPN-Hand
     metrics:
     - type: accuracy
-      value: 0.9648
+      value: 0.9618
     - type: f1
-      value: 0.9638
+      value: 0.9632
 ---
 
-# two_stream_attn_v1_finetune_20260513T093937Z
+# two_stream_attn_v1_finetune_20260513T160347Z
 
 A real-time hand gesture classifier trained on
-a Hybrid Jester+IPN gesture dataset (Jester dynamic classes + IPN pointing classes).
+an LD-CONG Hybrid gesture dataset (LD-CONG unique gestures + Jester zooms + IPN pointing, W=16 frames).
 
 This model is part of the **Maestro** pipeline that enables touchless
 control of presentation and meeting software through hand gestures captured from a
@@ -43,12 +43,12 @@ standard webcam using MediaPipe for landmark extraction.
 ## Model Description
 
 - **Architecture**: EnhancedTwoStreamLSTM (BiLSTM h=96×2, MHA 8 heads, proj=96, mean+max pool, MLP gate)
-- **Parameters**: 2,099,434
-- **Input**: `(batch, 32, 147)`
-    — 32-frame sliding window at 30 FPS ≈ 1067 ms
-- **Output**: Softmax logits over 10 gesture classes
+- **Parameters**: 2,100,206
+- **Input**: `(batch, 16, 147)`
+    — 16-frame sliding window at 30 FPS ≈ 533 ms
+- **Output**: Softmax logits over 14 gesture classes
 - **Inference latency**: < 1 ms per call (CPU, single sample)
-- **Feature schema**: `feature-schema-v5`
+- **Feature schema**: `feature-schema-v1`
 
 ## Architecture
 
@@ -78,7 +78,7 @@ Input (B, T=32, 147)
     │   ctx_b  = LN(ctx_b × gate_b + ctx_b)
     │
     └─ cat(ctx_a, ctx_b) → (384,)
-       LN → Linear(384→192) → GELU → Dropout → Linear(192→10)
+       LN → Linear(384→192) → GELU → Dropout → Linear(192→14)
 ```
 
 **Design rationale:**
@@ -91,33 +91,41 @@ Input (B, T=32, 147)
 
 | Class | Description |
 |-------|-------------|
-| `unknown` | Background / transition / no gesture |
+| `palm` | Open palm held flat toward camera (static hand shape) |
+| `fist` | Closed fist (all fingers curled, thumb tucked) |
+| `thumb_up` | Thumbs-up — thumb extended upward, other fingers closed |
+| `pinch` | Thumb and index finger brought together (pinch grip) |
+| `click` | Brief tap / click gesture — index finger snaps toward thumb |
+| `swiping_right` | Horizontal swipe from left to right |
+| `swiping_left` | Horizontal swipe from right to left |
+| `swiping_down` | Vertical swipe downward |
+| `swiping_up` | Vertical swipe upward |
+| `zooming_in_full_hand` | Pinch-open / spread fingers away from each other |
+| `zooming_out_full_hand` | Pinch-close / bring fingers together |
 | `point_one` | Single-finger pointing gesture (continuous laser-pointer control) |
 | `point_two` | Two-finger pointing gesture (continuous annotation-pen control) |
-| `stop_sign` | Static open palm facing camera (Jester class) |
-| `swiping_down` | Vertical swipe downward (Jester class) |
-| `swiping_left` | Horizontal swipe from right to left (Jester class) |
-| `swiping_right` | Horizontal swipe from left to right (Jester class) |
-| `swiping_up` | Vertical swipe upward (Jester class) |
-| `zooming_in_full_hand` | Pinch-open / spread fingers away from each other (Jester class) |
-| `zooming_out_full_hand` | Pinch-close / bring fingers together (Jester class) |
+| `unknown` | Background / transition / no gesture |
 
 ## Gesture Usage In Presentation System
 
 | Class | Mode | Command | Runtime handling |
 |-------|------|---------|------------------|
-| `unknown` | `discrete` | `no_action` | No-op background class |
-| `point_one` | `continuous` | `—` | Continuous tracker: LaserPointerTracker (bypasses discrete dispatcher) |
-| `point_two` | `continuous` | `—` | Continuous tracker: AnnotationPenTracker (bypasses discrete dispatcher) |
-| `stop_sign` | `discrete` | `erase_annotations` | Discrete command via GestureActivationController → CommandDispatcher |
-| `swiping_down` | `discrete` | `stop_presentation` | Discrete command via GestureActivationController → CommandDispatcher |
-| `swiping_left` | `discrete` | `previous_slide` | Discrete command via GestureActivationController → CommandDispatcher |
+| `palm` | `discrete` | `erase_annotations` | Discrete command via GestureActivationController → CommandDispatcher |
+| `fist` | `discrete` | `no_action` | No-op background class |
+| `thumb_up` | `discrete` | `no_action` | No-op background class |
+| `pinch` | `discrete` | `no_action` | No-op background class |
+| `click` | `discrete` | `no_action` | No-op background class |
 | `swiping_right` | `discrete` | `next_slide` | Discrete command via GestureActivationController → CommandDispatcher |
+| `swiping_left` | `discrete` | `previous_slide` | Discrete command via GestureActivationController → CommandDispatcher |
+| `swiping_down` | `discrete` | `stop_presentation` | Discrete command via GestureActivationController → CommandDispatcher |
 | `swiping_up` | `discrete` | `start_presentation` | Discrete command via GestureActivationController → CommandDispatcher |
 | `zooming_in_full_hand` | `discrete` | `zoom_in_view` | Discrete command via GestureActivationController → CommandDispatcher |
 | `zooming_out_full_hand` | `discrete` | `zoom_out_view` | Discrete command via GestureActivationController → CommandDispatcher |
+| `point_one` | `continuous` | `—` | Continuous tracker: LaserPointerTracker (bypasses discrete dispatcher) |
+| `point_two` | `continuous` | `—` | Continuous tracker: AnnotationPenTracker (bypasses discrete dispatcher) |
+| `unknown` | `discrete` | `no_action` | No-op background class |
 
-## Feature Schema (`feature-schema-v5`)
+## Feature Schema (`feature-schema-v1`)
 
 | Block | Dims | Description |
 |-------|------|-------------|
@@ -139,7 +147,7 @@ from maestro.infrastructure.model.checkpoint_loader import load_inference_artifa
 # Download the artifact (cached after first call)
 local_path = hf_hub_download(
     repo_id="ntsrigaud/maestro-lstm-hybrid",
-    filename="two_stream_attn_v1_finetune_20260513T093937Z_inference.pt",
+    filename="two_stream_attn_v1_finetune_20260513T160347Z_inference.pt",
 )
 
 # Load the artifact (includes model, class labels, and feature schema)
@@ -160,19 +168,19 @@ with torch.no_grad():
 
 ## Training Dataset
 
-- **Source**: Hybrid merge of Jester and IPN-Hand windows: Jester provides unknown/swiping/zoom/stop_sign classes; IPN-Hand provides point_one and point_two
-- **Used classes**: 10 (9 active gestures + `unknown` background)
+- **Source**: Three-source hybrid: **LD-CONG** provides palm/fist/thumb_up/pinch/click and directional swipes (right/left/upward/downward → renamed); **Jester** provides zooming_in/out_full_hand and additional swipe data (center-cropped W32→16); **IPN-Hand** provides point_one/point_two/unknown (center-cropped W32→16). Up to 3,000 windows per class, seed=42.
+- **Used classes**: 14 (13 active gestures + `unknown` background)
 - **Dataset split**: 70% train / 15% val / 15% test (stratified by class)
 - **Augmentation**: temporal scale ±20%, spatial jitter σ=0.005
 
 ## Training Strategy
 
 Two-phase transfer learning pipeline:
-- **Phase 1 (pretraining):** backbone pretrained on external checkpoint `two_stream_attn_v1_20260513T093445Z.pt` to learn generic gesture dynamics.
-- **Phase 2 (fine-tuning):** head replaced and model adapted on Hybrid Jester+IPN 10-gesture vocabulary.
+- **Phase 1 (pretraining):** backbone pretrained on external checkpoint `two_stream_attn_v1_20260513T155730Z.pt` to learn generic gesture dynamics.
+- **Phase 2 (fine-tuning):** head replaced and model adapted on LD-CONG Hybrid 14-class vocabulary (LD-CONG unique + Jester zooms + pooled swipes + IPN pointing, W=16).
 - **Stage A (frozen backbone):** 10 epoch(s) head-only warmup.
-- **Stage B (full model):** up to 60 epoch(s) joint fine-tuning with scheduler/early stopping.
-- **Stage B retention defences:** replay_max_samples_per_class=500, distillation_weight=0.5, replay_ce_weight=0.3, backbone_lr_multiplier=0.1, ewc_weight=100.0, gpm_components=20, forgetting_penalty_weight=0.5.
+- **Stage B (full model):** up to 28 epoch(s) joint fine-tuning with scheduler/early stopping.
+- **Stage B retention defences:** replay_max_samples_per_class=500, distillation_weight=0.0, replay_ce_weight=0.0, backbone_lr_multiplier=0.1, ewc_weight=2000.0, gpm_components=0, forgetting_penalty_weight=0.5.
 
 ## Training Configuration
 
@@ -192,30 +200,34 @@ Two-phase transfer learning pipeline:
 | Early stopping patience | 12 |
 | Label smoothing | 0.05 |
 | Class weighting | disabled |
-| Max samples per class | 5000 |
+| Max samples per class | 3000 |
 | LR scheduler | ReduceLROnPlateau (factor=0.5, patience=8) |
 
 ## Evaluation Results (Test Set)
 
 | Metric | Value |
 |--------|-------|
-| Accuracy | 96.5% |
-| Macro F1 | 96.4% |
+| Accuracy | 96.2% |
+| Macro F1 | 96.3% |
 
 ### Per-Class Recall
 
 | Class | Recall |
 |-------|--------|
-| `unknown` | 91.4% |
-| `point_one` | 98.1% |
-| `point_two` | 98.0% |
-| `stop_sign` | 98.7% |
-| `swiping_down` | 93.1% |
-| `swiping_left` | 96.7% |
+| `palm` | 99.6% |
+| `fist` | 99.1% |
+| `thumb_up` | 99.0% |
+| `pinch` | 99.5% |
+| `click` | 97.3% |
 | `swiping_right` | 95.5% |
-| `swiping_up` | 98.7% |
-| `zooming_in_full_hand` | 96.7% |
-| `zooming_out_full_hand` | 95.3% |
+| `swiping_left` | 97.5% |
+| `swiping_down` | 98.4% |
+| `swiping_up` | 96.8% |
+| `zooming_in_full_hand` | 95.9% |
+| `zooming_out_full_hand` | 93.9% |
+| `point_one` | 97.1% |
+| `point_two` | 93.5% |
+| `unknown` | 87.7% |
 
 ## Comparison with Previous Architecture
 
@@ -226,7 +238,7 @@ Two-phase transfer learning pipeline:
 | Feature projection | No | **Yes (→96)** |
 | Temporal pooling | Mean only | **Mean + Max** |
 | Cross-stream fusion | Concat only | **2-layer MLP gate** |
-| Parameters | ~182 K | ~2,099,434 |
+| Parameters | ~182 K | ~2,100,206 |
 
 ## Limitations and Risks
 
@@ -246,4 +258,4 @@ Estimated CO₂ equivalent: negligible (<0.001 kg CO₂eq).
 
 ---
 
-*Generated by the Maestro training pipeline on 2026-05-13.*
+*Generated by the Maestro training pipeline on 2026-05-14.*

config.json

@@ -1,19 +1,19 @@
 {
-  "model_version": "two_stream_attn_v1_finetune_20260513T093937Z",
+  "model_version": "two_stream_attn_v1_finetune_20260513T160347Z",
   "model_config": {
     "model_name": "two_stream_attn_v1",
     "input_size": 147,
     "hidden_size": 96,
     "num_layers": 4,
     "dropout": 0.35,
-    "num_classes": 10
+    "num_classes": 14
   },
   "feature_schema": {
-    "feature_schema_version": "feature-schema-v5",
+    "feature_schema_version": "feature-schema-v1",
     "feature_dim": 147,
     "orientation_normalization": false,
-    "window_length": 32,
-    "window_step": null
+    "window_length": 16,
+    "window_step": 3
   },
   "training_config": {
     "epochs": 60,
@@ -24,7 +24,7 @@
     "seed": 42,
     "label_smoothing": 0.05,
     "class_weighting": false,
-    "max_samples_per_class": 5000,
+    "max_samples_per_class": 3000,
     "scheduler": {
       "factor": 0.5,
       "patience": 8,
@@ -32,48 +32,60 @@
     }
   },
   "evaluation": {
-    "test_accuracy": 0.964765601672141,
-    "test_macro_f1": 0.9638047676525682,
-    "test_loss": 0.3919019465946233,
-    "calibration_ece": 0.041047210158003236,
+    "test_accuracy": 0.9617670364500792,
+    "test_macro_f1": 0.9632447186344674,
+    "test_loss": 0.43081179908262757,
+    "calibration_ece": 0.032766067943935724,
     "per_class_recall": {
-      "unknown": 0.9143576826196473,
-      "point_one": 0.9810964083175804,
-      "point_two": 0.98,
-      "stop_sign": 0.9873684210526316,
-      "swiping_down": 0.9310344827586207,
-      "swiping_left": 0.9666666666666667,
-      "swiping_right": 0.9553571428571429,
-      "swiping_up": 0.9871794871794872,
-      "zooming_in_full_hand": 0.9672131147540983,
-      "zooming_out_full_hand": 0.9528688524590164
+      "palm": 0.996415770609319,
+      "fist": 0.9912280701754386,
+      "thumb_up": 0.98989898989899,
+      "pinch": 0.9951923076923077,
+      "click": 0.972568578553616,
+      "swiping_right": 0.9546925566343042,
+      "swiping_left": 0.9748603351955307,
+      "swiping_down": 0.9840425531914894,
+      "swiping_up": 0.9682539682539683,
+      "zooming_in_full_hand": 0.958997722095672,
+      "zooming_out_full_hand": 0.9389978213507625,
+      "point_one": 0.9711286089238845,
+      "point_two": 0.9347826086956522,
+      "unknown": 0.8765432098765432
     },
     "per_class_precision": {
-      "unknown": 0.9477806788511749,
-      "point_one": 0.941923774954628,
-      "point_two": 0.974155069582505,
-      "stop_sign": 0.9770833333333333,
-      "swiping_down": 0.9926470588235294,
-      "swiping_left": 0.9354838709677419,
-      "swiping_right": 0.981651376146789,
-      "swiping_up": 0.9625,
-      "zooming_in_full_hand": 0.971764705882353,
-      "zooming_out_full_hand": 0.9728033472803347
+      "palm": 0.996415770609319,
+      "fist": 0.9576271186440678,
+      "thumb_up": 1.0,
+      "pinch": 0.9495412844036697,
+      "click": 0.9948979591836735,
+      "swiping_right": 0.9546925566343042,
+      "swiping_left": 0.9721448467966574,
+      "swiping_down": 0.961038961038961,
+      "swiping_up": 0.977116704805492,
+      "zooming_in_full_hand": 0.9376391982182628,
+      "zooming_out_full_hand": 0.9620535714285714,
+      "point_one": 0.891566265060241,
+      "point_two": 0.9690140845070423,
+      "unknown": 0.9491978609625669
     }
   },
   "class_labels": [
-    "unknown",
-    "point_one",
-    "point_two",
-    "stop_sign",
-    "swiping_down",
-    "swiping_left",
+    "palm",
+    "fist",
+    "thumb_up",
+    "pinch",
+    "click",
     "swiping_right",
+    "swiping_left",
+    "swiping_down",
     "swiping_up",
     "zooming_in_full_hand",
-    "zooming_out_full_hand"
+    "zooming_out_full_hand",
+    "point_one",
+    "point_two",
+    "unknown"
   ],
-  "created_at": "2026-05-13T09:50:38.473285+00:00",
+  "created_at": "2026-05-14T00:26:14.536961+00:00",
   "gesture_command_mapping": {
     "commands": {
       "swiping_up": "start_presentation",
@@ -82,7 +94,11 @@
       "swiping_left": "previous_slide",
       "zooming_in_full_hand": "zoom_in_view",
       "zooming_out_full_hand": "zoom_out_view",
-      "stop_sign": "erase_annotations",
+      "palm": "erase_annotations",
+      "fist": "no_action",
+      "thumb_up": "no_action",
+      "pinch": "no_action",
+      "click": "no_action",
       "unknown": "no_action"
     },
     "modes": {
@@ -92,7 +108,11 @@
       "swiping_left": "discrete",
       "zooming_in_full_hand": "discrete",
       "zooming_out_full_hand": "discrete",
-      "stop_sign": "discrete",
+      "palm": "discrete",
+      "fist": "discrete",
+      "thumb_up": "discrete",
+      "pinch": "discrete",
+      "click": "discrete",
       "point_one": "continuous",
       "point_two": "continuous"
     }