fix: define REPO_NAME in hf_upload.sh (ensure_blade_space referenced it)
Browse filesThis view is limited to 50 files because it contains too many changes. Β See raw diff
- .gitattributes +37 -35
- .gitignore +21 -0
- .hfignore +37 -0
- CLAUDE.md +199 -0
- MODEL_BUDGET.md +24 -0
- README.md +112 -7
- RECON.md +57 -0
- app.py +461 -0
- docs/FormScout-FMS-Spec.md +277 -0
- docs/FormScout-Starter-Kit.md +169 -0
- docs/plans/FormScout-Build-Prompt.md +168 -0
- docs/superpowers/plans/2026-06-04-formscout-full-build.md +0 -0
- docs/superpowers/plans/2026-06-09-pose-model-selector.md +734 -0
- docs/superpowers/plans/2026-06-09-pose-visualizer.md +914 -0
- docs/superpowers/plans/2026-06-13-full-fms-session-pdf.md +1209 -0
- docs/superpowers/specs/2026-06-09-pose-model-selector-design.md +171 -0
- docs/superpowers/specs/2026-06-09-pose-visualizer-design.md +197 -0
- docs/superpowers/specs/2026-06-13-full-fms-session-pdf-design.md +154 -0
- formscout/__init__.py +0 -0
- formscout/agents/__init__.py +0 -0
- formscout/agents/biomechanics.py +608 -0
- formscout/agents/body3d.py +221 -0
- formscout/agents/classifier.py +102 -0
- formscout/agents/ingest.py +91 -0
- formscout/agents/judge.py +125 -0
- formscout/agents/pdf_report.py +175 -0
- formscout/agents/pose2d.py +232 -0
- formscout/agents/prompts/c1_classifier.md +17 -0
- formscout/agents/prompts/c2_judge.md +43 -0
- formscout/agents/report.py +139 -0
- formscout/agents/visualizer.py +418 -0
- formscout/analysis/__init__.py +1 -0
- formscout/analysis/charts.py +171 -0
- formscout/analysis/laban.py +127 -0
- formscout/analysis/relevant_joints.py +122 -0
- formscout/analysis/timeseries.py +49 -0
- formscout/config.py +181 -0
- formscout/pipeline.py +111 -0
- formscout/rubric/__init__.py +32 -0
- formscout/rubric/active_slr.py +51 -0
- formscout/rubric/deep_squat.py +113 -0
- formscout/rubric/hurdle_step.py +60 -0
- formscout/rubric/inline_lunge.py +58 -0
- formscout/rubric/rotary_stability.py +56 -0
- formscout/rubric/shoulder_mobility.py +46 -0
- formscout/rubric/trunk_stability_pushup.py +55 -0
- formscout/run.py +84 -0
- formscout/serving/__init__.py +20 -0
- formscout/serving/llama_cpp.py +148 -0
- formscout/serving/transformers_vlm.py +125 -0
.gitattributes
CHANGED
|
@@ -1,35 +1,37 @@
|
|
| 1 |
-
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
-
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
-
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
-
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
-
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
-
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
-
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
-
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
-
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
-
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
-
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
-
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
-
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
-
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
-
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
-
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
-
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
-
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
-
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
-
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
-
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
-
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
-
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
-
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
-
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
-
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
-
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
-
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
-
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
-
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
-
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
-
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
-
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
-
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
-
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
| 1 |
+
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
+
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
+
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
+
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
+
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
+
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
+
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
+
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
+
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
+
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
+
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
+
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
+
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
+
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
+
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
+
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
+
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
+
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
+
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
+
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
+
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
+
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
+
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
+
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
+
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
+
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
+
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
+
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
+
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
+
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
+
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
+
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
docs/FormScout-FMS-Spec.md.pdf filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
docs/plans/FormScout-Build-Prompt.md.pdf filter=lfs diff=lfs merge=lfs -text
|
.gitignore
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
__pycache__/
|
| 2 |
+
*.py[cod]
|
| 3 |
+
*$py.class
|
| 4 |
+
*.egg-info/
|
| 5 |
+
dist/
|
| 6 |
+
build/
|
| 7 |
+
.eggs/
|
| 8 |
+
*.egg
|
| 9 |
+
.env
|
| 10 |
+
.venv/
|
| 11 |
+
venv/
|
| 12 |
+
env/
|
| 13 |
+
.DS_Store
|
| 14 |
+
checkpoints/
|
| 15 |
+
*.pt
|
| 16 |
+
*.pth
|
| 17 |
+
*.gguf
|
| 18 |
+
*.bin
|
| 19 |
+
traces/
|
| 20 |
+
*.mp4
|
| 21 |
+
!tests/fixtures/*.mp4
|
.hfignore
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Python
|
| 2 |
+
__pycache__/
|
| 3 |
+
*.py[cod]
|
| 4 |
+
*.egg-info/
|
| 5 |
+
dist/
|
| 6 |
+
build/
|
| 7 |
+
.eggs/
|
| 8 |
+
*.egg
|
| 9 |
+
|
| 10 |
+
# Virtual environments
|
| 11 |
+
.venv/
|
| 12 |
+
venv/
|
| 13 |
+
env/
|
| 14 |
+
|
| 15 |
+
# Secrets / local config
|
| 16 |
+
.env
|
| 17 |
+
.env.*
|
| 18 |
+
|
| 19 |
+
# Model weights (managed separately)
|
| 20 |
+
checkpoints/
|
| 21 |
+
*.pt
|
| 22 |
+
*.pth
|
| 23 |
+
*.gguf
|
| 24 |
+
*.bin
|
| 25 |
+
|
| 26 |
+
# Run artifacts
|
| 27 |
+
traces/
|
| 28 |
+
*.mp4
|
| 29 |
+
|
| 30 |
+
# Dev tooling
|
| 31 |
+
.pytest_cache/
|
| 32 |
+
.ruff_cache/
|
| 33 |
+
.DS_Store
|
| 34 |
+
.claude/
|
| 35 |
+
|
| 36 |
+
# Git
|
| 37 |
+
.git/
|
CLAUDE.md
ADDED
|
@@ -0,0 +1,199 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# CLAUDE.md
|
| 2 |
+
|
| 3 |
+
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
| 4 |
+
|
| 5 |
+
## Project overview
|
| 6 |
+
|
| 7 |
+
FormScout is a Gradio app (Hugging Face Space) that scores Functional Movement Screen (FMS) videos 0β3 per test with a written rationale and an annotated overlay. It is a **screening aid** β not a diagnosis, not an injury predictor. Built for the Build Small Hackathon (Backyard AI track). Full product spec is in `docs/FormScout-FMS-Spec.md`; the engineering contract is in `docs/plans/FormScout-Build-Prompt.md`.
|
| 8 |
+
|
| 9 |
+
**Current status:** Phase 2 complete. All 7 FMS test rubric scorers, JudgeAgent, MovementClassifierAgent, ReportAgent, PoseVisualizer (overlay video), and a user-selectable pose-model registry are implemented and tested (86/87 passing). Phase 3 is next (ST-GCN fine-tune + RAG retrieval).
|
| 10 |
+
|
| 11 |
+
## Common commands
|
| 12 |
+
|
| 13 |
+
```bash
|
| 14 |
+
# Run the Gradio app locally
|
| 15 |
+
python3 app.py
|
| 16 |
+
|
| 17 |
+
# Headless pipeline test (no Gradio)
|
| 18 |
+
python3 -m formscout.run sample.mp4
|
| 19 |
+
|
| 20 |
+
# Run all tests
|
| 21 |
+
pytest tests/
|
| 22 |
+
|
| 23 |
+
# Run a single test file or test
|
| 24 |
+
pytest tests/test_phase2.py
|
| 25 |
+
pytest tests/test_biomechanics.py::TestBiomechanicsAgent::test_deep_squat_score
|
| 26 |
+
|
| 27 |
+
# Lint / format
|
| 28 |
+
ruff check . && ruff format .
|
| 29 |
+
|
| 30 |
+
# Start the local VLM judge server (llama.cpp, port 8080)
|
| 31 |
+
./scripts/serve_judge.sh
|
| 32 |
+
|
| 33 |
+
# Push source tree to the HF model repo + Space (PRs; message from last commit)
|
| 34 |
+
./scripts/hf_upload.sh
|
| 35 |
+
|
| 36 |
+
# Run Svelte component tests (when frontend work is added)
|
| 37 |
+
npx vitest run
|
| 38 |
+
```
|
| 39 |
+
|
| 40 |
+
## Architecture
|
| 41 |
+
|
| 42 |
+
The pipeline is a sequence of **typed specialist agents**. Each agent accepts and returns a frozen dataclass from `formscout/types.py`. The Director in `formscout/pipeline.py` orchestrates them as a deterministic state machine (not an LLM).
|
| 43 |
+
|
| 44 |
+
### Agent pipeline
|
| 45 |
+
|
| 46 |
+
```
|
| 47 |
+
IngestAgent β Pose2DAgent β [Body3DAgent β optional]
|
| 48 |
+
β MovementClassifierAgent β BiomechanicsAgent
|
| 49 |
+
β rubric/score_test() β JudgeAgent β ReportAgent
|
| 50 |
+
```
|
| 51 |
+
|
| 52 |
+
The **Director** (`pipeline.py`) owns the flow. `app.py` creates one `Director()` instance and calls `director.run(video_path, test_name, side, model_key)` per submission. The Gradio UI passes `test_name` directly (from dropdown), bypassing the classifier; `model_key` selects the pose backend from `config.POSE_MODELS`.
|
| 53 |
+
|
| 54 |
+
`PoseVisualizer` (`formscout/agents/visualizer.py`) renders the annotated overlay video (skeleton, trails, velocity arrows) from `IngestResult` + `Pose2DResult`. It is called from `app.py` after the pipeline run β it is a UI-layer component, not a Director stage. It returns `None` on failure, never raises.
|
| 55 |
+
|
| 56 |
+
### The tiering rule (most important invariant)
|
| 57 |
+
|
| 58 |
+
**The 2D path is the default and must stand alone as a complete, functional pipeline.** `Body3DAgent` is only activated when `config.ENABLE_3D == True` AND the checkpoint loads successfully. If 3D is off or fails, `Body3DResult(used=False, ...)` is returned β this is a normal success path, not an error. `BiomechFeatures.view` is `"2d"` or `"3d"` so the `JudgeAgent` can caveat its rationale appropriately. Never put `Body3DAgent` on the critical path.
|
| 59 |
+
|
| 60 |
+
### Feature flags in `config.py` and their current state
|
| 61 |
+
|
| 62 |
+
| Flag | Default | Meaning |
|
| 63 |
+
|------|---------|---------|
|
| 64 |
+
| `ENABLE_JUDGE` | `True` | Judge/Classifier call Qwen3-VL via llama-server; graceful rubric fallback when the server is down |
|
| 65 |
+
| `ENABLE_3D` | `False` | When False, Body3DAgent returns `used=False` immediately |
|
| 66 |
+
| `ENABLE_STGCN` | `False` | Phase 3 β ST-GCN learned scoring head |
|
| 67 |
+
| `ENABLE_RAG` | `False` | Phase 3 β RetrievalAgent exemplar lookup |
|
| 68 |
+
|
| 69 |
+
All model IDs, thresholds, k-values, and feature flags live in `config.py` β never scattered literals.
|
| 70 |
+
|
| 71 |
+
### Judge backend selection (local vs Space)
|
| 72 |
+
|
| 73 |
+
`config.resolve_judge_backend()` picks the VLM backend via `FORMSCOUT_JUDGE_BACKEND` (`llama_cpp` | `transformers` | `auto`). `auto` (default) uses **llama-server locally** and the **in-process transformers backend on a Space** (detected via `SPACE_ID`). `JudgeAgent` gets its client from `serving.get_vlm_client()`.
|
| 74 |
+
|
| 75 |
+
- **`llama_cpp`** β `LlamaCppClient` β llama-server at `127.0.0.1:8080` (start with `scripts/serve_judge.sh`). The local path; works perfectly.
|
| 76 |
+
- **`transformers`** β `TransformersVLMClient` loads Qwen3-VL-8B via transformers, GPU-wrapped with `spaces.GPU` (ZeroGPU). Lazy model load, cached per process. On any load/inference failure it returns `{"fallback": True}` and the Judge falls back to the rubric. **Needs validation on real ZeroGPU hardware** β not exercised in CPU tests.
|
| 77 |
+
|
| 78 |
+
### Fallback chain (important for local dev and Spaces)
|
| 79 |
+
|
| 80 |
+
1. `ENABLE_JUDGE=False` β JudgeAgent returns rubric score wrapped as JudgeResult (no VLM needed)
|
| 81 |
+
2. `ENABLE_JUDGE=True` + selected backend unavailable / transformers load fails β same rubric fallback, logs a warning
|
| 82 |
+
3. `ENABLE_JUDGE=True` + backend available β calls Qwen3-VL-8B-Instruct (llama-server locally, transformers/ZeroGPU on a Space)
|
| 83 |
+
|
| 84 |
+
Start the VLM server with `scripts/serve_judge.sh` (downloads live in `checkpoints/qwen3-vl/`, gitignored). To use a fine-tuned GGUF, set `FORMSCOUT_JUDGE_GGUF` (and `FORMSCOUT_JUDGE_MMPROJ` if it ships its own projector) β no code change needed. Multimodal requests go through the OpenAI-compatible `/v1/chat/completions` endpoint (the legacy `/completion` + `image_data` path does not work with modern llama-server).
|
| 85 |
+
|
| 86 |
+
This means the app is **fully functional without any GPU or llama.cpp** β rubric scoring is pure Python.
|
| 87 |
+
|
| 88 |
+
### Rubric scorers
|
| 89 |
+
|
| 90 |
+
Each FMS test has a pure-function scorer in `formscout/rubric/`:
|
| 91 |
+
|
| 92 |
+
```
|
| 93 |
+
score_deep_squat / score_hurdle_step / score_inline_lunge /
|
| 94 |
+
score_shoulder_mobility / score_active_slr /
|
| 95 |
+
score_trunk_stability_pushup / score_rotary_stability
|
| 96 |
+
```
|
| 97 |
+
|
| 98 |
+
All accept `BiomechFeatures` and return `ScoreResult`. Dispatch via `rubric.score_test(features)`. **Rubric functions must remain pure** β no model calls, no I/O.
|
| 99 |
+
|
| 100 |
+
### Bilateral tests
|
| 101 |
+
|
| 102 |
+
`hurdle_step`, `inline_lunge`, `shoulder_mobility`, `active_slr` are bilateral. `ReportAgent` groups them by test name, takes the **lower** score, and always emits the asymmetry delta even when scores are equal. `composite` is `None` when any test is unscored.
|
| 103 |
+
|
| 104 |
+
### Types contract
|
| 105 |
+
|
| 106 |
+
Every agent I/O is a frozen dataclass from `formscout/types.py`. Key types:
|
| 107 |
+
|
| 108 |
+
- `IngestResult` β decoded frames (np.ndarray list), fps, duration, dimensions
|
| 109 |
+
- `Pose2DResult` β per-frame keypoints as `dict[int, {x, y, conf}]` (COCO 17 joints)
|
| 110 |
+
- `Body3DResult` β optional 3D joints, always has `used: bool`
|
| 111 |
+
- `MovementResult` β `test_name` (validated enum), `side` ("left"|"right"|"na")
|
| 112 |
+
- `BiomechFeatures` β `angles: dict`, `alignments: dict`, `view: "2d"|"3d"`, `symmetry_delta`
|
| 113 |
+
- `ScoreResult` β `score: int` (0β3), `rationale`, `needs_human`
|
| 114 |
+
- `JudgeResult` β same as ScoreResult + `compensation_tags`, `corrective_hint`; `score=None` when `needs_human=True`
|
| 115 |
+
- `PipelineState` β mutable accumulator threaded through the Director
|
| 116 |
+
|
| 117 |
+
`MovementResult` and `JudgeResult` validate their fields in `__post_init__` β passing invalid values raises immediately.
|
| 118 |
+
|
| 119 |
+
### Pose model selection and checkpoints
|
| 120 |
+
|
| 121 |
+
`config.POSE_MODELS` is a registry of pose backends: MediaPipe (CPU-friendly), five YOLO26 sizes (n/s/m/l/x), and Sapiens2 variants (Phase 3, need the custom `sapiens` repo installed). `config.DEFAULT_POSE_MODEL` is YOLO26n. The Gradio UI exposes a dropdown built from `config.available_pose_models()` (filters to checkpoints actually present) and passes the chosen `model_key` through `Director.run` to `Pose2DAgent`. `config.YOLO_POSE_MODEL` is a backward-compat alias only.
|
| 122 |
+
|
| 123 |
+
Checkpoints are **not** committed (`checkpoints/` is gitignored). `formscout/startup.py:ensure_checkpoints()` downloads missing YOLO26/MediaPipe files from the `silas-therapy/formscout-checkpoints` HF repo once at app startup. Models load once per process and are cached β never inside the inference hot path.
|
| 124 |
+
|
| 125 |
+
### llama.cpp serving
|
| 126 |
+
|
| 127 |
+
`formscout/serving/llama_cpp.py` provides `LlamaCppClient` (VLM, port 8080) and `EmbeddingClient` (embeddings, port 8081). Both check `/health` before use and return safe error dicts when unavailable. Only active when the corresponding `ENABLE_*` flag is True.
|
| 128 |
+
|
| 129 |
+
### Deploying to Hugging Face
|
| 130 |
+
|
| 131 |
+
The repo deploys to both `silas-therapy/small-functional-movement-screening` (model repo) and the Space of the same name (README frontmatter is the Space config). Use `./scripts/hf_upload.sh` β never raw `hf upload .`: the `hf` CLI does **not** read `.hfignore`, so a raw upload hashes the entire `.venv` (~44k files) and pushes torch binaries. The script parses `.hfignore` into `--exclude` globs, preflights the file count, creates PRs on both repos, and auto-switches to `hf upload-large-folder` (resumable, but no PR / no commit message) above 500 files.
|
| 132 |
+
|
| 133 |
+
## Key constraints and invariants
|
| 134 |
+
|
| 135 |
+
- **No cloud model APIs.** All inference runs on-Space (ZeroGPU). No OpenAI/Anthropic/Gemini calls.
|
| 136 |
+
- **Pain is never auto-scored.** Any clearing test or visible distress sets `needs_human=True` β enforced in rubric functions and JudgeAgent. `JudgeResult.score` must be `None` when `needs_human=True`.
|
| 137 |
+
- **Quality gates (Director, never silently skip):**
|
| 138 |
+
- Any agent `confidence < config.MIN_CONFIDENCE` (0.6) β warn or stop
|
| 139 |
+
- `|rubric.score - judge.score| >= 1` β flag disagreement
|
| 140 |
+
- `MovementResult.test_name == "unknown"` β stop pipeline, surface manual override
|
| 141 |
+
- `JudgeAgent.needs_human == True` β no numeric score emitted
|
| 142 |
+
- **Composite is null** when any test is unscored. Never show a partial 0β21 as complete.
|
| 143 |
+
- **Pipeline runs headless.** No Gradio imports in any agent file.
|
| 144 |
+
- **Safety banner** ("Screening aid β not a diagnosisβ¦") must always be visible in the UI β appears at top and bottom of `app.py`.
|
| 145 |
+
|
| 146 |
+
## Engineering standards
|
| 147 |
+
|
| 148 |
+
- Every agent: one public entrypoint, typed dataclass I/O from `types.py`, `confidence: float` and `notes: str` on every result.
|
| 149 |
+
- Models load once at module/instance init β never inside the inference hot path.
|
| 150 |
+
- Every agent module docstring states: purpose, inputs, outputs, failure behavior, model param count, license, and gated status.
|
| 151 |
+
- `tracing.py` records structured per-agent I/O for any run; one full run gets exported to the Hub.
|
| 152 |
+
- Every agent ships with a pytest in `tests/` that runs without model downloads and asserts the typed contract.
|
| 153 |
+
|
| 154 |
+
## Model stack (~17.6B total β stay under 32B)
|
| 155 |
+
|
| 156 |
+
| Component | Model | Params | Status |
|
| 157 |
+
|---|---|---|---|
|
| 158 |
+
| 2D pose (primary) | YOLO26-Pose n/s/m/l/x (default: n) | 0.0007β0.058B | Ready (auto-downloaded at startup) |
|
| 159 |
+
| 2D pose (CPU alt) | MediaPipe Pose Landmarker (full) | ~0.004B | Ready (auto-downloaded at startup) |
|
| 160 |
+
| 2D pose (HQ alt) | `facebook/sapiens2-pose-0.4b/0.8b/1b/5b` | 0.4β5B | Phase 3 β needs custom `sapiens` repo |
|
| 161 |
+
| Segmentation | SAM 3.1 base | ~0.85B | Access accepted |
|
| 162 |
+
| 3D biomechanics | `facebook/sam-3d-body-dinov3` | ~0.84B | **Access ACCEPTED Jun 4 2026** |
|
| 163 |
+
| Learned scoring | ST-GCN (pyskl) | ~0.03B | Phase 3 |
|
| 164 |
+
| Judge + Classifier | Qwen3-VL-8B-Instruct (llama.cpp) | 8B | **Online** β `scripts/serve_judge.sh`, ENABLE_JUDGE=True |
|
| 165 |
+
| Retrieval | Qwen3-VL-Embedding-8B (llama.cpp) | 8B | Phase 3 |
|
| 166 |
+
|
| 167 |
+
Track the running sum in `MODEL_BUDGET.md`. The two Qwen3-VL-8B models share a backbone.
|
| 168 |
+
|
| 169 |
+
## Gradio + Svelte UI guidance
|
| 170 |
+
|
| 171 |
+
The UI uses **Gradio `gr.Blocks`** with custom CSS/theme (`formscout/ui/theme.py`). Custom Svelte components for score dial, asymmetry bars, rubric drawer are planned for Phase 4. Use `gradio-svelte-expert` agent for Svelte component work.
|
| 172 |
+
|
| 173 |
+
- ZeroGPU: wrap heavy inference (`Pose2DAgent.run`, `Body3DAgent.run`) in `@spaces.GPU` before deploying to Spaces.
|
| 174 |
+
- Verify Gradio APIs against current docs before use β pin exact versions in `requirements.txt`.
|
| 175 |
+
|
| 176 |
+
## Build phases
|
| 177 |
+
|
| 178 |
+
1. **Phase 0 β Recon:** β
Complete. See `RECON.md`.
|
| 179 |
+
2. **Phase 1 β Spine:** β
Complete. Deep Squat end-to-end.
|
| 180 |
+
3. **Phase 2 β All 7 tests:** β
Complete. Classifier, Judge, Report agents; all rubric scorers; Gradio UI.
|
| 181 |
+
4. **Phase 3 β Learned scoring + retrieval:** ST-GCN fine-tune on physio clips, publish to Hub. RetrievalAgent with embedding index.
|
| 182 |
+
5. **Phase 4 β Polish + ship:** Custom Svelte UI components, agent trace to Hub, blog post. (Overlay video done via `PoseVisualizer`; full 7-test session + PDF export done via `formscout/session.py` + `PdfReportAgent`.)
|
| 183 |
+
|
| 184 |
+
## Known issues
|
| 185 |
+
|
| 186 |
+
- `tests/test_biomechanics.py::TestBiomechanicsAgent::test_unimplemented_test_returns_low_confidence` fails: expects `"not yet implemented"` in `result.notes` but biomechanics returns empty string. Minor β low priority.
|
| 187 |
+
|
| 188 |
+
## Badge checklist (definition of done)
|
| 189 |
+
|
| 190 |
+
- [ ] Space runs green; upload β scorecard works on real clips
|
| 191 |
+
- [ ] Param sum verified β€ 32B in `MODEL_BUDGET.md`
|
| 192 |
+
- [ ] π **Off the Grid** β no cloud model APIs anywhere in the pipeline
|
| 193 |
+
- [ ] π― **Well-Tuned** β fine-tuned ST-GCN head published to Hub with honest model card
|
| 194 |
+
- [ ] π¨ **Off-Brand** β custom, non-default Gradio UI (scout/trail theme)
|
| 195 |
+
- [ ] π¦ **Llama Champion** β VLM + embedder served via llama.cpp (GGUF)
|
| 196 |
+
- [ ] π‘ **Sharing is Caring** β one full agent trace (all I/O) published to Hub
|
| 197 |
+
- [ ] π **Field Notes** β blog post written, honesty section (FMS limitations) front-and-center
|
| 198 |
+
- [ ] Demo video + social post recorded
|
| 199 |
+
- [ ] Safety banner present; pain/clearing never auto-scored; low-confidence flagged
|
MODEL_BUDGET.md
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# MODEL_BUDGET.md
|
| 2 |
+
|
| 3 |
+
Running sum must stay β€ 32B params.
|
| 4 |
+
|
| 5 |
+
| Component | Model | Params |
|
| 6 |
+
|---|---|---|
|
| 7 |
+
| 2D Pose (primary) | YOLO26l-Pose | 0.026B |
|
| 8 |
+
| 2D Pose (HQ alt) | YOLO26x-Pose | 0.058B |
|
| 9 |
+
| 2D Pose (fallback) | Sapiens2 Pose | 0.6B |
|
| 10 |
+
| Segmentation | SAM 3.1 base | 0.85B |
|
| 11 |
+
| 3D Body (optional) | SAM 3D Body DINOv3-H+ | 0.84B |
|
| 12 |
+
| Scoring Head | ST-GCN (pyskl) | 0.03B |
|
| 13 |
+
| Judge/Classifier | Qwen3-VL-8B-Instruct (Q4_K_M GGUF + F16 mmproj, llama.cpp) | 8B |
|
| 14 |
+
| Retrieval | Qwen3-VL-Embedding-8B | 8B |
|
| 15 |
+
| **Total** | | **~18.37B** |
|
| 16 |
+
|
| 17 |
+
Headroom: ~13.63B under 32B cap.
|
| 18 |
+
|
| 19 |
+
Note: The two Qwen3-VL-8B models share a backbone (counted separately here for safety).
|
| 20 |
+
Only one pose backend runs at a time (YOLO or Sapiens2, not both).
|
| 21 |
+
|
| 22 |
+
Judge/Classifier serving: `scripts/serve_judge.sh` (llama-server, port 8080).
|
| 23 |
+
Default GGUF: `Qwen/Qwen3-VL-8B-Instruct-GGUF` β `checkpoints/qwen3-vl/` (gitignored).
|
| 24 |
+
Fine-tuned swap: set `FORMSCOUT_JUDGE_GGUF` (+ `FORMSCOUT_JUDGE_MMPROJ`) β no code change.
|
README.md
CHANGED
|
@@ -1,13 +1,118 @@
|
|
| 1 |
---
|
| 2 |
-
title:
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo:
|
| 6 |
sdk: gradio
|
| 7 |
-
sdk_version: 6.18.0
|
| 8 |
-
python_version: '3.13'
|
| 9 |
app_file: app.py
|
| 10 |
pinned: false
|
|
|
|
|
|
|
| 11 |
---
|
| 12 |
|
| 13 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: FormScout
|
| 3 |
+
emoji: ποΈ
|
| 4 |
+
colorFrom: green
|
| 5 |
+
colorTo: yellow
|
| 6 |
sdk: gradio
|
|
|
|
|
|
|
| 7 |
app_file: app.py
|
| 8 |
pinned: false
|
| 9 |
+
license: apache-2.0
|
| 10 |
+
short_description: FMS video scoring β movement screen aid
|
| 11 |
---
|
| 12 |
|
| 13 |
+
# FormScout
|
| 14 |
+
|
| 15 |
+
FMS (Functional Movement Screen) scoring pipeline β a screening aid that scores movement videos 0β3 per test with a written rationale and annotated overlay.
|
| 16 |
+
|
| 17 |
+
**β οΈ Screening aid β not a diagnosis. Pain or clearing tests require a clinician.**
|
| 18 |
+
|
| 19 |
+
## Running locally
|
| 20 |
+
|
| 21 |
+
### 1. Clone and install
|
| 22 |
+
|
| 23 |
+
```bash
|
| 24 |
+
git clone https://huggingface.co/silas-therapy/small-functional-movement-screening
|
| 25 |
+
cd small-functional-movement-screening
|
| 26 |
+
python3 -m venv .venv && source .venv/bin/activate
|
| 27 |
+
pip install -r requirements.txt
|
| 28 |
+
```
|
| 29 |
+
|
| 30 |
+
### 2. Start the VLM judge (optional but recommended)
|
| 31 |
+
|
| 32 |
+
The judge uses Qwen3-VL-8B-Instruct via llama.cpp. Without it the app falls back to the deterministic rubric score β fully functional, no GPU needed.
|
| 33 |
+
|
| 34 |
+
```bash
|
| 35 |
+
# Install llama.cpp once
|
| 36 |
+
brew install llama.cpp
|
| 37 |
+
|
| 38 |
+
# Download the model (one-time, ~6 GB)
|
| 39 |
+
python3 -c "
|
| 40 |
+
from huggingface_hub import hf_hub_download
|
| 41 |
+
for f in ['Qwen3VL-8B-Instruct-Q4_K_M.gguf', 'mmproj-Qwen3VL-8B-Instruct-F16.gguf']:
|
| 42 |
+
hf_hub_download('Qwen/Qwen3-VL-8B-Instruct-GGUF', f, local_dir='checkpoints/qwen3-vl')
|
| 43 |
+
"
|
| 44 |
+
|
| 45 |
+
# Start the server (keep this terminal open)
|
| 46 |
+
./scripts/serve_judge.sh
|
| 47 |
+
```
|
| 48 |
+
|
| 49 |
+
To use a fine-tuned GGUF instead of the default:
|
| 50 |
+
```bash
|
| 51 |
+
FORMSCOUT_JUDGE_GGUF=/path/to/finetuned.gguf ./scripts/serve_judge.sh
|
| 52 |
+
```
|
| 53 |
+
|
| 54 |
+
### 3. Launch the Gradio app
|
| 55 |
+
|
| 56 |
+
```bash
|
| 57 |
+
python3 app.py
|
| 58 |
+
# β http://127.0.0.1:7860
|
| 59 |
+
```
|
| 60 |
+
|
| 61 |
+
Upload a video, select the FMS test from the dropdown, and click **Analyze**.
|
| 62 |
+
|
| 63 |
+
### 4. Headless pipeline (no Gradio)
|
| 64 |
+
|
| 65 |
+
```bash
|
| 66 |
+
python3 -m formscout.run sample.mp4
|
| 67 |
+
```
|
| 68 |
+
|
| 69 |
+
### 5. Tests
|
| 70 |
+
|
| 71 |
+
```bash
|
| 72 |
+
pytest tests/ -v
|
| 73 |
+
```
|
| 74 |
+
|
| 75 |
+
### 6. Upload to Hugging Face
|
| 76 |
+
|
| 77 |
+
```bash
|
| 78 |
+
# Pushes source to both model repo and Space, opens a PR on each
|
| 79 |
+
./scripts/hf_upload.sh
|
| 80 |
+
|
| 81 |
+
# Or with a custom commit message
|
| 82 |
+
./scripts/hf_upload.sh "feat: my change"
|
| 83 |
+
```
|
| 84 |
+
|
| 85 |
+
## Architecture
|
| 86 |
+
|
| 87 |
+
Typed specialist agents orchestrated by a deterministic Director:
|
| 88 |
+
|
| 89 |
+
```
|
| 90 |
+
Ingest β Pose2D β [Body3D optional] β Biomechanics β Rubric Score β [Judge] β Report
|
| 91 |
+
```
|
| 92 |
+
|
| 93 |
+
| Agent | Model | Status |
|
| 94 |
+
|---|---|---|
|
| 95 |
+
| Pose2D | YOLO26l-Pose (0.026B) + MediaPipe fallback | β
|
|
| 96 |
+
| Body3D | SAM 3D Body DINOv3 (0.84B) | gated, off by default |
|
| 97 |
+
| Judge + Classifier | Qwen3-VL-8B-Instruct via llama.cpp (8B) | β
|
|
| 98 |
+
| Scoring Head | ST-GCN (0.03B) | Phase 3 |
|
| 99 |
+
| Retrieval | Qwen3-VL-Embedding-8B (8B) | Phase 3 |
|
| 100 |
+
|
| 101 |
+
See [CLAUDE.md](CLAUDE.md) for full architecture and invariants.
|
| 102 |
+
|
| 103 |
+
## Feature flags (`formscout/config.py`)
|
| 104 |
+
|
| 105 |
+
| Flag | Default | Meaning |
|
| 106 |
+
|---|---|---|
|
| 107 |
+
| `ENABLE_JUDGE` | `True` | VLM judge via llama-server; rubric fallback when server is down |
|
| 108 |
+
| `ENABLE_3D` | `False` | SAM 3D Body β off until integrated |
|
| 109 |
+
| `ENABLE_STGCN` | `False` | Phase 3 |
|
| 110 |
+
| `ENABLE_RAG` | `False` | Phase 3 |
|
| 111 |
+
|
| 112 |
+
## Model budget
|
| 113 |
+
|
| 114 |
+
~18B params total (under 32B cap). See [MODEL_BUDGET.md](MODEL_BUDGET.md).
|
| 115 |
+
|
| 116 |
+
## License
|
| 117 |
+
|
| 118 |
+
Apache-2.0. Built for the Build Small Hackathon (Backyard AI track).
|
RECON.md
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# RECON.md
|
| 2 |
+
|
| 3 |
+
Phase 0 reconnaissance findings β model verification, Gradio APIs, access status.
|
| 4 |
+
Updated: June 4, 2026.
|
| 5 |
+
|
| 6 |
+
## Gradio
|
| 7 |
+
- Version: TBD (will verify on first `pip install gradio`)
|
| 8 |
+
- gr.Blocks: expected β (used in app.py skeleton)
|
| 9 |
+
- gr.Video: expected β
|
| 10 |
+
- gr.Walkthrough / gr.Step: TBD (verify in Phase 2)
|
| 11 |
+
- gr.Navbar: TBD (verify in Phase 2)
|
| 12 |
+
- UI approach: gr.Blocks + custom CSS/theme (escalate to Server only if needed)
|
| 13 |
+
|
| 14 |
+
## Python
|
| 15 |
+
- Python 3.13.9 (local dev)
|
| 16 |
+
- pytest 9.0.2, numpy, opencv-python installed
|
| 17 |
+
|
| 18 |
+
## Model Verification
|
| 19 |
+
|
| 20 |
+
| Model | Params | License | GGUF | ZeroGPU | Status |
|
| 21 |
+
|---|---|---|---|---|---|
|
| 22 |
+
| YOLO26l-Pose (primary) | 0.026B | AGPL-3.0 | n/a | β (6.5ms T4) | ready |
|
| 23 |
+
| YOLO26x-Pose (HQ alt) | 0.058B | AGPL-3.0 | n/a | β (12.2ms T4) | ready |
|
| 24 |
+
| SAM 3.1 base (sam2.1_hiera_base_plus) | ~0.85B | SAM License | n/a | β | access accepted |
|
| 25 |
+
| SAM 3D Body (facebook/sam-3d-body-dinov3) | 0.84B (DINOv3-H+) | SAM License | n/a | β | **INTEGRATED** |
|
| 26 |
+
| Sapiens2 Pose (noahcao/sapiens-pose-coco) | ~0.6B | CC-BY-NC-4.0 | n/a | β | access accepted |
|
| 27 |
+
| ST-GCN (pyskl) | ~0.03B | Apache-2.0 | n/a | β | ready |
|
| 28 |
+
| Qwen3-VL-8B-Instruct | 8B | Apache-2.0 | β | llama.cpp | ready |
|
| 29 |
+
| Qwen3-VL-Embedding-8B | 8B | Apache-2.0 | β | llama.cpp | ready |
|
| 30 |
+
|
| 31 |
+
## Param Sum
|
| 32 |
+
~17.63B β well under 32B limit.
|
| 33 |
+
|
| 34 |
+
## Gated Access Status (as of Jun 4, 2026)
|
| 35 |
+
- [x] SAM 3.1 (facebookresearch/sam3) β accepted
|
| 36 |
+
- [x] SAM 3D Body (facebook/sam-3d-body-dinov3) β **ACCEPTED** (confirmed Jun 4)
|
| 37 |
+
- [x] Sapiens2 Pose (noahcao/sapiens-pose-coco) β accepted
|
| 38 |
+
|
| 39 |
+
## Open Questions
|
| 40 |
+
- [ ] Confirm "β€32B" = summed vs per-model in Discord AMA
|
| 41 |
+
- [ ] AGPL-3.0 YOLO OK for hackathon submission? (Likely yes for non-commercial demo)
|
| 42 |
+
|
| 43 |
+
## llama.cpp Build Plan
|
| 44 |
+
- CPU-only build first (avoids libcudart.so issues on Spaces)
|
| 45 |
+
- Fallback: transformers + spaces.GPU for VLM inference
|
| 46 |
+
- GGUF quantized Qwen3-VL-8B at Q4_K_M (~4.5GB)
|
| 47 |
+
|
| 48 |
+
## Key Decisions
|
| 49 |
+
- Primary pose: YOLO11x-Pose (fastest, well-tested)
|
| 50 |
+
- Fallback pose: Sapiens2 (more keypoints, slower)
|
| 51 |
+
- 3D body: INTEGRATED β uses `setup_sam_3d_body()` from `notebook.utils`, outputs MHR joints
|
| 52 |
+
- API: `estimator.process_one_image(rgb_image)` β single RGB np.ndarray
|
| 53 |
+
- Model variants: DINOv3-H+ (840M) default, ViT-H (631M) smaller
|
| 54 |
+
- Temporal smoothing via EMA (alpha=0.3) to reduce single-frame jitter
|
| 55 |
+
- config.enable_3d=False by default; flipped when checkpoint verified on Space
|
| 56 |
+
- VLM: Qwen3-VL-8B via llama.cpp (Judge + Classifier)
|
| 57 |
+
- Embeddings: Qwen3-VL-Embedding-8B via llama.cpp (Retrieval)
|
app.py
ADDED
|
@@ -0,0 +1,461 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
FormScout β Gradio app entrypoint.
|
| 3 |
+
Screening aid for Functional Movement Screen (FMS) scoring.
|
| 4 |
+
NOT a diagnosis. NOT an injury predictor.
|
| 5 |
+
|
| 6 |
+
Custom scout/trail themed UI with score dial, pipeline visualization,
|
| 7 |
+
rubric breakdown, and persistent safety banner.
|
| 8 |
+
"""
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import os
|
| 12 |
+
import tempfile
|
| 13 |
+
|
| 14 |
+
import gradio as gr
|
| 15 |
+
|
| 16 |
+
from formscout.pipeline import Director
|
| 17 |
+
from formscout.rubric import score_test
|
| 18 |
+
from formscout.ui.theme import formscout_theme, FORMSCOUT_CSS
|
| 19 |
+
from formscout import config
|
| 20 |
+
from formscout import session as session_mod
|
| 21 |
+
from formscout.startup import ensure_checkpoints
|
| 22 |
+
|
| 23 |
+
ensure_checkpoints()
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
# βββ Constants βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 27 |
+
|
| 28 |
+
DISCLAIMER = (
|
| 29 |
+
"β οΈ **Screening aid β not a diagnosis. "
|
| 30 |
+
"Pain or clearing tests require a clinician.**"
|
| 31 |
+
)
|
| 32 |
+
|
| 33 |
+
FMS_TESTS = [
|
| 34 |
+
("Deep Squat", "deep_squat"),
|
| 35 |
+
("Hurdle Step", "hurdle_step"),
|
| 36 |
+
("In-Line Lunge", "inline_lunge"),
|
| 37 |
+
("Shoulder Mobility", "shoulder_mobility"),
|
| 38 |
+
("Active Straight-Leg Raise", "active_slr"),
|
| 39 |
+
("Trunk Stability Push-Up", "trunk_stability_pushup"),
|
| 40 |
+
("Rotary Stability", "rotary_stability"),
|
| 41 |
+
]
|
| 42 |
+
|
| 43 |
+
SCORE_DESCRIPTIONS = {
|
| 44 |
+
3: "Movement performed to criterion β no compensation",
|
| 45 |
+
2: "Movement completed with compensation or regression",
|
| 46 |
+
1: "Unable to perform the movement pattern",
|
| 47 |
+
0: "Pain reported β clinician referral required",
|
| 48 |
+
}
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
# βββ Processing ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 52 |
+
|
| 53 |
+
def process_video(video_path: str, test_name: str, side: str, model_key: str,
|
| 54 |
+
layers: list[str], session_state):
|
| 55 |
+
"""Analyse one clip and accumulate it into the screening session."""
|
| 56 |
+
if not video_path:
|
| 57 |
+
return (
|
| 58 |
+
session_state, _render_empty_state(), "Upload a video to begin analysis.",
|
| 59 |
+
"", "", None, "", _render_session_table(session_state),
|
| 60 |
+
gr.update(visible=False), gr.update(visible=False),
|
| 61 |
+
)
|
| 62 |
+
|
| 63 |
+
if session_state is None:
|
| 64 |
+
session_state = session_mod.new_session()
|
| 65 |
+
|
| 66 |
+
director = Director()
|
| 67 |
+
state = director.run(video_path, test_name=test_name, side=side, model_key=model_key)
|
| 68 |
+
|
| 69 |
+
score_html = _render_empty_state()
|
| 70 |
+
score_details = ""
|
| 71 |
+
|
| 72 |
+
if state.features:
|
| 73 |
+
result = score_test(state.features)
|
| 74 |
+
judge = state.judge
|
| 75 |
+
if judge and judge.score is not None:
|
| 76 |
+
score_html = _render_score_card(judge.score, judge.confidence, judge.needs_human)
|
| 77 |
+
score_details = _render_score_details_judge(judge, result, state.features)
|
| 78 |
+
elif judge and judge.needs_human:
|
| 79 |
+
score_html = _render_score_card(0, 0, True)
|
| 80 |
+
score_details = f"### Needs Clinician Review\n{judge.rationale}"
|
| 81 |
+
else:
|
| 82 |
+
score_html = _render_score_card(result.score, result.confidence, result.needs_human)
|
| 83 |
+
score_details = _render_score_details(result, state.features)
|
| 84 |
+
|
| 85 |
+
# Accumulate into the session (only when we have a real analysis)
|
| 86 |
+
if state.ingest and state.pose2d and state.judge:
|
| 87 |
+
draw_trails = "trails" in {lbl.lower().replace(" ", "_") for lbl in (layers or [])}
|
| 88 |
+
try:
|
| 89 |
+
session_mod.add_analysis(
|
| 90 |
+
session_state, ingest=state.ingest, pose2d=state.pose2d,
|
| 91 |
+
features=state.features, judge=state.judge,
|
| 92 |
+
test_name=test_name, side=side, draw_trails=draw_trails,
|
| 93 |
+
)
|
| 94 |
+
except Exception as e:
|
| 95 |
+
state.warnings.append(f"session accumulation failed: {e}")
|
| 96 |
+
|
| 97 |
+
pipeline_md = _render_pipeline_status(state)
|
| 98 |
+
alerts = _render_alerts(state)
|
| 99 |
+
|
| 100 |
+
overlay_path = None
|
| 101 |
+
vel_summary = ""
|
| 102 |
+
layer_set = {lbl.lower().replace(" ", "_") for lbl in (layers or [])}
|
| 103 |
+
if layer_set and state.ingest and state.pose2d:
|
| 104 |
+
try:
|
| 105 |
+
from formscout.agents.visualizer import PoseVisualizer, build_velocity_summary
|
| 106 |
+
vis = PoseVisualizer()
|
| 107 |
+
with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as f:
|
| 108 |
+
out_path = f.name
|
| 109 |
+
overlay_path = vis.render_video(state.ingest, state.pose2d, layer_set, out_path)
|
| 110 |
+
if overlay_path:
|
| 111 |
+
vel_summary = build_velocity_summary(state.pose2d.keypoints, vis.last_velocities)
|
| 112 |
+
except Exception as e:
|
| 113 |
+
alerts = (alerts or "") + f"\nβ οΈ Visualizer error: {e}"
|
| 114 |
+
|
| 115 |
+
has_entries = bool(session_state and session_state.entries)
|
| 116 |
+
return (
|
| 117 |
+
session_state, score_html, pipeline_md, score_details, alerts,
|
| 118 |
+
overlay_path, vel_summary, _render_session_table(session_state),
|
| 119 |
+
gr.update(visible=has_entries), gr.update(visible=has_entries),
|
| 120 |
+
)
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
def _render_score_card(score: int, confidence: float, needs_human: bool) -> str:
|
| 124 |
+
"""Render the score dial as HTML."""
|
| 125 |
+
if needs_human:
|
| 126 |
+
return """
|
| 127 |
+
<div class="score-card needs-review">
|
| 128 |
+
<div style="font-size: 1.2em; color: #cf922a; margin-bottom: 8px;">β οΈ Needs Clinician Review</div>
|
| 129 |
+
<div style="font-size: 0.9em; color: #4a5f57;">Pain or clearing test detected β cannot auto-score</div>
|
| 130 |
+
</div>
|
| 131 |
+
"""
|
| 132 |
+
|
| 133 |
+
conf_pct = int(confidence * 100)
|
| 134 |
+
conf_color = "#2b8a8a" if confidence >= 0.7 else "#cf922a" if confidence >= 0.4 else "#d9534f"
|
| 135 |
+
|
| 136 |
+
return f"""
|
| 137 |
+
<div class="score-card">
|
| 138 |
+
<div class="score-value">{score}/3</div>
|
| 139 |
+
<div style="font-size: 0.95em; color: #4a5f57; margin-top: 4px;">
|
| 140 |
+
{SCORE_DESCRIPTIONS.get(score, '')}
|
| 141 |
+
</div>
|
| 142 |
+
<div style="margin-top: 12px;">
|
| 143 |
+
<div style="display: flex; justify-content: space-between; font-size: 0.8em; color: #6b7d75;">
|
| 144 |
+
<span>Confidence</span>
|
| 145 |
+
<span style="color: {conf_color};">{conf_pct}%</span>
|
| 146 |
+
</div>
|
| 147 |
+
<div class="confidence-bar">
|
| 148 |
+
<div class="confidence-fill" style="width: {conf_pct}%;"></div>
|
| 149 |
+
</div>
|
| 150 |
+
</div>
|
| 151 |
+
</div>
|
| 152 |
+
"""
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
def _render_empty_state() -> str:
|
| 156 |
+
"""Render placeholder when no video processed yet."""
|
| 157 |
+
return """
|
| 158 |
+
<div class="score-card" style="opacity: 0.6;">
|
| 159 |
+
<div style="font-size: 2em; margin-bottom: 8px;">ποΈ</div>
|
| 160 |
+
<div style="color: #6b7d75;">Upload a video to begin</div>
|
| 161 |
+
</div>
|
| 162 |
+
"""
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
def _render_score_details(result, features) -> str:
|
| 166 |
+
"""Render the rubric breakdown."""
|
| 167 |
+
parts = [f"### Rationale\n{result.rationale}\n"]
|
| 168 |
+
|
| 169 |
+
if features.angles:
|
| 170 |
+
parts.append("### Measurements")
|
| 171 |
+
for key, val in features.angles.items():
|
| 172 |
+
label = key.replace("_", " ").title()
|
| 173 |
+
parts.append(f"- **{label}:** {val:.1f}Β°")
|
| 174 |
+
|
| 175 |
+
if features.alignments:
|
| 176 |
+
parts.append("\n### Alignment Checks")
|
| 177 |
+
for key, val in features.alignments.items():
|
| 178 |
+
label = key.replace("_", " ").title()
|
| 179 |
+
icon = "β" if val else "β"
|
| 180 |
+
parts.append(f"- {icon} {label}")
|
| 181 |
+
|
| 182 |
+
if features.view == "2d":
|
| 183 |
+
parts.append(
|
| 184 |
+
"\n> β οΈ *2D estimate β angles are camera-angle dependent. "
|
| 185 |
+
"For best accuracy, film from the side at hip height.*"
|
| 186 |
+
)
|
| 187 |
+
|
| 188 |
+
return "\n".join(parts)
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
def _render_score_details_judge(judge, rubric, features) -> str:
|
| 192 |
+
"""Render judge + rubric combined breakdown."""
|
| 193 |
+
parts = [f"### Judge Rationale\n{judge.rationale}\n"]
|
| 194 |
+
|
| 195 |
+
if judge.compensation_tags:
|
| 196 |
+
parts.append(f"**Compensations:** {', '.join(judge.compensation_tags)}")
|
| 197 |
+
if judge.corrective_hint:
|
| 198 |
+
parts.append(f"**Corrective:** {judge.corrective_hint}")
|
| 199 |
+
|
| 200 |
+
parts.append(f"\n### Rubric Score: {rubric.score}/3")
|
| 201 |
+
parts.append(f"*{rubric.rationale}*")
|
| 202 |
+
|
| 203 |
+
if features.angles:
|
| 204 |
+
parts.append("\n### Measurements")
|
| 205 |
+
for key, val in features.angles.items():
|
| 206 |
+
label = key.replace("_", " ").title()
|
| 207 |
+
parts.append(f"- **{label}:** {val:.1f}Β°" if isinstance(val, float) else f"- **{label}:** {val}")
|
| 208 |
+
|
| 209 |
+
if features.symmetry_delta is not None:
|
| 210 |
+
parts.append(f"\n### Asymmetry\n- **L/R Delta:** {features.symmetry_delta:.1f}Β°")
|
| 211 |
+
|
| 212 |
+
if features.view == "2d":
|
| 213 |
+
parts.append(
|
| 214 |
+
"\n> β οΈ *2D estimate β angles are camera-angle dependent.*"
|
| 215 |
+
)
|
| 216 |
+
|
| 217 |
+
return "\n".join(parts)
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
def _render_pipeline_status(state) -> str:
|
| 221 |
+
"""Render pipeline step summary."""
|
| 222 |
+
parts = []
|
| 223 |
+
if state.ingest:
|
| 224 |
+
parts.append(
|
| 225 |
+
f"πΉ **Ingest:** {len(state.ingest.frames)} frames Β· "
|
| 226 |
+
f"{state.ingest.fps:.0f}fps Β· {state.ingest.duration:.1f}s Β· "
|
| 227 |
+
f"{state.ingest.width}Γ{state.ingest.height}"
|
| 228 |
+
)
|
| 229 |
+
if state.pose2d:
|
| 230 |
+
n = sum(1 for kps in state.pose2d.keypoints if kps)
|
| 231 |
+
parts.append(
|
| 232 |
+
f"𦴠**Pose2D:** {n}/{len(state.pose2d.keypoints)} frames detected · "
|
| 233 |
+
f"conf={state.pose2d.confidence:.0%}"
|
| 234 |
+
)
|
| 235 |
+
if state.body3d:
|
| 236 |
+
if state.body3d.used:
|
| 237 |
+
parts.append(f"π§ **Body3D:** active Β· conf={state.body3d.confidence:.0%}")
|
| 238 |
+
else:
|
| 239 |
+
parts.append("π§ **Body3D:** 2D-only path (normal)")
|
| 240 |
+
if state.features:
|
| 241 |
+
parts.append(
|
| 242 |
+
f"π **Biomechanics:** view={state.features.view} Β· "
|
| 243 |
+
f"conf={state.features.confidence:.0%}"
|
| 244 |
+
)
|
| 245 |
+
return "\n\n".join(parts) if parts else "*Processing...*"
|
| 246 |
+
|
| 247 |
+
|
| 248 |
+
def _render_alerts(state) -> str:
|
| 249 |
+
"""Render errors and warnings."""
|
| 250 |
+
parts = []
|
| 251 |
+
if state.errors:
|
| 252 |
+
for e in state.errors:
|
| 253 |
+
parts.append(f"π¨ {e}")
|
| 254 |
+
if state.warnings:
|
| 255 |
+
for w in state.warnings:
|
| 256 |
+
parts.append(f"β οΈ {w}")
|
| 257 |
+
return "\n\n".join(parts)
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
def _render_session_table(session_state) -> str:
|
| 261 |
+
"""Render the accumulated 'Session so far' table as markdown."""
|
| 262 |
+
if not session_state or not session_state.entries:
|
| 263 |
+
return "*No clips analysed yet.*"
|
| 264 |
+
lines = ["| Test | Side | Score | Status |", "|---|---|---|---|"]
|
| 265 |
+
for e in session_state.entries:
|
| 266 |
+
test = e.test_name.replace("_", " ").title()
|
| 267 |
+
side = e.side if e.side in ("left", "right") else "β"
|
| 268 |
+
if e.needs_human:
|
| 269 |
+
score, status = "β", "β οΈ Clinician review"
|
| 270 |
+
else:
|
| 271 |
+
score, status = f"{e.score}/3", "β scored"
|
| 272 |
+
lines.append(f"| {test} | {side} | {score} | {status} |")
|
| 273 |
+
return "\n".join(lines)
|
| 274 |
+
|
| 275 |
+
|
| 276 |
+
def _finish_session(session_state):
|
| 277 |
+
"""Build the composite report + PDF for the whole session."""
|
| 278 |
+
if not session_state or not session_state.entries:
|
| 279 |
+
return ("β οΈ No clips analysed yet β analyse at least one clip first.",
|
| 280 |
+
None, None)
|
| 281 |
+
|
| 282 |
+
report, pdf_path = session_mod.finish_session(session_state)
|
| 283 |
+
if report is None:
|
| 284 |
+
return ("β οΈ Nothing to report.", None, None)
|
| 285 |
+
|
| 286 |
+
if report.composite is not None:
|
| 287 |
+
summary = [f"## Composite: {report.composite} / 21"]
|
| 288 |
+
else:
|
| 289 |
+
n = len(session_state.entries)
|
| 290 |
+
summary = [f"## Composite: Incomplete β {n}/7 tests scored",
|
| 291 |
+
"*(One or more tests need clinician review or were unscored.)*"]
|
| 292 |
+
|
| 293 |
+
if report.asymmetries:
|
| 294 |
+
summary.append("\n### Asymmetries")
|
| 295 |
+
for a in report.asymmetries:
|
| 296 |
+
test = a["test"].replace("_", " ").title()
|
| 297 |
+
summary.append(f"- **{test}:** L={a['left_score']} R={a['right_score']} (Ξ {a['delta']})")
|
| 298 |
+
|
| 299 |
+
flags = list(report.low_confidence_flags) + list(report.disagreement_flags)
|
| 300 |
+
if flags:
|
| 301 |
+
summary.append("\n### Flags")
|
| 302 |
+
for fl in flags:
|
| 303 |
+
summary.append(f"- {fl}")
|
| 304 |
+
|
| 305 |
+
md_path = os.path.join(session_state.session_dir, "analysis.md")
|
| 306 |
+
md_out = md_path if os.path.exists(md_path) else None
|
| 307 |
+
return "\n".join(summary), pdf_path, md_out
|
| 308 |
+
|
| 309 |
+
|
| 310 |
+
# βββ App Builder βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 311 |
+
|
| 312 |
+
def build_app() -> gr.Blocks:
|
| 313 |
+
"""Build the FormScout Gradio app with custom scout/trail theme."""
|
| 314 |
+
with gr.Blocks(title="FormScout β FMS Screening Aid") as app:
|
| 315 |
+
|
| 316 |
+
session_state = gr.State(None)
|
| 317 |
+
|
| 318 |
+
# Header
|
| 319 |
+
gr.HTML("""
|
| 320 |
+
<div class="formscout-header">
|
| 321 |
+
<h1>ποΈ FormScout</h1>
|
| 322 |
+
<p style="color: #4a5f57; font-size: 0.95em;">
|
| 323 |
+
Functional Movement Screen Β· Automated Scoring Aid
|
| 324 |
+
</p>
|
| 325 |
+
</div>
|
| 326 |
+
""")
|
| 327 |
+
|
| 328 |
+
# Safety banner (always visible β non-negotiable)
|
| 329 |
+
gr.HTML(f'<div class="safety-banner">{DISCLAIMER}</div>')
|
| 330 |
+
|
| 331 |
+
with gr.Row(equal_height=False):
|
| 332 |
+
# Left column: Input
|
| 333 |
+
with gr.Column(scale=2):
|
| 334 |
+
gr.Markdown("### πΉ Input")
|
| 335 |
+
video_input = gr.Video(label="Upload FMS Video")
|
| 336 |
+
|
| 337 |
+
with gr.Row():
|
| 338 |
+
test_dropdown = gr.Dropdown(
|
| 339 |
+
choices=[name for name, _ in FMS_TESTS],
|
| 340 |
+
value="Deep Squat",
|
| 341 |
+
label="FMS Test",
|
| 342 |
+
scale=2,
|
| 343 |
+
)
|
| 344 |
+
side_dropdown = gr.Dropdown(
|
| 345 |
+
choices=["N/A", "Left", "Right"],
|
| 346 |
+
value="N/A",
|
| 347 |
+
label="Side",
|
| 348 |
+
scale=1,
|
| 349 |
+
)
|
| 350 |
+
|
| 351 |
+
_available_models = config.available_pose_models() or config.POSE_MODELS
|
| 352 |
+
_default_model = (
|
| 353 |
+
config.DEFAULT_POSE_MODEL
|
| 354 |
+
if config.DEFAULT_POSE_MODEL in _available_models
|
| 355 |
+
else list(_available_models.keys())[0]
|
| 356 |
+
)
|
| 357 |
+
pose_model_dropdown = gr.Dropdown(
|
| 358 |
+
choices=list(_available_models.keys()),
|
| 359 |
+
value=_default_model,
|
| 360 |
+
label="Pose Model",
|
| 361 |
+
)
|
| 362 |
+
|
| 363 |
+
overlay_layers = gr.CheckboxGroup(
|
| 364 |
+
choices=["Skeleton", "Trails", "Velocity arrows"],
|
| 365 |
+
value=["Skeleton", "Trails", "Velocity arrows"],
|
| 366 |
+
label="Overlay Layers",
|
| 367 |
+
)
|
| 368 |
+
|
| 369 |
+
submit_btn = gr.Button(
|
| 370 |
+
"π― Score Movement",
|
| 371 |
+
variant="primary",
|
| 372 |
+
size="lg",
|
| 373 |
+
)
|
| 374 |
+
with gr.Row():
|
| 375 |
+
new_clip_btn = gr.Button("β Analyse new clip", visible=False)
|
| 376 |
+
finish_btn = gr.Button("β
Finish & generate PDF",
|
| 377 |
+
variant="primary", visible=False)
|
| 378 |
+
|
| 379 |
+
gr.Markdown(
|
| 380 |
+
"*Tip: Film from the side at hip height for best accuracy. "
|
| 381 |
+
"One athlete, one rep per clip.*",
|
| 382 |
+
elem_classes=["topo-accent"],
|
| 383 |
+
)
|
| 384 |
+
|
| 385 |
+
# Right column: Results
|
| 386 |
+
with gr.Column(scale=3):
|
| 387 |
+
gr.Markdown("### π Results")
|
| 388 |
+
|
| 389 |
+
# Score display
|
| 390 |
+
score_html = gr.HTML(value=_render_empty_state())
|
| 391 |
+
|
| 392 |
+
# Tabs for details
|
| 393 |
+
with gr.Tabs():
|
| 394 |
+
with gr.TabItem("π Rubric Breakdown"):
|
| 395 |
+
score_details = gr.Markdown("")
|
| 396 |
+
|
| 397 |
+
with gr.TabItem("π§ Pipeline"):
|
| 398 |
+
pipeline_md = gr.Markdown("*Waiting for video...*")
|
| 399 |
+
|
| 400 |
+
with gr.TabItem("β οΈ Alerts"):
|
| 401 |
+
alerts_md = gr.Markdown("")
|
| 402 |
+
|
| 403 |
+
with gr.TabItem("π¬ Overlay Video"):
|
| 404 |
+
overlay_video = gr.Video(label="Annotated Movement")
|
| 405 |
+
velocity_md = gr.Markdown("")
|
| 406 |
+
|
| 407 |
+
with gr.TabItem("ποΈ Session"):
|
| 408 |
+
session_table = gr.Markdown("*No clips analysed yet.*")
|
| 409 |
+
finish_summary = gr.Markdown("")
|
| 410 |
+
pdf_file = gr.File(label="Screening Report (PDF)", visible=True)
|
| 411 |
+
md_file = gr.File(label="Analysis Log (Markdown)", visible=True)
|
| 412 |
+
|
| 413 |
+
# Footer safety banner
|
| 414 |
+
gr.HTML(f'<div class="safety-banner" style="margin-top: 20px;">{DISCLAIMER}</div>')
|
| 415 |
+
|
| 416 |
+
gr.Markdown(
|
| 417 |
+
"<center style='color: #6b7d75; font-size: 0.8em; margin-top: 12px;'>"
|
| 418 |
+
"FormScout Β· ~18B params Β· Off the Grid Β· "
|
| 419 |
+
"<a href='https://silastherapy.sk' style='color: #1f6e6e;'>Silas Therapy Β· Build Small Hackathon</a>"
|
| 420 |
+
"</center>"
|
| 421 |
+
)
|
| 422 |
+
|
| 423 |
+
# βββ Event wiring ββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 424 |
+
|
| 425 |
+
def _map_inputs(video, test_display_name, side_display, pose_model_key, overlay_layers, sess):
|
| 426 |
+
"""Map UI display values to internal values and accumulate into the session."""
|
| 427 |
+
test_map = {name: val for name, val in FMS_TESTS}
|
| 428 |
+
test_name = test_map.get(test_display_name, "deep_squat")
|
| 429 |
+
side = {"N/A": "na", "Left": "left", "Right": "right"}.get(side_display, "na")
|
| 430 |
+
return process_video(video, test_name, side, pose_model_key, overlay_layers, sess)
|
| 431 |
+
|
| 432 |
+
submit_btn.click(
|
| 433 |
+
fn=_map_inputs,
|
| 434 |
+
inputs=[video_input, test_dropdown, side_dropdown, pose_model_dropdown,
|
| 435 |
+
overlay_layers, session_state],
|
| 436 |
+
outputs=[session_state, score_html, pipeline_md, score_details, alerts_md,
|
| 437 |
+
overlay_video, velocity_md, session_table, new_clip_btn, finish_btn],
|
| 438 |
+
)
|
| 439 |
+
|
| 440 |
+
def _new_clip():
|
| 441 |
+
"""Clear inputs for the next clip; keep the session intact."""
|
| 442 |
+
return None, _render_empty_state(), ""
|
| 443 |
+
|
| 444 |
+
new_clip_btn.click(
|
| 445 |
+
fn=_new_clip,
|
| 446 |
+
inputs=[],
|
| 447 |
+
outputs=[video_input, score_html, score_details],
|
| 448 |
+
)
|
| 449 |
+
|
| 450 |
+
finish_btn.click(
|
| 451 |
+
fn=_finish_session,
|
| 452 |
+
inputs=[session_state],
|
| 453 |
+
outputs=[finish_summary, pdf_file, md_file],
|
| 454 |
+
)
|
| 455 |
+
|
| 456 |
+
return app
|
| 457 |
+
|
| 458 |
+
|
| 459 |
+
if __name__ == "__main__":
|
| 460 |
+
app = build_app()
|
| 461 |
+
app.launch(theme=formscout_theme(), css=FORMSCOUT_CSS)
|
docs/FormScout-FMS-Spec.md
ADDED
|
@@ -0,0 +1,277 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# FormScout β Functional Movement Screening, scored small
|
| 2 |
+
|
| 3 |
+
**Project specification & architecture documentation**
|
| 4 |
+
*Build Small Hackathon (Gradio Γ Hugging Face) β Track: Backyard AI*
|
| 5 |
+
*Working title; rename freely. Doc version 0.1, June 2026.*
|
| 6 |
+
|
| 7 |
+
---
|
| 8 |
+
|
| 9 |
+
## 1. One-paragraph pitch
|
| 10 |
+
|
| 11 |
+
A basketball team's physiotherapist screens players with the **Functional Movement Screen (FMS)** β seven movement patterns, each scored 0β3 by eye. The scoring is slow, subjective, and hard to reproduce across raters or across months. FormScout is a Gradio app that takes a video of an athlete performing an FMS test, extracts 2D and 3D body pose, measures the biomechanics the FMS rubric actually cares about, and produces a 0β3 score *with a written rationale and an annotated overlay* β anchored to the physio's own previously-scored clips. It is a **screening aid that standardizes and speeds up the physio's first pass**, not a diagnosis and not an injury predictor. Everything runs on models that fit on a laptop.
|
| 12 |
+
|
| 13 |
+
---
|
| 14 |
+
|
| 15 |
+
## 2. The problem, honestly
|
| 16 |
+
|
| 17 |
+
The FMS is a seven-test battery (Deep Squat, Hurdle Step, In-Line Lunge, Shoulder Mobility, Active Straight-Leg Raise, Trunk Stability Push-Up, Rotary Stability), each scored 0β3 for a composite 0β21. A score of 0 means **pain** during the movement and is an automatic red flag for clinical referral. Three of the tests have associated **clearing tests** (shoulder, spinal extension, spinal flexion) that also force a 0 on pain.
|
| 18 |
+
|
| 19 |
+
Two facts shape this project and should be stated plainly in the demo and the writeup:
|
| 20 |
+
|
| 21 |
+
- **Inter-rater reliability is decent but not perfect.** Composite-score reliability is moderate-to-good (ICC roughly 0.7β0.8), but novice and less-experienced raters grade component scores inconsistently. This is the real, addressable pain point: **variance between raters and over time.**
|
| 22 |
+
- **Predictive validity for injury is weak/mixed.** The popular "β€14 = higher injury risk" cutoff is not a reliable predictor on its own. So FormScout must **not** be sold as injury prediction.
|
| 23 |
+
|
| 24 |
+
**Where FormScout genuinely helps:**
|
| 25 |
+
1. A repeatable, objective **digital baseline** to track an athlete over a season.
|
| 26 |
+
2. **Asymmetry detection** (left vs. right), which is one of the FMS's most defensible outputs.
|
| 27 |
+
3. A fast, consistent **first-pass / second opinion** that reduces rater variance.
|
| 28 |
+
4. **Explainability** β it shows *which compensation* it saw, not just a number.
|
| 29 |
+
|
| 30 |
+
This honest framing is also strategic: the Backyard AI track is judged partly on "honest fit between problem and the small-model constraint." Overclaiming clinical power would hurt the submission, not help it.
|
| 31 |
+
|
| 32 |
+
---
|
| 33 |
+
|
| 34 |
+
## 3. Why this fits the hackathon
|
| 35 |
+
|
| 36 |
+
| Hackathon rule | How FormScout satisfies it |
|
| 37 |
+
|---|---|
|
| 38 |
+
| **Total params β€ 32B** | Recommended config sums to ~18B. A portfolio of small specialists beats one monolith β which is on-theme for "think small." |
|
| 39 |
+
| **Built on Gradio, hosted as a HF Space** | Gradio app with `gr.Video` input, a custom-styled results panel, on-Space inference (ZeroGPU or llama.cpp). |
|
| 40 |
+
| **Show, Don't Tell** | Demo video = physio uploads a real player clip, gets a scored overlay in seconds. Social post = before/after of a manual vs. assisted screening session. |
|
| 41 |
+
| **Track: Backyard AI** | The "someone you know" is the team physiotherapist. The deliverable is something they *actually use* on real players. |
|
| 42 |
+
|
| 43 |
+
**Badge targets (aim for all six):**
|
| 44 |
+
|
| 45 |
+
- π **Off the Grid** β no cloud APIs; all models served on the Space.
|
| 46 |
+
- π― **Well-Tuned** β the skeletal-temporal scoring head is fine-tuned on the physio's labels and published to the Hub.
|
| 47 |
+
- π¨ **Off-Brand** β custom Gradio frontend (scorecard UI, video overlay, per-test rubric panel), pushing past default Gradio.
|
| 48 |
+
- π¦ **Llama Champion** β VLM + embedding model served through llama.cpp (GGUF builds exist for both).
|
| 49 |
+
- π‘ **Sharing is Caring** β publish the agent trace (one full screening run, agent by agent) to the Hub.
|
| 50 |
+
- π **Field Notes** β a blog post on building a clinical-adjacent AQA pipeline under a 32B budget, with the honesty section front and center.
|
| 51 |
+
|
| 52 |
+
---
|
| 53 |
+
|
| 54 |
+
## 4. Core technical framing: FMS *is* Action Quality Assessment
|
| 55 |
+
|
| 56 |
+
Don't reinvent this from scratch. **Action Quality Assessment (AQA)** is the established field for "score how well a movement was performed." Skeleton-based AQA (sports scoring, surgical-skill and rehab assessment) is the directly relevant lineage. The "Skeletal-Temporal Transformer" idea maps onto the **AQA scoring head**.
|
| 57 |
+
|
| 58 |
+
The key design constraint is the **tiny labeled dataset** (a couple of physio-scored videos). That rules out training a large score regressor from scratch and dictates a hybrid approach:
|
| 59 |
+
|
| 60 |
+
1. **Deterministic biomechanics** carry most of the load. The FMS rubric is, to a large degree, a set of *angle and alignment thresholds* (e.g. Deep Squat "3" = femur below horizontal, torso parallel to tibia, knees tracking over feet, dowel over feet). These are computable from 3D pose with **zero training** and are inherently interpretable β exactly what earns a physio's trust.
|
| 61 |
+
2. **A small learned head** (ST-GCN or a compact temporal transformer) refines the score and captures the patterns rules miss. It is small enough to fine-tune on a few labeled clips, *especially* if pre-trained on public AQA/pose datasets first.
|
| 62 |
+
3. **Retrieval over the physio's labeled clips** (RAG) gives the language model few-shot anchors at judgment time β the right move when you have examples but not enough to train on.
|
| 63 |
+
4. **A VLM as the judge/explainer** synthesizes rubric + measurements + retrieved exemplars into a final score and a human-readable rationale, and conservatively flags anything pain-related for a human.
|
| 64 |
+
|
| 65 |
+
---
|
| 66 |
+
|
| 67 |
+
## 5. Parameter budget (the single most important table)
|
| 68 |
+
|
| 69 |
+
Assume "total parameters" = **sum of all model weights in the pipeline**. Design to this; confirm the exact interpretation in the Discord AMA.
|
| 70 |
+
|
| 71 |
+
### Recommended config β "Portfolio of specialists" (~18B)
|
| 72 |
+
|
| 73 |
+
| Component | Model | Params | Role |
|
| 74 |
+
|---|---|---:|---|
|
| 75 |
+
| 2D pose + tracking | YOLO26-Pose (L/X) | ~0.05B | Per-frame 17-keypoint skeletons, multi-person tracking |
|
| 76 |
+
| Segmentation | SAM 3.1 (base) | ~0.85B | Clean athlete mask, occlusion handling, prompt for 3D |
|
| 77 |
+
| 3D body | SAM 3D Body | ~0.7β1B* | Single-image 3D mesh β true joint angles, view-invariant |
|
| 78 |
+
| Scoring head | ST-GCN / temporal transformer (fine-tuned) | ~0.01β0.05B | Pose-sequence β candidate 0β3 + confidence |
|
| 79 |
+
| Judge / explainer | Qwen3-VL-8B-Instruct | 8B | Movement ID, rubric reasoning, final score + rationale |
|
| 80 |
+
| Retrieval | Qwen3-VL-Embedding-8B | 8B | Nearest physio-scored reference clips (RAG) |
|
| 81 |
+
| **Total** | | **~17.8B** | Comfortable headroom under 32B |
|
| 82 |
+
|
| 83 |
+
\* SAM 3D Body's exact count isn't published prominently β verify on the model card. It's SAM-3-family and sub-billion-class; budget impact is small either way. The two 8B Qwen models **share the Qwen3-VL-8B backbone** (the embedder is built on the instruct model), which is conceptually clean and operationally efficient.
|
| 84 |
+
|
| 85 |
+
### Alternative config β "Heavy reasoner" (~28.7B)
|
| 86 |
+
|
| 87 |
+
Swap the 8B judge for **Qwen3.6-27B** (multimodal, strong tool-calling, MTP speedups on llama.cpp). Budget then = 27 + ~0.85 + ~1 + small β **28.7B**. This **leaves no room for the 8B embedder**, so you'd drop RAG (or replace it with a sub-0.5B embedder, or use pose-feature similarity for retrieval). Note: Qwen3.6-27B's MTP speculative decoding currently can't run simultaneously with image input (`--mmproj`), so for vision you run it without MTP.
|
| 88 |
+
|
| 89 |
+
**Recommendation: ship the ~18B portfolio config.** RAG over the physio's few labeled clips is worth more than raw reasoning horsepower on this task, the headroom de-risks the budget, and "many small specialists" is the better hackathon story.
|
| 90 |
+
|
| 91 |
+
---
|
| 92 |
+
|
| 93 |
+
## 6. Model selection rationale
|
| 94 |
+
|
| 95 |
+
**YOLO26-Pose** β current-generation YOLO pose; single forward pass for detection + keypoints, NMS-free, real-time even on edge. Tiny param cost. It also handles **multiple people in frame** (important: team videos often have other players/staff visible) and feeds keypoints downstream. Off-the-shelf it predicts COCO human keypoints; can be fine-tuned for custom landmarks (e.g. dowel endpoints) if needed.
|
| 96 |
+
|
| 97 |
+
**SAM 3.1** β gives a clean athlete mask and stable multi-object video tracking (Object Multiplex makes it fast). Two jobs: (a) isolate the target athlete from teammates/background so pose and 3D aren't polluted, (b) provide the mask prompt that SAM 3D Body consumes. Concept prompts ("the person in the blue jersey performing the squat") are a bonus for disambiguation.
|
| 98 |
+
|
| 99 |
+
**SAM 3D Body** β *the addition that makes the scores trustworthy.* FMS criteria are joint angles and symmetry; 2D pose can't measure these reliably across camera angles (projection ambiguity). 3D mesh recovery from a single image, promptable with the 2D keypoints + mask you already have, yields view-invariant joint angles (the MHR rig even separates skeletal structure from soft-tissue shape, which is convenient for angle extraction). This is the difference between "looks bent" and "femur is 4Β° above horizontal β not a 3."
|
| 100 |
+
|
| 101 |
+
**Skeletal-temporal scoring head** β your AQA component and your **Well-Tuned** badge. Recommend a compact **ST-GCN** (graph conv over the skeleton, temporal conv over frames) over a from-scratch transformer, because it's far more data-efficient on a tiny labeled set. Pre-train on public AQA / pose-action data, then fine-tune on the physio's labels. Output: per-test candidate score + a confidence the judge can weigh.
|
| 102 |
+
|
| 103 |
+
**Qwen3-VL-8B-Instruct** β the judge. Strong video temporal modeling (Interleaved-MRoPE, timestamp alignment) suits movement clips. It identifies which of the 7 tests is being performed, reads the biomechanics, considers retrieved exemplars and the head's candidate, and emits the final score + rationale + detected compensation. GGUF β llama.cpp β Llama Champion.
|
| 104 |
+
|
| 105 |
+
**Qwen3-VL-Embedding-8B** β retrieval. Embeds the query clip (or its keyframes/pose-render) and finds the physio's most similar already-scored clips to anchor the judge. Top multimodal retriever on MMEB-V2; same backbone as the judge; GGUF available.
|
| 106 |
+
|
| 107 |
+
---
|
| 108 |
+
|
| 109 |
+
## 7. Architecture β an agentic pipeline
|
| 110 |
+
|
| 111 |
+
Structured as cooperating specialist agents (maps naturally onto an OFP-style orchestration, with a Director coordinating and quality-gating). Each agent has one job and a typed output.
|
| 112 |
+
|
| 113 |
+
```
|
| 114 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 115 |
+
video upload ββββββββΆβ IngestAgent β
|
| 116 |
+
β decode, normalize FPS, sample frames β
|
| 117 |
+
βββββββββββββββββ¬βββββββββββββββββββββββββββββββ
|
| 118 |
+
βΌ
|
| 119 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 120 |
+
β SegmentationAgent (SAM 3.1) β
|
| 121 |
+
β athlete mask + track id (reject teammates) β
|
| 122 |
+
βββββββββββββββββ¬βββββββββββββββββββββββββββββββ
|
| 123 |
+
βΌ
|
| 124 |
+
ββββββββββββββββββββββββββββ΄βββββββββββββββββββββββββββ
|
| 125 |
+
βΌ βΌ
|
| 126 |
+
βββββββββββββββββββββββββββββ βββββββββββββββββββββββββββββ
|
| 127 |
+
β PoseAgent (YOLO26-Pose) β β Body3DAgent (SAM 3D Body) β
|
| 128 |
+
β 2D keypoints per frame β βββkeypoints+maskβββΆ β 3D mesh / joint angles β
|
| 129 |
+
βββββββββββββββββ¬ββββββββββββ βββββββββββββββββ¬ββββββββββββ
|
| 130 |
+
βββββββββββββββββββββββ¬βββββββββββββββββββββββββββββ
|
| 131 |
+
βΌ
|
| 132 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 133 |
+
β MovementClassifierAgent β
|
| 134 |
+
β which of the 7 FMS tests? (VLM or small CLS) β
|
| 135 |
+
βββββββββββββββββ¬βββββββββββββββββββββββββββββββ
|
| 136 |
+
βΌ
|
| 137 |
+
ββββββββββββββββββββββββββββ΄βββββββββββββββββββββββββββ
|
| 138 |
+
βΌ βΌ βΌ
|
| 139 |
+
ββββββββββββββββββββββ βββββββββββββββββββββββββββ ββββββββββββββββββββββββββ
|
| 140 |
+
β BiomechanicsAgent β β ScoringAgent (ST-GCN) β β RetrievalAgent β
|
| 141 |
+
β rubric angles, β β candidate 0β3 + conf β β (Qwen3-VL-Embedding) β
|
| 142 |
+
β ROM, symmetry, β β from pose sequence β β k nearest physio clips β
|
| 143 |
+
β alignment, timing β β β β + their scores β
|
| 144 |
+
βββββββββββ¬βββββββββββ βββββββββββββ¬ββββββββββββββ βββββββββββββ¬βββββββββββββ
|
| 145 |
+
βββββββββββββββββββββββββββββ΄βββββββββββββββββββββββββββ
|
| 146 |
+
βΌ
|
| 147 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 148 |
+
β JudgeAgent (Qwen3-VL-8B) β
|
| 149 |
+
β rubric + measurements + exemplars + candidateβ
|
| 150 |
+
β β final 0β3, rationale, compensation tag, β
|
| 151 |
+
β corrective hint, PAIN/CLEARING β defer β
|
| 152 |
+
βββββββββββββββββ¬βββββββββββββββββββββββββββββββ
|
| 153 |
+
βΌ
|
| 154 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 155 |
+
β ReportAgent β
|
| 156 |
+
β per-test card, composite 0β21, asymmetry β
|
| 157 |
+
β flags, annotated video, exportable PDF β
|
| 158 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 159 |
+
```
|
| 160 |
+
|
| 161 |
+
**Agent contracts (sketch):**
|
| 162 |
+
|
| 163 |
+
- `IngestAgent` β `{frames[], fps, duration, n_people}`
|
| 164 |
+
- `SegmentationAgent` β `{athlete_track_id, masks[]}`
|
| 165 |
+
- `PoseAgent` β `{keypoints_2d[frame][joint]={x,y,conf}}`
|
| 166 |
+
- `Body3DAgent` β `{joints_3d[frame][joint]={x,y,z}, mesh_optional}`
|
| 167 |
+
- `MovementClassifierAgent` β `{test_name, side: left|right|n/a, confidence}`
|
| 168 |
+
- `BiomechanicsAgent` β `{features: {torso_tibia_angle, hip_flexion_deg, knee_valgus_deg, dowel_alignment, L_R_symmetry, ...}}`
|
| 169 |
+
- `ScoringAgent` β `{candidate_score: 0β3, confidence}`
|
| 170 |
+
- `RetrievalAgent` β `{exemplars: [{clip_id, score, similarity}]}`
|
| 171 |
+
- `JudgeAgent` β `{score: 0β3, rationale, compensation_tags[], corrective_hint, needs_human: bool}`
|
| 172 |
+
- `ReportAgent` β `{per_test[], composite, asymmetries[], overlay_video, pdf}`
|
| 173 |
+
|
| 174 |
+
**Quality gating:** if the ST-GCN candidate and the JudgeAgent disagree by β₯1 point, or any agent confidence is low, the report marks the test **"low confidence β physio review recommended."** This keeps the human in the loop and is itself a selling point.
|
| 175 |
+
|
| 176 |
+
---
|
| 177 |
+
|
| 178 |
+
## 8. Scoring methodology, per test
|
| 179 |
+
|
| 180 |
+
The seven tests reduce to measurable quantities. Build a small rubric module β one scoring function per test β that consumes the 3D features and returns a score with the triggering reason. Examples:
|
| 181 |
+
|
| 182 |
+
- **Deep Squat (3):** femur below horizontal AND torso parallel to tibia AND knees tracking over feet AND dowel over feet. **(2):** same but achieved only with heels elevated. **(1):** criteria unmet even with heels elevated. β all four conditions are angle/alignment checks on the 3D pose.
|
| 183 |
+
- **Hurdle Step / In-Line Lunge / Shoulder Mobility / ASLR:** bilateral β score each side, **record the lower** as the test score, and **always emit the asymmetry** even when the score is the same.
|
| 184 |
+
- **Trunk Stability Push-Up / Rotary Stability:** trunk rigidity / timing of limb movement β temporal features from the pose sequence; the ST-GCN head is most valuable here.
|
| 185 |
+
- **Pain / clearing tests (0):** the system **cannot** detect pain. Any clearing test, or a visible distress/abort, sets `needs_human = true` and the test is **not auto-scored**. Defer to the physio. State this loudly.
|
| 186 |
+
|
| 187 |
+
Final composite = sum of seven test scores (0β21), plus an asymmetry summary. The number is never shown without its rationale.
|
| 188 |
+
|
| 189 |
+
---
|
| 190 |
+
|
| 191 |
+
## 9. Data & fine-tuning plan (tiny-dataset survival guide)
|
| 192 |
+
|
| 193 |
+
You have "a couple" of physio-scored clips. Treat them as gold, not as a training set.
|
| 194 |
+
|
| 195 |
+
1. **Deterministic backbone first.** Get the biomechanics rubric working with no training. Validate the measured angles against the physio's scores qualitatively. This alone may be demo-ready.
|
| 196 |
+
2. **Pre-train the ST-GCN** on public pose-action / AQA data (action recognition or generic AQA) so it learns temporal movement structure, not FMS labels.
|
| 197 |
+
3. **Fine-tune on the physio's clips** with heavy augmentation: temporal crops/speed jitter, mirror (leftβright, doubles your bilateral data), camera-angle perturbation in 3D, joint noise. Few-shot, regularized, early-stopped.
|
| 198 |
+
4. **Hold out at least one physio-scored clip** as a sanity check the judge never sees.
|
| 199 |
+
5. **RAG instead of more training.** Every labeled clip goes into the embedding index as a scoring anchor. New clips added later improve the system with no retraining β a nice longitudinal story for the physio.
|
| 200 |
+
6. **Publish the fine-tuned head** to the Hub with a model card (β Well-Tuned badge). Include the augmentation recipe and the honest "trained on N clips, treat as assistive" caveat.
|
| 201 |
+
|
| 202 |
+
**Label schema to collect from the physio** (if you can get a bit more data): `clip_id, athlete_id, test_name, side, score(0β3), pain(bool), compensation_notes, camera_view`. Even 20β30 well-labeled clips meaningfully helps.
|
| 203 |
+
|
| 204 |
+
---
|
| 205 |
+
|
| 206 |
+
## 10. Gradio Space & deployment
|
| 207 |
+
|
| 208 |
+
**UI (targets Off-Brand badge):**
|
| 209 |
+
- `gr.Video` upload (or webcam capture) + a test-type selector (auto-detect, with manual override).
|
| 210 |
+
- Results panel: the 0β3 score as a large dial/patch, the composite 0β21, an asymmetry strip (L/R bars), and the **rationale text**.
|
| 211 |
+
- The annotated overlay video: skeleton + the specific angle that decided the score drawn on the frame where it mattered.
|
| 212 |
+
- A rubric drawer that shows the official 3/2/1 criteria for the detected test, with the met/unmet conditions checked off.
|
| 213 |
+
- A persistent **"Screening aid β not a diagnosis. Pain or clearing tests require a clinician."** banner.
|
| 214 |
+
- Custom CSS / `gr.Server` for a non-default look (scout/trail-map theme would rhyme with the hackathon, and with your design instincts).
|
| 215 |
+
|
| 216 |
+
**Compute:**
|
| 217 |
+
- ZeroGPU (H200 slice) can host the ~18B portfolio; load pose/SAM/3D eagerly, the VLM + embedder via llama.cpp.
|
| 218 |
+
- For **Off the Grid**, ensure zero external API calls β everything served on-Space.
|
| 219 |
+
- For **Llama Champion**, route the VLM + embedding through llama.cpp (GGUF builds exist for Qwen3-VL-8B-Instruct, Qwen3-VL-Embedding-8B, and Qwen3.6-27B). On a Space, watch the CUDA/llama-cpp build flags β recent hackathon Spaces hit `libcudart` issues; a CPU-only or pinned-CUDA build is the usual fix.
|
| 220 |
+
- Persist the embedding index and accumulated labels in Space storage for the longitudinal baseline.
|
| 221 |
+
|
| 222 |
+
---
|
| 223 |
+
|
| 224 |
+
## 11. Clinical safety & ethics (bake this in, don't bolt it on)
|
| 225 |
+
|
| 226 |
+
- **Not a medical device.** Screening aid only. No diagnosis, no injury prediction, no treatment advice beyond generic FMS-style correctives.
|
| 227 |
+
- **Pain is out of scope** for automatic scoring β always defer to the physio.
|
| 228 |
+
- **Human-in-the-loop by design:** low-confidence and disagreement cases are surfaced, not hidden.
|
| 229 |
+
- **Consent & privacy:** athlete videos are biometric data. Get consent; don't log/persist clips beyond what the physio approves; document retention in the writeup.
|
| 230 |
+
- **Honesty in the demo:** show a case the system gets right *and* one it flags as uncertain. Judges (and physios) trust calibrated tools more than confident ones.
|
| 231 |
+
|
| 232 |
+
---
|
| 233 |
+
|
| 234 |
+
## 12. Build plan β two weekends (June 5β15)
|
| 235 |
+
|
| 236 |
+
**Weekend 1 β the spine works end to end:**
|
| 237 |
+
- Day 1: Space scaffold, `gr.Video` in β skeleton overlay out (YOLO26-Pose). Ingest + Segmentation + Pose agents.
|
| 238 |
+
- Day 2: SAM 3D Body integrated; BiomechanicsAgent computing Deep-Squat angles; first deterministic score on a real clip.
|
| 239 |
+
- Goal: upload a squat video, get a rationalized 0β3. *This alone is a viable demo.*
|
| 240 |
+
|
| 241 |
+
**Midweek:** wire the JudgeAgent (Qwen3-VL via llama.cpp), MovementClassifier, and the rubric module for all 7 tests. Attend the AMA β confirm the param-sum interpretation.
|
| 242 |
+
|
| 243 |
+
**Weekend 2 β make it sing:**
|
| 244 |
+
- ST-GCN pre-train + few-shot fine-tune on physio clips; publish to Hub.
|
| 245 |
+
- RetrievalAgent + embedding index over labeled clips.
|
| 246 |
+
- Custom UI polish, asymmetry view, PDF export, safety banners.
|
| 247 |
+
- Record the demo video (physio uses it on a real player), write the social post, publish the agent trace and the blog post.
|
| 248 |
+
|
| 249 |
+
---
|
| 250 |
+
|
| 251 |
+
## 13. Risks & open questions
|
| 252 |
+
|
| 253 |
+
- **Param-sum interpretation** β biggest unknown. The ~18B config is safe under either reading; confirm anyway.
|
| 254 |
+
- **SAM 3D Body on a Space** β verify weights, license, and that it runs within ZeroGPU limits; have a 2D-only fallback (angles from 2D + camera-angle caveats) if it's too heavy.
|
| 255 |
+
- **Single-camera angle limits** even with 3D β note it; recommend a consistent capture protocol (fixed camera position) for the physio, which also improves the longitudinal baseline.
|
| 256 |
+
- **Tiny dataset** β the deterministic rubric must stand on its own so the demo doesn't hinge on the learned head generalizing from a few clips.
|
| 257 |
+
- **llama.cpp + vision build** on Spaces β budget time for the CUDA build dance; CPU fallback for the embedder is fine.
|
| 258 |
+
- **Movement misclassification** β if the wrong test is detected, scoring is meaningless; keep the manual override prominent.
|
| 259 |
+
|
| 260 |
+
---
|
| 261 |
+
|
| 262 |
+
## 14. Quick reference β the stack
|
| 263 |
+
|
| 264 |
+
| Layer | Choice | Badge it helps |
|
| 265 |
+
|---|---|---|
|
| 266 |
+
| 2D pose | YOLO26-Pose | β |
|
| 267 |
+
| Segmentation/track | SAM 3.1 | β |
|
| 268 |
+
| 3D biomechanics | SAM 3D Body | β |
|
| 269 |
+
| Learned scoring | ST-GCN (fine-tuned, published) | Well-Tuned |
|
| 270 |
+
| Judge/explainer | Qwen3-VL-8B-Instruct (llama.cpp) | Llama Champion |
|
| 271 |
+
| Retrieval | Qwen3-VL-Embedding-8B (llama.cpp) | Llama Champion |
|
| 272 |
+
| Serving | On-Space, no cloud APIs | Off the Grid |
|
| 273 |
+
| Frontend | Custom Gradio (scout theme) | Off-Brand |
|
| 274 |
+
| Trace | Published agent run on Hub | Sharing is Caring |
|
| 275 |
+
| Writeup | Blog post w/ honesty section | Field Notes |
|
| 276 |
+
|
| 277 |
+
*Total β 18B params. Honest, explainable, human-in-the-loop, runs on a laptop.*
|
docs/FormScout-Starter-Kit.md
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# FormScout β Starter Kit & Resource Pack
|
| 2 |
+
|
| 3 |
+
Companion to `FormScout-FMS-Spec.md` and `FormScout-Build-Prompt.md`. Every link below was checked. Read Β§1 first β some items are time-sensitive and block the build if you leave them late.
|
| 4 |
+
|
| 5 |
+
---
|
| 6 |
+
|
| 7 |
+
## 1. Do this NOW (before the hack window β some take hours to clear)
|
| 8 |
+
|
| 9 |
+
- [ ] **Request access to the gated Meta checkpoints today.** Both are gated on Hugging Face and approval isn't instant:
|
| 10 |
+
- SAM 3 / SAM 3.1 β request on the SAM 3 repos (you need the latest code for the 3.1 checkpoints).
|
| 11 |
+
- SAM 3D Body β `facebook/sam-3d-body-dinov3` and `facebook/sam-3d-body-vith` both require an access request, then an authenticated download. **Note:** data/checkpoints are blocked in sanctioned jurisdictions β shouldn't affect SK, but verify.
|
| 12 |
+
- [ ] **Put your HF token in the Space secrets** so the Space can pull the gated weights at build time.
|
| 13 |
+
- [ ] **Check licenses before you commit to a model** (this affects whether you can even submit):
|
| 14 |
+
- Qwen3-VL-8B / Qwen3-VL-Embedding-8B / Qwen3.6 β **Apache-2.0** (clean).
|
| 15 |
+
- SAM 3 / SAM 3.1 / SAM 3D Body β **SAM License** (not Apache; read the terms β there are use restrictions).
|
| 16 |
+
- Ultralytics YOLO26 β historically **AGPL-3.0** (open-sourcing obligations; commercial license exists). Verify on the model/repo and make sure an AGPL dependency is OK for your submission. If it's a problem, RTMPose/ViTPose are alternatives.
|
| 17 |
+
- pyskl / MMAction2 β Apache-2.0.
|
| 18 |
+
- KIMORE / UI-PRMD β academic/research terms; check before redistributing anything derived.
|
| 19 |
+
- [ ] **Confirm the param-counting rule in the Discord AMA.** Specifically: (a) is it summed across the pipeline or per-model? (b) do **frozen** base models count? (c) does a LoRA adapter's base count? Your ~18B config is safe under the strict reading either way, but get it on record.
|
| 20 |
+
|
| 21 |
+
---
|
| 22 |
+
|
| 23 |
+
## 2. Literature package
|
| 24 |
+
|
| 25 |
+
### 2.1 The framing that wins β "evaluate like an FMS reliability study"
|
| 26 |
+
|
| 27 |
+
The single most credible move in your writeup: evaluate FormScout the way the clinical literature evaluates human FMS raters. Treat the model as a *second rater* and report **weighted Cohen's ΞΊ** and **ICC** against the physio, the exact metrics the reliability papers use. That instantly makes your results legible to any sports-medicine reader and is far more honest than a vanity accuracy number.
|
| 28 |
+
|
| 29 |
+
| Resource | What it gives you | Link |
|
| 30 |
+
|---|---|---|
|
| 31 |
+
| Physiopedia β FMS | Clean overview of the 7 tests + 0β21 scoring | https://www.physio-pedia.com/Functional_Movement_Screen_(FMS) |
|
| 32 |
+
| FMS reliability study (JOSPT 2012) | The ICC/ΞΊ numbers and method you'll mirror in your eval | https://www.jospt.org/doi/10.2519/jospt.2012.3838 |
|
| 33 |
+
| FMS in elite youth soccer (PMC) | Per-test scores, asymmetries, clearing-test order | https://pmc.ncbi.nlm.nih.gov/articles/PMC5675373/ |
|
| 34 |
+
| Clinician's guide to FMS scoring | Per-test 3/2/1 criteria in plain language (rubric source) | https://meloqdevices.com/blogs/meloq-updates/functional-movement-screening |
|
| 35 |
+
|
| 36 |
+
> **Honesty anchor for the blog post:** the popular "β€14 β injury risk" cutoff has weak/mixed predictive validity. Sell standardization, asymmetry detection, and a repeatable baseline β not prediction.
|
| 37 |
+
|
| 38 |
+
### 2.2 Action Quality Assessment β surveys & living lists
|
| 39 |
+
|
| 40 |
+
| Resource | Why | Link |
|
| 41 |
+
|---|---|---|
|
| 42 |
+
| *A Decade of AQA* (survey, 2025, 200+ papers, PRISMA) | The map of the whole field; start here | https://arxiv.org/abs/2502.02817 Β· code: https://github.com/HaoYin116/Survey_of_AQA |
|
| 43 |
+
| *Comprehensive Survey of AQA: Method & Benchmark* (2024) | Taxonomy by modality (video / **skeleton** / multimodal) + unified benchmark | https://arxiv.org/abs/2412.11149 Β· page: https://zhoukanglei.github.io/AQA-Survey |
|
| 44 |
+
| Awesome-AQA (ZhouKanglei) | Curated, **has a Medical-Care/rehab section** β your closest analogues | https://github.com/ZhouKanglei/Awesome-AQA |
|
| 45 |
+
| Awesome-AQA (Lyman-Smoker) | Second list; catches papers the other misses (FLEX, ExAct, etc.) | https://github.com/Lyman-Smoker/Awesome-AQA |
|
| 46 |
+
|
| 47 |
+
### 2.3 Skeleton-based scoring β the methods your head will borrow from
|
| 48 |
+
|
| 49 |
+
| Paper | Relevance to FormScout | Link |
|
| 50 |
+
|---|---|---|
|
| 51 |
+
| ST-GCN (original) | The graph-over-skeleton + temporal-conv backbone | https://github.com/open-mmlab/mmaction2/blob/main/configs/skeleton/stgcn/README.md |
|
| 52 |
+
| AQA via Hierarchical **Pose-guided** Multi-Stage Contrastive Regression (TIP 2025) | Pose-guided + contrastive regression with few labels β close to your setup | https://arxiv.org/abs/2501.03674 |
|
| 53 |
+
| Attention-guided Movement **Quality** Assessment + skeletal augmentation (UI-PRMD/KIMORE) | Transformer MQA on clinician-scored rehab data; **augmentation recipe for tiny sets** | https://arxiv.org/pdf/2204.07840 |
|
| 54 |
+
| SSL-Rehab: self-supervised 3D skeleton + **LoRA** fine-tune (KIMORE/UI-PRMD) | PretrainβLoRA recipe for small clinical datasets (uses your LoRA muscle) | https://www.sciencedirect.com/science/article/abs/pii/S1077314224003564 |
|
| 55 |
+
| Skeleton-based AQA w/ anomaly-aware DTW (Sensors 2025) | DTW alignment + anomaly scoring; cheap, label-light baseline | https://www.ncbi.nlm.nih.gov/pmc/articles/PMC12693942/ |
|
| 56 |
+
|
| 57 |
+
---
|
| 58 |
+
|
| 59 |
+
## 3. Models & tooling (verified)
|
| 60 |
+
|
| 61 |
+
| Component | Repo / card | Params | License | Gated? |
|
| 62 |
+
|---|---|---:|---|---|
|
| 63 |
+
| YOLO26-Pose | https://docs.ultralytics.com/tasks/pose | <0.1B | AGPL-3.0* | no |
|
| 64 |
+
| SAM 3.1 | https://github.com/facebookresearch/sam3 | ~0.85B | SAM License | **yes** |
|
| 65 |
+
| SAM 3D Body | https://github.com/facebookresearch/sam-3d-body Β· https://huggingface.co/facebook/sam-3d-body-dinov3 | sub-1Bβ | SAM License | **yes** |
|
| 66 |
+
| ST-GCN++ / PoseConv3D | https://github.com/kennymckormick/pyskl | ~0.01β0.05B | Apache-2.0 | no |
|
| 67 |
+
| Qwen3-VL-8B-Instruct | https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct | 8B | Apache-2.0 | no |
|
| 68 |
+
| Qwen3-VL-Embedding-8B | https://huggingface.co/Qwen/Qwen3-VL-Embedding-8B (GGUF: dam2452/...-GGUF) | 8B | Apache-2.0 | no |
|
| 69 |
+
| Qwen3.6-27B (alt brain) | https://huggingface.co/unsloth/Qwen3.6-27B-GGUF | 27B | Apache-2.0 | no |
|
| 70 |
+
|
| 71 |
+
\* verify the current YOLO26 license. β two variants (`dinov3`, `vith`); confirm exact count on the card β budget impact is small either way. SAM 3 itself is 848M.
|
| 72 |
+
|
| 73 |
+
**Useful extras:** SAM 3D Body uses a Momentum Human Rig (MHR) that separates skeleton from soft-tissue shape β convenient for clean joint-angle extraction. The repo ships a notebook combining SAM 3D Body + SAM 3D Objects in one frame of reference. SAM 3D Body demo: https://www.aidemos.meta.com/segment-anything/editor/convert-body-to-3d
|
| 74 |
+
|
| 75 |
+
---
|
| 76 |
+
|
| 77 |
+
## 4. Datasets for transfer / pretraining
|
| 78 |
+
|
| 79 |
+
You have a couple of labeled clips. Pretrain on clinician-scored movement-quality data first, then few-shot fine-tune. These are the most transferable to FMS (ranked by relevance):
|
| 80 |
+
|
| 81 |
+
| Dataset | Why it's the closest analogue | Link |
|
| 82 |
+
|---|---|---|
|
| 83 |
+
| **KIMORE** | Clinician **scores** of low-back-pain rehab exercises (trunk control, multi-plane) β same "score movement quality" task as FMS; partially overlaps Deep Squat / Rotary Stability / TSPU mechanics | https://www.researchgate.net/publication/333791841 (search "KIMORE dataset") |
|
| 84 |
+
| **UI-PRMD** | 10 rehab movements, correct vs. incorrect executions; standard MQA benchmark, pairs with KIMORE | search "UI-PRMD University of Idaho Physical Rehabilitation Movements" |
|
| 85 |
+
| **Fitness-AQA** | Real gym **squat/deadlift form errors** β directly relevant to Deep Squat compensations | https://github.com/ParitoshParmar/MTL-AQA (links Fitness-AQA) |
|
| 86 |
+
| **FLEX** | Large multi-modal fitness AQA dataset | via Lyman-Smoker/Awesome-AQA |
|
| 87 |
+
| **MTL-AQA / AQA-7 / FineFS** | General sports AQA for backbone pretraining (diving, skating) | https://github.com/ParitoshParmar/MTL-AQA |
|
| 88 |
+
|
| 89 |
+
**FMS-specific public video data is scarce** β don't expect a drop-in set. Your physio's clips are the gold; everything above is for pretraining the temporal backbone so it learns movement structure before it ever sees an FMS label.
|
| 90 |
+
|
| 91 |
+
---
|
| 92 |
+
|
| 93 |
+
## 5. Build & deploy tooling
|
| 94 |
+
|
| 95 |
+
| Need | Link |
|
| 96 |
+
|---|---|
|
| 97 |
+
| Gradio docs (v6) | https://www.gradio.app/docs |
|
| 98 |
+
| `gradio.Server` β custom frontend + Gradio backend (Off-Brand badge) | https://www.gradio.app/guides/server-mode Β· blog: https://huggingface.co/blog/introducing-gradio-server |
|
| 99 |
+
| Gradio AI coding-assistant skill | `gradio skills add --claude` (PyPI: https://pypi.org/project/gradio/) |
|
| 100 |
+
| Gradio changelog (confirm `gr.Walkthrough`, `gr.Navbar`, `gr.Video.playback_position`) | https://www.gradio.app/changelog |
|
| 101 |
+
| HF Spaces ZeroGPU (`@spaces.GPU`) | https://huggingface.co/docs/hub/spaces-zerogpu |
|
| 102 |
+
| llama.cpp | https://github.com/ggml-org/llama.cpp |
|
| 103 |
+
| pyskl (ST-GCN++/PoseConv3D, custom-video tutorial incl. diving48) | https://github.com/kennymckormick/pyskl |
|
| 104 |
+
| MMAction2 (broader video understanding) | https://github.com/open-mmlab/mmaction2 |
|
| 105 |
+
| Hackathon's own trailheads (ML Intern, Gradio guides) | https://github.com/huggingface/ml-intern |
|
| 106 |
+
|
| 107 |
+
> **Hackathon-specific gotcha already seen in the org:** another team's Space hit `libcudart.so.12` errors and had to swap llama.cpp for transformers + `spaces.GPU`. Plan for it β isolate the llama.cpp build (CPU-only or pinned-CUDA) and keep a transformers fallback. For the scoring head, a small hand-rolled ST-GCN may deploy more cleanly on a Space than the full MMAction2/pyskl stack β prototype with pyskl, ship lean.
|
| 108 |
+
|
| 109 |
+
---
|
| 110 |
+
|
| 111 |
+
## 6. Two artifacts you probably haven't made yet
|
| 112 |
+
|
| 113 |
+
### 6.1 Data & capture protocol (highest-leverage non-code work)
|
| 114 |
+
|
| 115 |
+
With a tiny dataset, controlling *how* clips are captured beats any model tweak. Give the physio a one-pager:
|
| 116 |
+
|
| 117 |
+
- **Camera:** one fixed position, tripod, ~3 m back, lens at hip height, landscape, 1080p/30fps+. Same setup every session β this is what makes 3D consistent and the longitudinal baseline meaningful.
|
| 118 |
+
- **Framing:** whole body in frame for the whole rep, including the dowel. Plain-ish background, even lighting, no backlight.
|
| 119 |
+
- **One athlete in frame** at scoring time (or note who to track). For bilateral tests, capture **both sides** and label each.
|
| 120 |
+
- **Label schema (CSV):** `clip_id, athlete_id, date, test_name, side(L/R/NA), score(0β3), pain(bool), compensation_notes(free text), camera_view, consent_on_file(bool)`.
|
| 121 |
+
- **One rep per clip** to start (simplest). If sessions are continuous, you'll need temporal segmentation first β flag it to the build agent at Phase 1.
|
| 122 |
+
|
| 123 |
+
### 6.2 Evaluation plan
|
| 124 |
+
|
| 125 |
+
Define "good" before you train, given so few labels:
|
| 126 |
+
|
| 127 |
+
- **Primary:** Spearman Ο between predicted and physio scores (the AQA-standard metric), plus **exact-match** and **Β±1 accuracy** per test.
|
| 128 |
+
- **Clinical credibility:** **weighted Cohen's ΞΊ** and **ICC** of model-vs-physio, reported alongside the human inter-rater numbers from the JOSPT study β i.e. "how does FormScout compare to a second human rater?"
|
| 129 |
+
- **Asymmetry:** detection rate of L/R asymmetries the physio flagged (this is one of the FMS's most defensible outputs).
|
| 130 |
+
- **Validation:** leave-one-clip-out CV (you can't afford a held-out test split). Keep β₯1 clip the judge never sees for the demo.
|
| 131 |
+
- **Calibration:** report when the system says "low confidence / physio review" and show it's right to do so. A well-calibrated, humble tool reads as more trustworthy than a confident one.
|
| 132 |
+
|
| 133 |
+
---
|
| 134 |
+
|
| 135 |
+
## 7. Ethics, consent & data handling (EU / Slovakia)
|
| 136 |
+
|
| 137 |
+
You're filming identifiable athletes, possibly **minors** on a youth team. This is biometric personal data under GDPR β treat it as first-class, and say so in your submission (judges and physios both reward it):
|
| 138 |
+
|
| 139 |
+
- **Consent:** written consent from each athlete (and a parent/guardian for anyone under 18) before any footage is used. No consent β not in the dataset, not in the demo.
|
| 140 |
+
- **Data minimization & retention:** keep only what you need; don't persist raw clips on the Space beyond what's approved; document a retention/deletion policy. Prefer storing derived skeletons over raw video where possible.
|
| 141 |
+
- **Demo footage:** use a consenting adult (you, a teammate) for the public demo video rather than a minor athlete, even if you trained on team data privately.
|
| 142 |
+
- **Framing:** screening aid, not a medical device; pain/clearing tests always defer to the clinician; human-in-the-loop by design.
|
| 143 |
+
|
| 144 |
+
---
|
| 145 |
+
|
| 146 |
+
## 8. The transfer-learning recipe (ties it together)
|
| 147 |
+
|
| 148 |
+
1. **Backbone pretrain** β ST-GCN++ on a general skeleton-action set (NTU/Kinetics skeletons via pyskl) so it learns motion structure.
|
| 149 |
+
2. **Domain adapt** β continue on **KIMORE + UI-PRMD** (clinician-scored movement quality) so it learns *quality*, not just *what action*.
|
| 150 |
+
3. **Few-shot fine-tune** β **LoRA** on the physio's FMS clips with heavy augmentation (temporal jitter, **LβR mirror** to double bilateral data, 3D camera-angle perturbation, joint noise). The SSL-Rehab paper (Β§2.3) is your blueprint and it's exactly your LoRA wheelhouse.
|
| 151 |
+
4. **Don't over-train the head** β let deterministic biomechanics carry the demo; the learned head and RAG are the refinement and the badges, not the foundation.
|
| 152 |
+
|
| 153 |
+
---
|
| 154 |
+
|
| 155 |
+
## 9. Demo & submission storyboard (the "make it sing" 30%)
|
| 156 |
+
|
| 157 |
+
The submission needs a demo video + social post; "Show, Don't Tell" is a literal rule. A tight 60β90s cut:
|
| 158 |
+
|
| 159 |
+
1. **0β10s** β the problem: physio eyeballing a squat, scribbling a score. "Same player, two raters, two scores."
|
| 160 |
+
2. **10β35s** β upload the clip to FormScout β skeleton overlay β 0β3 with the *deciding angle drawn on the frame* (`playback_position` jump). The "aha" shot.
|
| 161 |
+
3. **35β55s** β the scorecard: composite 0β21, the L/R asymmetry strip, a "low confidence β physio review" flag on a borderline case (honesty sells).
|
| 162 |
+
4. **55β75s** β the physio reacting / using it on a real player (the Backyard AI "they actually used it" proof).
|
| 163 |
+
5. **End card** β "Runs on a laptop. ~18B params. Screening aid, not a diagnosis." Link the Space, the published head, the agent trace, the blog.
|
| 164 |
+
|
| 165 |
+
Social post: lead with the overlay GIF + the asymmetry-detection angle; tag Gradio/HF; one line of honest framing.
|
| 166 |
+
|
| 167 |
+
---
|
| 168 |
+
|
| 169 |
+
*Built to give FormScout the best shot. The two things most teams underinvest in β the capture protocol (Β§6.1) and the honest, clinical-style evaluation (Β§6.2, Β§2.1) β are exactly where this project can out-class flashier entries. Good luck. π*
|
docs/plans/FormScout-Build-Prompt.md
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Build Prompt β FormScout (FMS scoring on Gradio, β€32B)
|
| 2 |
+
|
| 3 |
+
> **How to use this:** paste everything below the line into your coding agent (Claude Code, Codex, Cursor, etc.) as the opening instruction. Attach `FormScout-FMS-Spec.md` alongside it β that file is the product source of truth; this file is the engineering contract and process. Work through it phase by phase.
|
| 4 |
+
|
| 5 |
+
---
|
| 6 |
+
|
| 7 |
+
## ROLE
|
| 8 |
+
|
| 9 |
+
You are a **senior Python + Gradio architect with ~10 years of shipping ML web apps**, including production Hugging Face Spaces, custom-frontend Gradio deployments, ZeroGPU services, and llama.cpp-served models. You are pragmatic, opinionated about defaults, allergic to dead code, and you **verify APIs against current docs instead of trusting your memory** β Gradio and the model ecosystem move fast and your training data may be stale. You build **vertical slices** that run end to end early, then deepen. You never hand back a broken app.
|
| 10 |
+
|
| 11 |
+
## MISSION
|
| 12 |
+
|
| 13 |
+
Build **FormScout**, a Gradio app hosted as a Hugging Face Space that scores Functional Movement Screen (FMS) videos 0β3 per test with an explainable rationale and an annotated overlay, for the Build Small Hackathon (Backyard AI track). Full product requirements are in the attached `FormScout-FMS-Spec.md`. Honor it; if you deviate, say why.
|
| 14 |
+
|
| 15 |
+
## PRIME DIRECTIVES (read before writing any code)
|
| 16 |
+
|
| 17 |
+
1. **Verify before you build.** Do Phase 0 recon first. Do not write against a Gradio/model API you have not confirmed exists in the current version. When unsure, read the doc or the model card, don't guess.
|
| 18 |
+
2. **Vertical slice first.** The fastest path to a working `video in β scored overlay out` for *one* test beats a half-built version of all seven. Get something running on day one, then expand.
|
| 19 |
+
3. **Stay under budget.** Total model parameters across the whole pipeline must be **β€ 32B**. Track a running sum in `MODEL_BUDGET.md` and update it whenever you add or swap a model. The target config is ~18B (see spec Β§5). If a choice would exceed 32B, stop and flag it.
|
| 20 |
+
4. **No cloud model APIs.** All inference runs on the Space (Off the Grid badge). No OpenAI/Anthropic/Gemini/etc. calls for the core pipeline.
|
| 21 |
+
5. **Honesty & safety are features, not footnotes.** This is a screening aid, not a diagnosis and not injury prediction. Pain and clearing tests are never auto-scored β they set `needs_human=true`. A safety banner is always visible. Low-confidence and agent-disagreement cases are surfaced, not hidden.
|
| 22 |
+
6. **Modular agents, typed contracts.** Each pipeline stage is an independent module with a typed input/output (see spec Β§7). No god-functions. The pipeline must be runnable headless (no Gradio) for testing.
|
| 23 |
+
|
| 24 |
+
---
|
| 25 |
+
|
| 26 |
+
## PHASE 0 β Recon & environment (do this first, report findings before coding)
|
| 27 |
+
|
| 28 |
+
**Goal:** confirm the ground truth, then write a short `RECON.md` summarizing what you found and any deviations from the spec.
|
| 29 |
+
|
| 30 |
+
1. **Install the Gradio skill** for this agent so you get current Gradio knowledge:
|
| 31 |
+
`gradio skills add --claude` (use the right flag for your agent; `--global` is fine).
|
| 32 |
+
2. **Pin and confirm Gradio.** Determine the current major version (expect Gradio 6.x). Record the exact version you'll target in `requirements.txt`. Confirm these still exist and note their current signatures:
|
| 33 |
+
- `gr.Blocks`, `gr.Video` (incl. `playback_position` for jumping to the decisive frame), `gr.Walkthrough` / `gr.Step` (for the 7-test flow), `gr.Navbar` (multipage), custom theming / CSS.
|
| 34 |
+
- `gradio.Server` (custom-frontend mode) β decide **Blocks vs Server** for the UI (see UI section).
|
| 35 |
+
- ZeroGPU usage: the `@spaces.GPU` decorator pattern, and the caveat that with `gradio.Server` + ZeroGPU you must call endpoints via `@gradio/client` from the browser.
|
| 36 |
+
3. **Verify every model** on its Hugging Face card β confirm it exists, its **license**, its **parameter count**, and whether a **GGUF** build exists for llama.cpp:
|
| 37 |
+
- YOLO26-Pose (Ultralytics) β pick a variant (l/x) and confirm license implications.
|
| 38 |
+
- SAM 3.1 (`facebookresearch/sam3`) β base checkpoint size.
|
| 39 |
+
- **SAM 3D Body** β *this is the uncertain one.* Confirm weights are public, the license, the **exact param count**, and that it runs within a ZeroGPU slice. If it's too heavy or not usable, fall back to **2D-only biomechanics** (angles from 2D pose + explicit camera-angle caveats) and note it.
|
| 40 |
+
- Qwen3-VL-8B-Instruct + Qwen3-VL-Embedding-8B β confirm GGUF builds and that they share the Qwen3-VL backbone.
|
| 41 |
+
4. **llama.cpp on Spaces reality check.** Confirm a working install path; prior hackathon Spaces hit `libcudart.so` errors. Decide CPU-only vs pinned-CUDA build per model. Have a `transformers`/`spaces.GPU` fallback ready for any model that won't build under llama.cpp in time.
|
| 42 |
+
5. **Open question to surface, not solve:** does "total parameters β€ 32B" mean *per model* or *summed across the pipeline*? Design for the **summed** reading (safe under either). Note in `RECON.md` to confirm via the Discord AMA.
|
| 43 |
+
|
| 44 |
+
**Exit criteria for Phase 0:** `RECON.md` exists with the Gradio version, a verified model table (name, params, license, GGUF y/n, runs-on-ZeroGPU y/n), the running param sum, the chosen UI approach, and any fallbacks triggered.
|
| 45 |
+
|
| 46 |
+
---
|
| 47 |
+
|
| 48 |
+
## PHASE 1 β The spine (one test, end to end, headless + Gradio)
|
| 49 |
+
|
| 50 |
+
**Goal:** upload a Deep Squat clip β get a rationalized 0β3 + skeleton overlay.
|
| 51 |
+
|
| 52 |
+
- Scaffold the repo (structure below). Pipeline runs **headless** via `python -m formscout.run sample.mp4` before any UI.
|
| 53 |
+
- Implement `IngestAgent` β `SegmentationAgent` (SAM 3.1) β `PoseAgent` (YOLO26-Pose). Reject non-target people via the mask/track id.
|
| 54 |
+
- Implement `Body3DAgent` (SAM 3D Body) **or** the 2D fallback from Phase 0.
|
| 55 |
+
- Implement `BiomechanicsAgent` for Deep Squat only: torsoβtibia angle, hip-flexion depth (femur vs horizontal), knee tracking, dowel alignment.
|
| 56 |
+
- Implement a **deterministic** rubric scorer for Deep Squat (3/2/1 per spec Β§8). No ML scoring yet.
|
| 57 |
+
- Minimal Gradio UI: `gr.Video` in, score + rationale + overlay out.
|
| 58 |
+
|
| 59 |
+
**Exit criteria:** a real squat clip produces a defensible score, a one-line reason citing the deciding measurement, and an overlay video. Runs on the Space.
|
| 60 |
+
|
| 61 |
+
---
|
| 62 |
+
|
| 63 |
+
## PHASE 2 β All seven tests + the judge
|
| 64 |
+
|
| 65 |
+
- Extend `BiomechanicsAgent` + rubric scorers to all 7 tests. Bilateral tests score each side, **report the lower**, and **always emit the asymmetry**.
|
| 66 |
+
- `MovementClassifierAgent`: identify which test is in the clip (VLM or a small classifier) with a **manual override** in the UI.
|
| 67 |
+
- `JudgeAgent` (Qwen3-VL-8B via llama.cpp): consumes rubric + measurements + the deterministic candidate β final 0β3, rationale, compensation tag, corrective hint. Pain/clearing β `needs_human=true`, **not scored**.
|
| 68 |
+
- `ReportAgent`: per-test card, composite 0β21, asymmetry strip, annotated overlay, PDF export.
|
| 69 |
+
|
| 70 |
+
**Exit criteria:** a multi-test session produces a full scorecard with composite + asymmetries; pain/clearing cases defer to human; disagreements between deterministic and judge scores are flagged.
|
| 71 |
+
|
| 72 |
+
---
|
| 73 |
+
|
| 74 |
+
## PHASE 3 β Learned scoring + retrieval (the badges)
|
| 75 |
+
|
| 76 |
+
- `ScoringAgent`: compact **ST-GCN** scoring head. Pre-train on public AQA/pose data, then **few-shot fine-tune** on the physio's labeled clips with heavy augmentation (temporal jitter, **leftβright mirror**, 3D camera-angle perturbation, joint noise). Hold out β₯1 labeled clip. **Publish the fine-tuned head to the Hub** with an honest model card β *Well-Tuned*.
|
| 77 |
+
- `RetrievalAgent`: build a Qwen3-VL-Embedding-8B index over the physio's labeled clips; return k nearest + their scores to anchor the judge β RAG.
|
| 78 |
+
- Wire the judge to weigh: deterministic candidate + ST-GCN candidate + retrieved exemplars.
|
| 79 |
+
|
| 80 |
+
**Exit criteria:** scores incorporate the learned head and exemplars; adding a new labeled clip improves retrieval with **no retraining**.
|
| 81 |
+
|
| 82 |
+
---
|
| 83 |
+
|
| 84 |
+
## PHASE 4 β Polish, ship, document
|
| 85 |
+
|
| 86 |
+
- Custom UI pass (Off-Brand): scout/trail theme, score dial, asymmetry bars, rubric drawer with met/unmet checkboxes, decisive-frame jump via `playback_position`, persistent safety banner.
|
| 87 |
+
- Persist the embedding index + accumulated labels in Space storage (longitudinal baseline).
|
| 88 |
+
- **Publish one full agent trace** to the Hub (every agent's I/O for one run) β *Sharing is Caring*.
|
| 89 |
+
- Write the **blog post / field notes** with the honesty section front-and-center β *Field Notes*.
|
| 90 |
+
- Record the demo video (physio scores a real player) + the social post.
|
| 91 |
+
|
| 92 |
+
**Exit criteria:** all six badges attempted, Space is green, demo + post + trace + blog are linked from the README.
|
| 93 |
+
|
| 94 |
+
---
|
| 95 |
+
|
| 96 |
+
## REPO STRUCTURE (target)
|
| 97 |
+
|
| 98 |
+
```
|
| 99 |
+
formscout/
|
| 100 |
+
app.py # Gradio entrypoint (Blocks or Server)
|
| 101 |
+
formscout/
|
| 102 |
+
__init__.py
|
| 103 |
+
config.py # paths, model ids, thresholds, feature flags
|
| 104 |
+
pipeline.py # Director: orchestrates agents, quality-gates
|
| 105 |
+
run.py # headless CLI entrypoint (no Gradio)
|
| 106 |
+
agents/
|
| 107 |
+
ingest.py
|
| 108 |
+
segmentation.py # SAM 3.1
|
| 109 |
+
pose2d.py # YOLO26-Pose
|
| 110 |
+
body3d.py # SAM 3D Body (+ 2d fallback)
|
| 111 |
+
classify.py # movement classifier
|
| 112 |
+
biomechanics.py # rubric features per test
|
| 113 |
+
scoring.py # ST-GCN learned head
|
| 114 |
+
retrieval.py # Qwen3-VL-Embedding index
|
| 115 |
+
judge.py # Qwen3-VL-8B judge
|
| 116 |
+
report.py # scorecard, overlay, pdf
|
| 117 |
+
rubric/
|
| 118 |
+
deep_squat.py ... # one scorer per FMS test, pure functions
|
| 119 |
+
types.py # typed dataclasses for every agent contract
|
| 120 |
+
serving/
|
| 121 |
+
llama_cpp.py # llama.cpp client wrappers + fallbacks
|
| 122 |
+
ui/
|
| 123 |
+
theme.py, components.py, custom/ # frontend assets
|
| 124 |
+
tracing.py # structured per-agent I/O logging (for the trace badge)
|
| 125 |
+
tests/ # headless tests per agent + a golden-clip e2e test
|
| 126 |
+
requirements.txt
|
| 127 |
+
README.md # Space card: pitch, demo, trace, blog, safety
|
| 128 |
+
MODEL_BUDGET.md # running param sum, must stay β€32B
|
| 129 |
+
RECON.md # Phase 0 findings
|
| 130 |
+
```
|
| 131 |
+
|
| 132 |
+
## ENGINEERING STANDARDS
|
| 133 |
+
|
| 134 |
+
- **Typing everywhere.** Every agent takes and returns a dataclass from `types.py`. Validate at boundaries.
|
| 135 |
+
- **Pure rubric functions.** Each test scorer is a pure function `(features) -> ScoreResult` with the triggering reason. Unit-test each against hand-computed cases.
|
| 136 |
+
- **Defensive by default.** Handle: no person detected, multiple people, wrong/ambiguous test, occlusion, too-short clip, bad FPS, 3D model OOM. Degrade gracefully and tell the user what happened β never crash the Space.
|
| 137 |
+
- **Confidence is first-class.** Every agent emits a confidence; the Director flags low confidence and β₯1-point judge/ST-GCN disagreement as "physio review recommended."
|
| 138 |
+
- **Config over constants.** Thresholds, model ids, k for retrieval, feature flags live in `config.py`, not scattered literals.
|
| 139 |
+
- **Tracing for free badge.** `tracing.py` records structured per-agent inputs/outputs for any run; one run gets exported for the Hub trace.
|
| 140 |
+
- **Determinism in demos.** Fix seeds; cache model loads at startup; warm the pipeline so the demo isn't a cold-start.
|
| 141 |
+
- **Tests:** per-agent unit tests on fixtures + one golden-clip end-to-end test asserting score, `needs_human`, and overlay presence. Keep a tiny committed sample clip.
|
| 142 |
+
|
| 143 |
+
## GRADIO-SPECIFIC GUIDANCE
|
| 144 |
+
|
| 145 |
+
- **Blocks vs Server:** start with `gr.Blocks` + custom CSS/theme β fastest to a polished result and enough for Off-Brand. Escalate to `gradio.Server` with your own frontend **only if** Blocks can't express the UI; document the reason. (Server still gives queuing, ZeroGPU, MCP.)
|
| 146 |
+
- Use `gr.Walkthrough`/`gr.Step` to guide the physio through a 7-test session; `gr.Navbar` if you split pages.
|
| 147 |
+
- Use `gr.Video`'s `playback_position` to jump the result video to the frame that decided the score.
|
| 148 |
+
- ZeroGPU: wrap heavy inference in `@spaces.GPU`; load models once at module scope; mind the per-call GPU time limit. If using `gradio.Server` + ZeroGPU, call endpoints via `@gradio/client` from the browser.
|
| 149 |
+
- `requirements.txt`: pin Gradio and every model lib; isolate the llama.cpp build (CPU-only or pinned-CUDA) to dodge `libcudart` failures; keep a `transformers` + `spaces.GPU` fallback path.
|
| 150 |
+
|
| 151 |
+
## DEFINITION OF DONE (badge checklist)
|
| 152 |
+
|
| 153 |
+
- [ ] Space runs green; upload β scorecard works on real clips.
|
| 154 |
+
- [ ] Param sum verified β€ 32B in `MODEL_BUDGET.md`.
|
| 155 |
+
- [ ] π No cloud model APIs anywhere in the pipeline.
|
| 156 |
+
- [ ] π― Fine-tuned ST-GCN head published to the Hub w/ honest card.
|
| 157 |
+
- [ ] π¨ Custom, non-default Gradio UI.
|
| 158 |
+
- [ ] π¦ VLM + embedder served via llama.cpp.
|
| 159 |
+
- [ ] π‘ One full agent trace published to the Hub.
|
| 160 |
+
- [ ] π Blog post / field notes written, honesty section included.
|
| 161 |
+
- [ ] Demo video + social post recorded.
|
| 162 |
+
- [ ] Safety banner present; pain/clearing never auto-scored; low-confidence flagged.
|
| 163 |
+
|
| 164 |
+
## INTERACTION PROTOCOL
|
| 165 |
+
|
| 166 |
+
- **After each phase**, post: what runs now, the updated param sum, deviations from the spec, and the next step. Don't silently change architecture.
|
| 167 |
+
- **Ask the human only when blocked on a real decision** β e.g. single-test clips vs continuous sessions (changes segmentation + UI), SAM 3D Body unusable (triggers 2D fallback), or the param-sum interpretation. Otherwise proceed with the spec's defaults and note your assumption inline.
|
| 168 |
+
- **Never claim a Gradio/model API works without having verified it** this session. If you didn't check it, say so.
|
docs/superpowers/plans/2026-06-04-formscout-full-build.md
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
docs/superpowers/plans/2026-06-09-pose-model-selector.md
ADDED
|
@@ -0,0 +1,734 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Pose Model Selector Implementation Plan
|
| 2 |
+
|
| 3 |
+
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
| 4 |
+
|
| 5 |
+
**Goal:** Replace the hard-coded YOLO26l default with a 10-model dropdown (MediaPipe, YOLO26 nβx, Sapiens2 0.4Bβ5B) wired end-to-end from UI through the Director to `Pose2DAgent`.
|
| 6 |
+
|
| 7 |
+
**Architecture:** Unified `POSE_MODELS` registry in `config.py` drives a `gr.Dropdown` in `app.py`; the selected key flows through `Director.run()` into `Pose2DAgent.run(model_key)`, which dispatches to one of three private sub-runners (`_run_yolo`, `_run_mediapipe`, `_run_sapiens2`), all producing the same COCO-17 `list[dict]` contract.
|
| 8 |
+
|
| 9 |
+
**Tech Stack:** `ultralytics` (YOLO), `onnxruntime` + `huggingface_hub` (MediaPipe), `transformers` (Sapiens2), `gradio` (UI).
|
| 10 |
+
|
| 11 |
+
---
|
| 12 |
+
|
| 13 |
+
## File map
|
| 14 |
+
|
| 15 |
+
| File | Change |
|
| 16 |
+
|---|---|
|
| 17 |
+
| `formscout/config.py` | Replace `YOLO_POSE_MODELS` with `POSE_MODELS` dict + `DEFAULT_POSE_MODEL` |
|
| 18 |
+
| `formscout/agents/pose2d.py` | Add `_run_yolo`, `_run_mediapipe`, `_run_sapiens2`; update `run()` signature |
|
| 19 |
+
| `formscout/pipeline.py` | Change `pose_model_path` param to `model_key` |
|
| 20 |
+
| `app.py` | Add `pose_model_dropdown`, fix `_map_inputs` + `process_video` |
|
| 21 |
+
| `requirements.txt` | Add `onnxruntime>=1.18` |
|
| 22 |
+
| `tests/test_pose2d.py` | Add mocked tests for each backend |
|
| 23 |
+
|
| 24 |
+
---
|
| 25 |
+
|
| 26 |
+
## Task 1: Add unified `POSE_MODELS` registry to `config.py`
|
| 27 |
+
|
| 28 |
+
**Files:**
|
| 29 |
+
- Modify: `formscout/config.py`
|
| 30 |
+
|
| 31 |
+
- [ ] **Step 1: Open `formscout/config.py` and replace the `YOLO_POSE_MODELS` block**
|
| 32 |
+
|
| 33 |
+
Replace lines 12β20 (the `YOLO_POSE_MODELS` dict and `YOLO_POSE_MODEL` / `YOLO_POSE_MODEL_HQ` lines) with:
|
| 34 |
+
|
| 35 |
+
```python
|
| 36 |
+
_YOLO_DIR = ROOT / "checkpoints" / "yolo26"
|
| 37 |
+
|
| 38 |
+
POSE_MODELS: dict[str, dict] = {
|
| 39 |
+
# ββ MediaPipe (Qualcomm HF, ONNX Runtime) ββββββββββββββββββββββββββββββ
|
| 40 |
+
"MediaPipe-Pose β¬ ~16 MB, CPU-friendly": {
|
| 41 |
+
"backend": "mediapipe",
|
| 42 |
+
"hf_id": "qualcomm/MediaPipe-Pose-Estimation",
|
| 43 |
+
"params_m": 4.2,
|
| 44 |
+
},
|
| 45 |
+
# ββ YOLO26 (local checkpoints) βββββββββββββββββββββββββββββββββββββββββ
|
| 46 |
+
"YOLO26n β nano (0.7M, fastest)": {
|
| 47 |
+
"backend": "yolo",
|
| 48 |
+
"path": str(_YOLO_DIR / "yolo26n-pose.pt"),
|
| 49 |
+
"params_m": 0.7,
|
| 50 |
+
},
|
| 51 |
+
"YOLO26s β small (3.5M)": {
|
| 52 |
+
"backend": "yolo",
|
| 53 |
+
"path": str(_YOLO_DIR / "yolo26s-pose.pt"),
|
| 54 |
+
"params_m": 3.5,
|
| 55 |
+
},
|
| 56 |
+
"YOLO26m β medium (9M)": {
|
| 57 |
+
"backend": "yolo",
|
| 58 |
+
"path": str(_YOLO_DIR / "yolo26m-pose.pt"),
|
| 59 |
+
"params_m": 9.0,
|
| 60 |
+
},
|
| 61 |
+
"YOLO26l β large (25.9M)": {
|
| 62 |
+
"backend": "yolo",
|
| 63 |
+
"path": str(_YOLO_DIR / "yolo26l-pose.pt"),
|
| 64 |
+
"params_m": 25.9,
|
| 65 |
+
},
|
| 66 |
+
"YOLO26x β extra-large (57.6M)": {
|
| 67 |
+
"backend": "yolo",
|
| 68 |
+
"path": str(_YOLO_DIR / "yolo26x-pose.pt"),
|
| 69 |
+
"params_m": 57.6,
|
| 70 |
+
},
|
| 71 |
+
# ββ Sapiens2 (HF download, transformers) βββββββββββββββββββββββββββββββ
|
| 72 |
+
"Sapiens2-0.4B β¬ ~1.6 GB": {
|
| 73 |
+
"backend": "sapiens2",
|
| 74 |
+
"hf_id": "facebook/sapiens2-pose-0.4b",
|
| 75 |
+
"params_m": 400,
|
| 76 |
+
},
|
| 77 |
+
"Sapiens2-0.8B β¬ ~3.2 GB": {
|
| 78 |
+
"backend": "sapiens2",
|
| 79 |
+
"hf_id": "facebook/sapiens2-pose-0.8b",
|
| 80 |
+
"params_m": 800,
|
| 81 |
+
},
|
| 82 |
+
"Sapiens2-1B β¬ ~4 GB": {
|
| 83 |
+
"backend": "sapiens2",
|
| 84 |
+
"hf_id": "facebook/sapiens2-pose-1b",
|
| 85 |
+
"params_m": 1000,
|
| 86 |
+
},
|
| 87 |
+
"Sapiens2-5B β¬ ~20 GB, large GPU": {
|
| 88 |
+
"backend": "sapiens2",
|
| 89 |
+
"hf_id": "facebook/sapiens2-pose-5b",
|
| 90 |
+
"params_m": 5000,
|
| 91 |
+
},
|
| 92 |
+
}
|
| 93 |
+
|
| 94 |
+
DEFAULT_POSE_MODEL = "YOLO26n β nano (0.7M, fastest)"
|
| 95 |
+
|
| 96 |
+
# Backward-compat aliases β kept for any direct references outside the agent
|
| 97 |
+
YOLO_POSE_MODEL = str(_YOLO_DIR / "yolo26l-pose.pt")
|
| 98 |
+
YOLO_POSE_MODEL_HQ = str(_YOLO_DIR / "yolo26x-pose.pt")
|
| 99 |
+
```
|
| 100 |
+
|
| 101 |
+
- [ ] **Step 2: Verify import is clean**
|
| 102 |
+
|
| 103 |
+
```bash
|
| 104 |
+
python3 -c "from formscout import config; print(list(config.POSE_MODELS.keys()))"
|
| 105 |
+
```
|
| 106 |
+
|
| 107 |
+
Expected: list of 10 model labels, starting with `MediaPipe-Pose...`
|
| 108 |
+
|
| 109 |
+
- [ ] **Step 3: Commit**
|
| 110 |
+
|
| 111 |
+
```bash
|
| 112 |
+
git add formscout/config.py
|
| 113 |
+
git commit -m "feat: unified POSE_MODELS registry with MediaPipe, YOLO26 n-x, Sapiens2 0.4-5B"
|
| 114 |
+
git push
|
| 115 |
+
```
|
| 116 |
+
|
| 117 |
+
---
|
| 118 |
+
|
| 119 |
+
## Task 2: Refactor `Pose2DAgent` β YOLO sub-runner + new `run()` signature
|
| 120 |
+
|
| 121 |
+
**Files:**
|
| 122 |
+
- Modify: `formscout/agents/pose2d.py`
|
| 123 |
+
- Modify: `tests/test_pose2d.py`
|
| 124 |
+
|
| 125 |
+
- [ ] **Step 1: Write failing test for the new `model_key` signature**
|
| 126 |
+
|
| 127 |
+
Add to `tests/test_pose2d.py`:
|
| 128 |
+
|
| 129 |
+
```python
|
| 130 |
+
def test_run_accepts_model_key(pose2d_agent):
|
| 131 |
+
"""run() must accept model_key kwarg, not model_path."""
|
| 132 |
+
import inspect
|
| 133 |
+
sig = inspect.signature(pose2d_agent.run)
|
| 134 |
+
assert "model_key" in sig.parameters
|
| 135 |
+
assert "model_path" not in sig.parameters
|
| 136 |
+
```
|
| 137 |
+
|
| 138 |
+
- [ ] **Step 2: Run to confirm it fails**
|
| 139 |
+
|
| 140 |
+
```bash
|
| 141 |
+
pytest tests/test_pose2d.py::TestPose2DAgent::test_run_accepts_model_key -v
|
| 142 |
+
```
|
| 143 |
+
|
| 144 |
+
Expected: FAIL β `model_path` still present in signature.
|
| 145 |
+
|
| 146 |
+
- [ ] **Step 3: Rewrite `formscout/agents/pose2d.py`**
|
| 147 |
+
|
| 148 |
+
Replace the entire file with:
|
| 149 |
+
|
| 150 |
+
```python
|
| 151 |
+
"""
|
| 152 |
+
Pose2DAgent β 2D per-frame keypoint extraction.
|
| 153 |
+
|
| 154 |
+
Backends: yolo (local ONNX), mediapipe (Qualcomm HF/ONNX Runtime),
|
| 155 |
+
sapiens2 (Meta HF/transformers).
|
| 156 |
+
All backends output COCO-17 keypoints: dict[int, {x, y, conf}] per frame.
|
| 157 |
+
|
| 158 |
+
Input: IngestResult
|
| 159 |
+
Output: Pose2DResult(keypoints per frame, fps, confidence)
|
| 160 |
+
Failure: Pose2DResult(confidence=0.0, notes=<reason>) β never raises.
|
| 161 |
+
"""
|
| 162 |
+
from __future__ import annotations
|
| 163 |
+
|
| 164 |
+
import logging
|
| 165 |
+
import numpy as np
|
| 166 |
+
|
| 167 |
+
from formscout import config
|
| 168 |
+
from formscout.types import IngestResult, Pose2DResult
|
| 169 |
+
|
| 170 |
+
logger = logging.getLogger(__name__)
|
| 171 |
+
|
| 172 |
+
COCO_KEYPOINTS = [
|
| 173 |
+
"nose", "left_eye", "right_eye", "left_ear", "right_ear",
|
| 174 |
+
"left_shoulder", "right_shoulder", "left_elbow", "right_elbow",
|
| 175 |
+
"left_wrist", "right_wrist", "left_hip", "right_hip",
|
| 176 |
+
"left_knee", "right_knee", "left_ankle", "right_ankle",
|
| 177 |
+
]
|
| 178 |
+
|
| 179 |
+
# BlazePose-33 β COCO-17 index mapping
|
| 180 |
+
_BLAZEPOSE_TO_COCO: dict[int, int] = {
|
| 181 |
+
0: 0, # nose
|
| 182 |
+
1: 2, # left_eye (inner β left_eye)
|
| 183 |
+
2: 1, # right_eye (inner β right_eye) β swapped: BlazePose 1=left_eye_inner
|
| 184 |
+
3: 3, # left_ear
|
| 185 |
+
4: 4, # right_ear
|
| 186 |
+
5: 5, # left_shoulder β COCO left_shoulder... wait
|
| 187 |
+
# Correct BlazePose-33 COCO mapping (canonical):
|
| 188 |
+
# BlazePose idx : COCO idx
|
| 189 |
+
# 0 nose β COCO 0
|
| 190 |
+
# 2 left_eye β COCO 1
|
| 191 |
+
# 5 right_eye β COCO 2
|
| 192 |
+
# 7 left_ear β COCO 3
|
| 193 |
+
# 8 right_ear β COCO 4
|
| 194 |
+
# 11 left_shoulder β COCO 5
|
| 195 |
+
# 12 right_shoulder β COCO 6
|
| 196 |
+
# 13 left_elbow β COCO 7
|
| 197 |
+
# 14 right_elbow β COCO 8
|
| 198 |
+
# 15 left_wrist β COCO 9
|
| 199 |
+
# 16 right_wrist β COCO 10
|
| 200 |
+
# 23 left_hip β COCO 11
|
| 201 |
+
# 24 right_hip β COCO 12
|
| 202 |
+
# 25 left_knee β COCO 13
|
| 203 |
+
# 26 right_knee β COCO 14
|
| 204 |
+
# 27 left_ankle β COCO 15
|
| 205 |
+
# 28 right_ankle β COCO 16
|
| 206 |
+
}
|
| 207 |
+
|
| 208 |
+
# BlazePose source index β COCO target index (correct mapping, no duplicates)
|
| 209 |
+
_BP_SRC = [0, 2, 5, 7, 8, 11, 12, 13, 14, 15, 16, 23, 24, 25, 26, 27, 28]
|
| 210 |
+
_BP_DST = list(range(17)) # COCO 0..16
|
| 211 |
+
|
| 212 |
+
_model_cache: dict[str, object] = {}
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
# ββ YOLO backend βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 216 |
+
|
| 217 |
+
def _get_yolo(path: str) -> object:
|
| 218 |
+
if path not in _model_cache:
|
| 219 |
+
from ultralytics import YOLO
|
| 220 |
+
_model_cache[path] = YOLO(path)
|
| 221 |
+
return _model_cache[path]
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
def _run_yolo(frames: list, path: str) -> list[dict]:
|
| 225 |
+
model = _get_yolo(path)
|
| 226 |
+
out = []
|
| 227 |
+
for frame in frames:
|
| 228 |
+
try:
|
| 229 |
+
results = model(frame, verbose=False)
|
| 230 |
+
kps: dict[int, dict] = {}
|
| 231 |
+
if results and results[0].keypoints is not None:
|
| 232 |
+
kp = results[0].keypoints
|
| 233 |
+
if kp.xy is not None and len(kp.xy) > 0:
|
| 234 |
+
xy = kp.xy[0].cpu().numpy()
|
| 235 |
+
conf = kp.conf[0].cpu().numpy()
|
| 236 |
+
for j in range(min(len(xy), 17)):
|
| 237 |
+
kps[j] = {"x": float(xy[j, 0]), "y": float(xy[j, 1]), "conf": float(conf[j])}
|
| 238 |
+
out.append(kps)
|
| 239 |
+
except Exception:
|
| 240 |
+
out.append({})
|
| 241 |
+
return out
|
| 242 |
+
|
| 243 |
+
|
| 244 |
+
# ββ MediaPipe backend ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 245 |
+
|
| 246 |
+
def _get_mediapipe_sessions(hf_id: str):
|
| 247 |
+
"""Return (detector_session, landmark_session) cached by hf_id."""
|
| 248 |
+
cache_key = f"mp:{hf_id}"
|
| 249 |
+
if cache_key not in _model_cache:
|
| 250 |
+
from huggingface_hub import snapshot_download
|
| 251 |
+
import onnxruntime as ort
|
| 252 |
+
from pathlib import Path
|
| 253 |
+
|
| 254 |
+
snap = Path(snapshot_download(hf_id))
|
| 255 |
+
onnx_files = sorted(snap.glob("**/*.onnx"), key=lambda p: p.stat().st_size)
|
| 256 |
+
if len(onnx_files) < 2:
|
| 257 |
+
raise RuntimeError(f"Expected 2 ONNX files in {snap}, found {len(onnx_files)}")
|
| 258 |
+
# Smaller file = pose detector; larger = pose landmark detector
|
| 259 |
+
det_sess = ort.InferenceSession(str(onnx_files[0]))
|
| 260 |
+
lmk_sess = ort.InferenceSession(str(onnx_files[-1]))
|
| 261 |
+
_model_cache[cache_key] = (det_sess, lmk_sess)
|
| 262 |
+
return _model_cache[cache_key]
|
| 263 |
+
|
| 264 |
+
|
| 265 |
+
def _preprocess_mediapipe(frame: np.ndarray, size: int = 256) -> np.ndarray:
|
| 266 |
+
"""Resize to sizeΓsize, normalize to [0,1], add batch dim β (1,3,H,W)."""
|
| 267 |
+
import cv2
|
| 268 |
+
img = cv2.resize(frame, (size, size)).astype(np.float32) / 255.0
|
| 269 |
+
return img.transpose(2, 0, 1)[None] # (1, 3, 256, 256)
|
| 270 |
+
|
| 271 |
+
|
| 272 |
+
def _run_mediapipe(frames: list, hf_id: str) -> list[dict]:
|
| 273 |
+
try:
|
| 274 |
+
det_sess, lmk_sess = _get_mediapipe_sessions(hf_id)
|
| 275 |
+
except Exception as e:
|
| 276 |
+
logger.warning("mediapipe load failed: %s", e)
|
| 277 |
+
return [{} for _ in frames]
|
| 278 |
+
|
| 279 |
+
import cv2
|
| 280 |
+
h_orig, w_orig = frames[0].shape[:2] if frames else (480, 640)
|
| 281 |
+
out = []
|
| 282 |
+
|
| 283 |
+
for frame in frames:
|
| 284 |
+
try:
|
| 285 |
+
h, w = frame.shape[:2]
|
| 286 |
+
inp = _preprocess_mediapipe(frame)
|
| 287 |
+
|
| 288 |
+
# Run landmark detector directly on full frame (single-person FMS use-case)
|
| 289 |
+
lmk_input_name = lmk_sess.get_inputs()[0].name
|
| 290 |
+
lmk_out = lmk_sess.run(None, {lmk_input_name: inp})
|
| 291 |
+
|
| 292 |
+
# lmk_out[0] shape: (1, 33, 3) β [x, y, visibility] normalized 0..1
|
| 293 |
+
landmarks = lmk_out[0][0] # (33, 3)
|
| 294 |
+
|
| 295 |
+
kps: dict[int, dict] = {}
|
| 296 |
+
for coco_idx, bp_idx in zip(_BP_DST, _BP_SRC):
|
| 297 |
+
if bp_idx < len(landmarks):
|
| 298 |
+
lm = landmarks[bp_idx]
|
| 299 |
+
kps[coco_idx] = {
|
| 300 |
+
"x": float(lm[0] * w),
|
| 301 |
+
"y": float(lm[1] * h),
|
| 302 |
+
"conf": float(lm[2]), # visibility score
|
| 303 |
+
}
|
| 304 |
+
out.append(kps)
|
| 305 |
+
except Exception:
|
| 306 |
+
out.append({})
|
| 307 |
+
|
| 308 |
+
return out
|
| 309 |
+
|
| 310 |
+
|
| 311 |
+
# ββ Sapiens2 backend βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 312 |
+
|
| 313 |
+
# COCO-17 keypoint names in order (used to map Sapiens2 named output β COCO index)
|
| 314 |
+
_COCO_NAMES = [
|
| 315 |
+
"nose", "left_eye", "right_eye", "left_ear", "right_ear",
|
| 316 |
+
"left_shoulder", "right_shoulder", "left_elbow", "right_elbow",
|
| 317 |
+
"left_wrist", "right_wrist", "left_hip", "right_hip",
|
| 318 |
+
"left_knee", "right_knee", "left_ankle", "right_ankle",
|
| 319 |
+
]
|
| 320 |
+
|
| 321 |
+
|
| 322 |
+
def _get_sapiens2(hf_id: str) -> object:
|
| 323 |
+
if hf_id not in _model_cache:
|
| 324 |
+
from transformers import pipeline as hf_pipeline
|
| 325 |
+
_model_cache[hf_id] = hf_pipeline("pose-estimation", model=hf_id)
|
| 326 |
+
return _model_cache[hf_id]
|
| 327 |
+
|
| 328 |
+
|
| 329 |
+
def _run_sapiens2(frames: list, hf_id: str) -> list[dict]:
|
| 330 |
+
try:
|
| 331 |
+
pipe = _get_sapiens2(hf_id)
|
| 332 |
+
except Exception as e:
|
| 333 |
+
logger.warning("sapiens2 load failed: %s", e)
|
| 334 |
+
return [{} for _ in frames]
|
| 335 |
+
|
| 336 |
+
from PIL import Image
|
| 337 |
+
out = []
|
| 338 |
+
|
| 339 |
+
for frame in frames:
|
| 340 |
+
try:
|
| 341 |
+
pil_img = Image.fromarray(frame)
|
| 342 |
+
result = pipe(pil_img)
|
| 343 |
+
|
| 344 |
+
# result is a list of person dicts; take the first (highest confidence)
|
| 345 |
+
if not result:
|
| 346 |
+
out.append({})
|
| 347 |
+
continue
|
| 348 |
+
|
| 349 |
+
person = result[0]
|
| 350 |
+
keypoints = person.get("keypoints", [])
|
| 351 |
+
scores = person.get("keypoint_scores", [])
|
| 352 |
+
|
| 353 |
+
# Build nameβ(x,y,score) lookup from pipeline output
|
| 354 |
+
kp_lookup: dict[str, tuple] = {}
|
| 355 |
+
for i, kp in enumerate(keypoints):
|
| 356 |
+
name = kp.get("label", "") if isinstance(kp, dict) else ""
|
| 357 |
+
x = kp.get("x", 0.0) if isinstance(kp, dict) else float(kp[0])
|
| 358 |
+
y = kp.get("y", 0.0) if isinstance(kp, dict) else float(kp[1])
|
| 359 |
+
score = scores[i] if i < len(scores) else 0.0
|
| 360 |
+
if name:
|
| 361 |
+
kp_lookup[name] = (x, y, float(score))
|
| 362 |
+
|
| 363 |
+
kps: dict[int, dict] = {}
|
| 364 |
+
for coco_idx, name in enumerate(_COCO_NAMES):
|
| 365 |
+
if name in kp_lookup:
|
| 366 |
+
x, y, s = kp_lookup[name]
|
| 367 |
+
kps[coco_idx] = {"x": x, "y": y, "conf": s}
|
| 368 |
+
|
| 369 |
+
out.append(kps)
|
| 370 |
+
except Exception:
|
| 371 |
+
out.append({})
|
| 372 |
+
|
| 373 |
+
return out
|
| 374 |
+
|
| 375 |
+
|
| 376 |
+
# ββ Agent ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 377 |
+
|
| 378 |
+
class Pose2DAgent:
|
| 379 |
+
"""Extracts COCO-17 keypoints per frame; dispatches to YOLO, MediaPipe, or Sapiens2."""
|
| 380 |
+
|
| 381 |
+
def run(self, ingest: IngestResult, model_key: str | None = None) -> Pose2DResult:
|
| 382 |
+
if not ingest.frames:
|
| 383 |
+
return Pose2DResult(keypoints=[], fps=ingest.fps, confidence=0.0, notes="no frames in ingest")
|
| 384 |
+
|
| 385 |
+
key = model_key or config.DEFAULT_POSE_MODEL
|
| 386 |
+
spec = config.POSE_MODELS.get(key)
|
| 387 |
+
if spec is None:
|
| 388 |
+
logger.warning("Unknown model_key %r β falling back to %s", key, config.DEFAULT_POSE_MODEL)
|
| 389 |
+
spec = config.POSE_MODELS[config.DEFAULT_POSE_MODEL]
|
| 390 |
+
|
| 391 |
+
backend = spec["backend"]
|
| 392 |
+
try:
|
| 393 |
+
if backend == "yolo":
|
| 394 |
+
kps_per_frame = _run_yolo(ingest.frames, spec["path"])
|
| 395 |
+
elif backend == "mediapipe":
|
| 396 |
+
kps_per_frame = _run_mediapipe(ingest.frames, spec["hf_id"])
|
| 397 |
+
elif backend == "sapiens2":
|
| 398 |
+
kps_per_frame = _run_sapiens2(ingest.frames, spec["hf_id"])
|
| 399 |
+
else:
|
| 400 |
+
return Pose2DResult(
|
| 401 |
+
keypoints=[{} for _ in ingest.frames],
|
| 402 |
+
fps=ingest.fps, confidence=0.0,
|
| 403 |
+
notes=f"unknown backend: {backend}",
|
| 404 |
+
)
|
| 405 |
+
except Exception as e:
|
| 406 |
+
return Pose2DResult(
|
| 407 |
+
keypoints=[{} for _ in ingest.frames],
|
| 408 |
+
fps=ingest.fps, confidence=0.0,
|
| 409 |
+
notes=str(e),
|
| 410 |
+
)
|
| 411 |
+
|
| 412 |
+
n_detected = sum(1 for f in kps_per_frame if f)
|
| 413 |
+
total_conf = sum(
|
| 414 |
+
sum(kp["conf"] for kp in f.values()) / len(f)
|
| 415 |
+
for f in kps_per_frame if f
|
| 416 |
+
)
|
| 417 |
+
overall_conf = (total_conf / n_detected) if n_detected > 0 else 0.0
|
| 418 |
+
notes = "" if n_detected > 0 else "no person detected in any frame"
|
| 419 |
+
|
| 420 |
+
return Pose2DResult(
|
| 421 |
+
keypoints=kps_per_frame,
|
| 422 |
+
fps=ingest.fps,
|
| 423 |
+
confidence=overall_conf,
|
| 424 |
+
notes=notes,
|
| 425 |
+
)
|
| 426 |
+
```
|
| 427 |
+
|
| 428 |
+
- [ ] **Step 4: Run the new signature test**
|
| 429 |
+
|
| 430 |
+
```bash
|
| 431 |
+
pytest tests/test_pose2d.py::TestPose2DAgent::test_run_accepts_model_key -v
|
| 432 |
+
```
|
| 433 |
+
|
| 434 |
+
Expected: PASS
|
| 435 |
+
|
| 436 |
+
- [ ] **Step 5: Run full existing pose2d test suite**
|
| 437 |
+
|
| 438 |
+
```bash
|
| 439 |
+
pytest tests/test_pose2d.py -v
|
| 440 |
+
```
|
| 441 |
+
|
| 442 |
+
Expected: all existing tests pass (they will skip if YOLO model unavailable in env β that's OK).
|
| 443 |
+
|
| 444 |
+
- [ ] **Step 6: Commit and push**
|
| 445 |
+
|
| 446 |
+
```bash
|
| 447 |
+
git add formscout/agents/pose2d.py tests/test_pose2d.py
|
| 448 |
+
git commit -m "feat: Pose2DAgent β three backends (yolo/mediapipe/sapiens2), model_key dispatch"
|
| 449 |
+
git push
|
| 450 |
+
```
|
| 451 |
+
|
| 452 |
+
---
|
| 453 |
+
|
| 454 |
+
## Task 3: Add `onnxruntime` to requirements
|
| 455 |
+
|
| 456 |
+
**Files:**
|
| 457 |
+
- Modify: `requirements.txt`
|
| 458 |
+
|
| 459 |
+
- [ ] **Step 1: Add onnxruntime**
|
| 460 |
+
|
| 461 |
+
Open `requirements.txt` and add after the existing `transformers` line:
|
| 462 |
+
|
| 463 |
+
```
|
| 464 |
+
onnxruntime>=1.18
|
| 465 |
+
```
|
| 466 |
+
|
| 467 |
+
- [ ] **Step 2: Verify it installs**
|
| 468 |
+
|
| 469 |
+
```bash
|
| 470 |
+
pip install onnxruntime --quiet && python3 -c "import onnxruntime; print(onnxruntime.__version__)"
|
| 471 |
+
```
|
| 472 |
+
|
| 473 |
+
Expected: version string printed, no errors.
|
| 474 |
+
|
| 475 |
+
- [ ] **Step 3: Commit and push**
|
| 476 |
+
|
| 477 |
+
```bash
|
| 478 |
+
git add requirements.txt
|
| 479 |
+
git commit -m "chore: add onnxruntime for MediaPipe ONNX backend"
|
| 480 |
+
git push
|
| 481 |
+
```
|
| 482 |
+
|
| 483 |
+
---
|
| 484 |
+
|
| 485 |
+
## Task 4: Update `Director.run()` β `pose_model_path` β `model_key`
|
| 486 |
+
|
| 487 |
+
**Files:**
|
| 488 |
+
- Modify: `formscout/pipeline.py`
|
| 489 |
+
|
| 490 |
+
- [ ] **Step 1: Update the signature and the `pose2d` call**
|
| 491 |
+
|
| 492 |
+
In `formscout/pipeline.py`, change `Director.run()`:
|
| 493 |
+
|
| 494 |
+
```python
|
| 495 |
+
def run(self, video_path: str, test_name: str = "deep_squat", side: str = "na", model_key: str | None = None) -> PipelineState:
|
| 496 |
+
"""
|
| 497 |
+
Run the full pipeline on a single video.
|
| 498 |
+
test_name/side serve as manual override when provided (skips classifier).
|
| 499 |
+
model_key selects the pose backend (see config.POSE_MODELS).
|
| 500 |
+
"""
|
| 501 |
+
state = PipelineState(video_path=video_path)
|
| 502 |
+
|
| 503 |
+
# βββ Ingest βββ
|
| 504 |
+
state.ingest = self._ingest.run(video_path)
|
| 505 |
+
if state.ingest.confidence < config.MIN_CONFIDENCE:
|
| 506 |
+
state.errors.append("ingest: low confidence β video may be corrupt")
|
| 507 |
+
return state
|
| 508 |
+
|
| 509 |
+
# βββ Pose 2D βββ
|
| 510 |
+
state.pose2d = self._pose2d.run(state.ingest, model_key=model_key)
|
| 511 |
+
# ... rest of method unchanged
|
| 512 |
+
```
|
| 513 |
+
|
| 514 |
+
(Only the signature line and the `self._pose2d.run(...)` call change β everything else stays the same.)
|
| 515 |
+
|
| 516 |
+
- [ ] **Step 2: Verify import is clean**
|
| 517 |
+
|
| 518 |
+
```bash
|
| 519 |
+
python3 -c "from formscout.pipeline import Director; d = Director(); print('ok')"
|
| 520 |
+
```
|
| 521 |
+
|
| 522 |
+
Expected: `ok` (models load lazily so no crash here).
|
| 523 |
+
|
| 524 |
+
- [ ] **Step 3: Commit and push**
|
| 525 |
+
|
| 526 |
+
```bash
|
| 527 |
+
git add formscout/pipeline.py
|
| 528 |
+
git commit -m "feat: Director.run() accepts model_key, threads to Pose2DAgent"
|
| 529 |
+
git push
|
| 530 |
+
```
|
| 531 |
+
|
| 532 |
+
---
|
| 533 |
+
|
| 534 |
+
## Task 5: Wire the UI β pose model dropdown in `app.py`
|
| 535 |
+
|
| 536 |
+
**Files:**
|
| 537 |
+
- Modify: `app.py`
|
| 538 |
+
|
| 539 |
+
- [ ] **Step 1: Update `process_video` to use `model_key` and the unified registry**
|
| 540 |
+
|
| 541 |
+
Replace the existing `process_video` function signature and the old `YOLO_POSE_MODELS.get()` lookup:
|
| 542 |
+
|
| 543 |
+
```python
|
| 544 |
+
def process_video(video_path: str, test_name: str, side: str, model_key: str):
|
| 545 |
+
"""Process an uploaded video through the FormScout pipeline."""
|
| 546 |
+
if not video_path:
|
| 547 |
+
return (
|
| 548 |
+
_render_empty_state(),
|
| 549 |
+
"Upload a video to begin analysis.",
|
| 550 |
+
"",
|
| 551 |
+
"",
|
| 552 |
+
)
|
| 553 |
+
|
| 554 |
+
director = Director()
|
| 555 |
+
state = director.run(video_path, test_name=test_name, side=side, model_key=model_key)
|
| 556 |
+
```
|
| 557 |
+
|
| 558 |
+
(Remove the `pose_model_path = config.YOLO_POSE_MODELS.get(...)` line entirely.)
|
| 559 |
+
|
| 560 |
+
- [ ] **Step 2: Add the `pose_model_dropdown` in `build_app()`**
|
| 561 |
+
|
| 562 |
+
Inside `build_app()`, after the `side_dropdown` block (around line 265) and before `submit_btn`, add:
|
| 563 |
+
|
| 564 |
+
```python
|
| 565 |
+
pose_model_dropdown = gr.Dropdown(
|
| 566 |
+
choices=list(config.POSE_MODELS.keys()),
|
| 567 |
+
value=config.DEFAULT_POSE_MODEL,
|
| 568 |
+
label="Pose Model",
|
| 569 |
+
)
|
| 570 |
+
```
|
| 571 |
+
|
| 572 |
+
- [ ] **Step 3: Update `_map_inputs` to pass the model key**
|
| 573 |
+
|
| 574 |
+
Replace the existing `_map_inputs` closure:
|
| 575 |
+
|
| 576 |
+
```python
|
| 577 |
+
def _map_inputs(video, test_display_name, side_display, pose_model_key):
|
| 578 |
+
"""Map UI display values to internal values."""
|
| 579 |
+
test_map = {name: val for name, val in FMS_TESTS}
|
| 580 |
+
test_name = test_map.get(test_display_name, "deep_squat")
|
| 581 |
+
side = {"N/A": "na", "Left": "left", "Right": "right"}.get(side_display, "na")
|
| 582 |
+
return process_video(video, test_name, side, pose_model_key)
|
| 583 |
+
```
|
| 584 |
+
|
| 585 |
+
- [ ] **Step 4: Update `submit_btn.click` to include `pose_model_dropdown`**
|
| 586 |
+
|
| 587 |
+
Replace the existing `.click(...)` call:
|
| 588 |
+
|
| 589 |
+
```python
|
| 590 |
+
submit_btn.click(
|
| 591 |
+
fn=_map_inputs,
|
| 592 |
+
inputs=[video_input, test_dropdown, side_dropdown, pose_model_dropdown],
|
| 593 |
+
outputs=[score_html, pipeline_md, score_details, alerts_md],
|
| 594 |
+
)
|
| 595 |
+
```
|
| 596 |
+
|
| 597 |
+
- [ ] **Step 5: Smoke-test the app starts**
|
| 598 |
+
|
| 599 |
+
```bash
|
| 600 |
+
python3 -c "from app import build_app; app = build_app(); print('app built ok')"
|
| 601 |
+
```
|
| 602 |
+
|
| 603 |
+
Expected: `app built ok` β no import or config errors.
|
| 604 |
+
|
| 605 |
+
- [ ] **Step 6: Commit and push**
|
| 606 |
+
|
| 607 |
+
```bash
|
| 608 |
+
git add app.py
|
| 609 |
+
git commit -m "feat: pose model dropdown in UI, wired through process_video β Director"
|
| 610 |
+
git push
|
| 611 |
+
```
|
| 612 |
+
|
| 613 |
+
---
|
| 614 |
+
|
| 615 |
+
## Task 6: Add mocked backend tests
|
| 616 |
+
|
| 617 |
+
**Files:**
|
| 618 |
+
- Modify: `tests/test_pose2d.py`
|
| 619 |
+
|
| 620 |
+
- [ ] **Step 1: Add mocked YOLO test**
|
| 621 |
+
|
| 622 |
+
Append to `tests/test_pose2d.py`:
|
| 623 |
+
|
| 624 |
+
```python
|
| 625 |
+
import unittest.mock as mock
|
| 626 |
+
import numpy as np
|
| 627 |
+
from formscout.types import IngestResult, Pose2DResult
|
| 628 |
+
|
| 629 |
+
|
| 630 |
+
def _blank_ingest_3():
|
| 631 |
+
frames = [np.zeros((480, 640, 3), dtype=np.uint8) for _ in range(3)]
|
| 632 |
+
return IngestResult(frames=frames, fps=30.0, duration=0.1, n_people=1, width=640, height=480)
|
| 633 |
+
|
| 634 |
+
|
| 635 |
+
class TestPose2DBackendsMocked:
|
| 636 |
+
"""Backend dispatch tests β no real model downloads."""
|
| 637 |
+
|
| 638 |
+
def test_yolo_backend_dispatches(self):
|
| 639 |
+
from formscout.agents.pose2d import Pose2DAgent, _run_yolo
|
| 640 |
+
fake_kps = [{0: {"x": 10.0, "y": 20.0, "conf": 0.9}} for _ in range(3)]
|
| 641 |
+
with mock.patch("formscout.agents.pose2d._run_yolo", return_value=fake_kps) as m:
|
| 642 |
+
agent = Pose2DAgent()
|
| 643 |
+
result = agent.run(_blank_ingest_3(), model_key="YOLO26n β nano (0.7M, fastest)")
|
| 644 |
+
m.assert_called_once()
|
| 645 |
+
assert isinstance(result, Pose2DResult)
|
| 646 |
+
assert len(result.keypoints) == 3
|
| 647 |
+
assert result.confidence > 0.0
|
| 648 |
+
|
| 649 |
+
def test_mediapipe_backend_dispatches(self):
|
| 650 |
+
from formscout.agents.pose2d import Pose2DAgent
|
| 651 |
+
fake_kps = [{i: {"x": float(i), "y": float(i), "conf": 0.8} for i in range(17)} for _ in range(3)]
|
| 652 |
+
with mock.patch("formscout.agents.pose2d._run_mediapipe", return_value=fake_kps) as m:
|
| 653 |
+
agent = Pose2DAgent()
|
| 654 |
+
result = agent.run(_blank_ingest_3(), model_key="MediaPipe-Pose β¬ ~16 MB, CPU-friendly")
|
| 655 |
+
m.assert_called_once()
|
| 656 |
+
assert isinstance(result, Pose2DResult)
|
| 657 |
+
assert len(result.keypoints) == 3
|
| 658 |
+
assert all(len(f) == 17 for f in result.keypoints)
|
| 659 |
+
|
| 660 |
+
def test_sapiens2_backend_dispatches(self):
|
| 661 |
+
from formscout.agents.pose2d import Pose2DAgent
|
| 662 |
+
fake_kps = [{i: {"x": float(i), "y": float(i), "conf": 0.85} for i in range(17)} for _ in range(3)]
|
| 663 |
+
with mock.patch("formscout.agents.pose2d._run_sapiens2", return_value=fake_kps) as m:
|
| 664 |
+
agent = Pose2DAgent()
|
| 665 |
+
result = agent.run(_blank_ingest_3(), model_key="Sapiens2-0.4B β¬ ~1.6 GB")
|
| 666 |
+
m.assert_called_once()
|
| 667 |
+
assert isinstance(result, Pose2DResult)
|
| 668 |
+
assert len(result.keypoints) == 3
|
| 669 |
+
|
| 670 |
+
def test_unknown_model_key_falls_back(self):
|
| 671 |
+
from formscout.agents.pose2d import Pose2DAgent
|
| 672 |
+
fake_kps = [{0: {"x": 1.0, "y": 2.0, "conf": 0.7}} for _ in range(3)]
|
| 673 |
+
with mock.patch("formscout.agents.pose2d._run_yolo", return_value=fake_kps):
|
| 674 |
+
agent = Pose2DAgent()
|
| 675 |
+
result = agent.run(_blank_ingest_3(), model_key="nonexistent-model-xyz")
|
| 676 |
+
assert isinstance(result, Pose2DResult) # graceful fallback, no crash
|
| 677 |
+
|
| 678 |
+
def test_confidence_zero_on_empty_keypoints(self):
|
| 679 |
+
from formscout.agents.pose2d import Pose2DAgent
|
| 680 |
+
with mock.patch("formscout.agents.pose2d._run_yolo", return_value=[{}, {}, {}]):
|
| 681 |
+
agent = Pose2DAgent()
|
| 682 |
+
result = agent.run(_blank_ingest_3(), model_key="YOLO26n β nano (0.7M, fastest)")
|
| 683 |
+
assert result.confidence == 0.0
|
| 684 |
+
assert "no person" in result.notes.lower()
|
| 685 |
+
```
|
| 686 |
+
|
| 687 |
+
- [ ] **Step 2: Run the new tests**
|
| 688 |
+
|
| 689 |
+
```bash
|
| 690 |
+
pytest tests/test_pose2d.py::TestPose2DBackendsMocked -v
|
| 691 |
+
```
|
| 692 |
+
|
| 693 |
+
Expected: all 5 tests PASS.
|
| 694 |
+
|
| 695 |
+
- [ ] **Step 3: Run the full test suite to check for regressions**
|
| 696 |
+
|
| 697 |
+
```bash
|
| 698 |
+
pytest tests/ -v --tb=short 2>&1 | tail -30
|
| 699 |
+
```
|
| 700 |
+
|
| 701 |
+
Expected: same pass/fail ratio as before (45/46 known passing). The one known failure (`test_unimplemented_test_returns_low_confidence`) is pre-existing β ignore it.
|
| 702 |
+
|
| 703 |
+
- [ ] **Step 4: Commit and push**
|
| 704 |
+
|
| 705 |
+
```bash
|
| 706 |
+
git add tests/test_pose2d.py
|
| 707 |
+
git commit -m "test: mocked backend dispatch tests for YOLO, MediaPipe, Sapiens2"
|
| 708 |
+
git push
|
| 709 |
+
```
|
| 710 |
+
|
| 711 |
+
---
|
| 712 |
+
|
| 713 |
+
## Self-review
|
| 714 |
+
|
| 715 |
+
**Spec coverage:**
|
| 716 |
+
- β
Unified `POSE_MODELS` registry (Task 1)
|
| 717 |
+
- β
`DEFAULT_POSE_MODEL = YOLO26n` (Task 1)
|
| 718 |
+
- β
Backward-compat `YOLO_POSE_MODEL` / `YOLO_POSE_MODEL_HQ` aliases (Task 1)
|
| 719 |
+
- β
`_run_yolo` sub-runner (Task 2)
|
| 720 |
+
- β
`_run_mediapipe` with ONNX Runtime + BlazePoseβCOCO-17 mapping (Task 2)
|
| 721 |
+
- β
`_run_sapiens2` with transformers pipeline + named-keypointβCOCO-17 mapping (Task 2)
|
| 722 |
+
- β
`Pose2DAgent.run(model_key)` dispatch + fallback on unknown key (Task 2)
|
| 723 |
+
- β
`onnxruntime` added to requirements (Task 3)
|
| 724 |
+
- β
`Director.run(model_key)` threads key to agent (Task 4)
|
| 725 |
+
- β
`pose_model_dropdown` in UI (Task 5)
|
| 726 |
+
- β
`_map_inputs` + `submit_btn.click` wired (Task 5)
|
| 727 |
+
- β
Error handling: unknown key β warning + fallback; download failure β confidence=0 (Task 2)
|
| 728 |
+
- β
Mocked tests for all three backends (Task 6)
|
| 729 |
+
|
| 730 |
+
**Placeholder scan:** None found.
|
| 731 |
+
|
| 732 |
+
**Type consistency:** `model_key: str | None` used consistently across `Pose2DAgent.run`, `Director.run`, `process_video`. `config.POSE_MODELS` and `config.DEFAULT_POSE_MODEL` referenced consistently.
|
| 733 |
+
|
| 734 |
+
**Note on Sapiens2 keypoint format:** The `_run_sapiens2` implementation uses **named keypoint lookup** (by label string) rather than assuming fixed indices 0β16 = COCO. This is the safe approach β the transformers pipeline returns labeled keypoints and the code maps by name. If the pipeline returns unnamed keypoints (index-only), the `kp_lookup` will be empty and the frame will gracefully return `{}`.
|
docs/superpowers/plans/2026-06-09-pose-visualizer.md
ADDED
|
@@ -0,0 +1,914 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Pose Overlay Visualizer Implementation Plan
|
| 2 |
+
|
| 3 |
+
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
| 4 |
+
|
| 5 |
+
**Goal:** Add a pose overlay video output to FormScout with skeleton, motion trails, and velocity arrows, plus a per-joint velocity summary table.
|
| 6 |
+
|
| 7 |
+
**Architecture:** A new `formscout/agents/visualizer.py` runs after `director.run()` in `process_video()`; it uses Kalman-filtered per-joint velocity and OpenCV rendering. `app.py` gains a `gr.CheckboxGroup` for layer selection, a new `gr.Video` output tab, and a `gr.Markdown` velocity summary.
|
| 8 |
+
|
| 9 |
+
**Tech Stack:** `opencv-python`, `numpy`, `colorsys` (stdlib), `gradio`.
|
| 10 |
+
|
| 11 |
+
---
|
| 12 |
+
|
| 13 |
+
## File map
|
| 14 |
+
|
| 15 |
+
| File | Change |
|
| 16 |
+
|---|---|
|
| 17 |
+
| `formscout/agents/visualizer.py` | Create β Kalman filter, velocity, PoseVisualizer, summary |
|
| 18 |
+
| `tests/test_visualizer.py` | Create β all visualizer tests |
|
| 19 |
+
| `app.py` | Modify β overlay_layers checkbox, new tab, wiring |
|
| 20 |
+
|
| 21 |
+
---
|
| 22 |
+
|
| 23 |
+
## Task 1: `SimpleKalmanFilter` + `compute_joint_velocity`
|
| 24 |
+
|
| 25 |
+
**Files:**
|
| 26 |
+
- Create: `formscout/agents/visualizer.py`
|
| 27 |
+
- Create: `tests/test_visualizer.py`
|
| 28 |
+
|
| 29 |
+
- [ ] **Step 1: Write failing tests**
|
| 30 |
+
|
| 31 |
+
Create `tests/test_visualizer.py`:
|
| 32 |
+
|
| 33 |
+
```python
|
| 34 |
+
"""Tests for PoseVisualizer β no GPU, no model downloads."""
|
| 35 |
+
import numpy as np
|
| 36 |
+
import pytest
|
| 37 |
+
from formscout.types import IngestResult, Pose2DResult
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def _make_ingest(n=5, h=480, w=640, fps=30.0):
|
| 41 |
+
frames = [np.zeros((h, w, 3), dtype=np.uint8) for _ in range(n)]
|
| 42 |
+
return IngestResult(frames=frames, fps=fps, duration=n/fps, n_people=1, width=w, height=h)
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def _make_pose(n=5, w=640, h=480):
|
| 46 |
+
"""Synthetic Pose2DResult: 17 joints at fixed pixel positions, conf=0.9."""
|
| 47 |
+
kps_per_frame = []
|
| 48 |
+
for i in range(n):
|
| 49 |
+
frame_kps = {}
|
| 50 |
+
for j in range(17):
|
| 51 |
+
frame_kps[j] = {
|
| 52 |
+
"x": float(50 + j * 30 + i * 2), # slight movement each frame
|
| 53 |
+
"y": float(100 + j * 20),
|
| 54 |
+
"conf": 0.9,
|
| 55 |
+
}
|
| 56 |
+
kps_per_frame.append(frame_kps)
|
| 57 |
+
return Pose2DResult(keypoints=kps_per_frame, fps=30.0, confidence=0.9, notes="")
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
class TestComputeJointVelocity:
|
| 61 |
+
def test_returns_17_joints(self):
|
| 62 |
+
from formscout.agents.visualizer import compute_joint_velocity
|
| 63 |
+
pose = _make_pose(n=5)
|
| 64 |
+
result = compute_joint_velocity(pose.keypoints, fps=30.0)
|
| 65 |
+
assert len(result) == 17
|
| 66 |
+
|
| 67 |
+
def test_each_list_has_n_frames(self):
|
| 68 |
+
from formscout.agents.visualizer import compute_joint_velocity
|
| 69 |
+
pose = _make_pose(n=5)
|
| 70 |
+
result = compute_joint_velocity(pose.keypoints, fps=30.0)
|
| 71 |
+
for joint_idx, speeds in result.items():
|
| 72 |
+
assert len(speeds) == 5, f"joint {joint_idx} has {len(speeds)} speeds, expected 5"
|
| 73 |
+
|
| 74 |
+
def test_speeds_are_non_negative(self):
|
| 75 |
+
from formscout.agents.visualizer import compute_joint_velocity
|
| 76 |
+
pose = _make_pose(n=5)
|
| 77 |
+
result = compute_joint_velocity(pose.keypoints, fps=30.0)
|
| 78 |
+
for speeds in result.values():
|
| 79 |
+
assert all(s >= 0.0 for s in speeds)
|
| 80 |
+
|
| 81 |
+
def test_missing_keypoints_give_zero_speed(self):
|
| 82 |
+
from formscout.agents.visualizer import compute_joint_velocity
|
| 83 |
+
# All frames empty
|
| 84 |
+
empty_kps = [{} for _ in range(5)]
|
| 85 |
+
result = compute_joint_velocity(empty_kps, fps=30.0)
|
| 86 |
+
for speeds in result.values():
|
| 87 |
+
assert all(s == 0.0 for s in speeds)
|
| 88 |
+
```
|
| 89 |
+
|
| 90 |
+
- [ ] **Step 2: Run to confirm failure**
|
| 91 |
+
|
| 92 |
+
```bash
|
| 93 |
+
pytest tests/test_visualizer.py::TestComputeJointVelocity -v
|
| 94 |
+
```
|
| 95 |
+
|
| 96 |
+
Expected: `ERROR` β `ModuleNotFoundError: No module named 'formscout.agents.visualizer'`
|
| 97 |
+
|
| 98 |
+
- [ ] **Step 3: Create `formscout/agents/visualizer.py` with Kalman + velocity**
|
| 99 |
+
|
| 100 |
+
```python
|
| 101 |
+
"""
|
| 102 |
+
PoseVisualizer β annotated overlay video with skeleton, trails, velocity arrows.
|
| 103 |
+
|
| 104 |
+
Input: IngestResult + Pose2DResult
|
| 105 |
+
Output: .mp4 path (or None on failure/empty layers)
|
| 106 |
+
Failure: returns None, never raises.
|
| 107 |
+
"""
|
| 108 |
+
from __future__ import annotations
|
| 109 |
+
|
| 110 |
+
import colorsys
|
| 111 |
+
import logging
|
| 112 |
+
import math
|
| 113 |
+
import tempfile
|
| 114 |
+
from collections import deque
|
| 115 |
+
|
| 116 |
+
import cv2
|
| 117 |
+
import numpy as np
|
| 118 |
+
|
| 119 |
+
logger = logging.getLogger(__name__)
|
| 120 |
+
|
| 121 |
+
# ββ COCO constants ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 122 |
+
|
| 123 |
+
COCO_KEYPOINTS = [
|
| 124 |
+
"nose", "left_eye", "right_eye", "left_ear", "right_ear",
|
| 125 |
+
"left_shoulder", "right_shoulder", "left_elbow", "right_elbow",
|
| 126 |
+
"left_wrist", "right_wrist", "left_hip", "right_hip",
|
| 127 |
+
"left_knee", "right_knee", "left_ankle", "right_ankle",
|
| 128 |
+
]
|
| 129 |
+
|
| 130 |
+
COCO_SKELETON = [
|
| 131 |
+
(0, 1), (0, 2), (1, 3), (2, 4), # face
|
| 132 |
+
(5, 6), (5, 7), (7, 9), (6, 8), (8, 10), # arms
|
| 133 |
+
(5, 11), (6, 12), (11, 12), # torso
|
| 134 |
+
(11, 13), (13, 15), (12, 14), (14, 16), # legs
|
| 135 |
+
]
|
| 136 |
+
|
| 137 |
+
TRAIL_LENGTH = 10
|
| 138 |
+
MAX_ARROW_PX = 40
|
| 139 |
+
CONF_THRESHOLD = 0.3
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
# ββ Kalman filter βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 143 |
+
|
| 144 |
+
class SimpleKalmanFilter:
|
| 145 |
+
"""4-state Kalman filter (x, y, vx, vy) for joint tracking."""
|
| 146 |
+
|
| 147 |
+
def __init__(self, process_noise: float = 0.01, measurement_noise: float = 0.1):
|
| 148 |
+
self.is_initialized = False
|
| 149 |
+
self.state = np.zeros(4)
|
| 150 |
+
self.cov = np.eye(4) * 0.1
|
| 151 |
+
self.Q = np.eye(4) * process_noise
|
| 152 |
+
self.R = np.eye(2) * measurement_noise
|
| 153 |
+
self.H = np.array([[1, 0, 0, 0], [0, 1, 0, 0]], dtype=float)
|
| 154 |
+
|
| 155 |
+
def predict(self, dt: float = 1.0):
|
| 156 |
+
F = np.array([[1, 0, dt, 0], [0, 1, 0, dt], [0, 0, 1, 0], [0, 0, 0, 1]], dtype=float)
|
| 157 |
+
self.state = F @ self.state
|
| 158 |
+
self.cov = F @ self.cov @ F.T + self.Q
|
| 159 |
+
|
| 160 |
+
def update(self, x: float, y: float):
|
| 161 |
+
z = np.array([x, y])
|
| 162 |
+
if not self.is_initialized:
|
| 163 |
+
self.state[:2] = z
|
| 164 |
+
self.is_initialized = True
|
| 165 |
+
return
|
| 166 |
+
S = self.H @ self.cov @ self.H.T + self.R
|
| 167 |
+
K = self.cov @ self.H.T @ np.linalg.inv(S)
|
| 168 |
+
self.state = self.state + K @ (z - self.H @ self.state)
|
| 169 |
+
self.cov = (np.eye(4) - K @ self.H) @ self.cov
|
| 170 |
+
|
| 171 |
+
def velocity_magnitude(self) -> float:
|
| 172 |
+
vx, vy = self.state[2], self.state[3]
|
| 173 |
+
return math.sqrt(vx * vx + vy * vy)
|
| 174 |
+
|
| 175 |
+
def velocity_vector(self) -> tuple[float, float]:
|
| 176 |
+
return float(self.state[2]), float(self.state[3])
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
# ββ Velocity computation ββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 180 |
+
|
| 181 |
+
def compute_joint_velocity(
|
| 182 |
+
keypoints_per_frame: list[dict],
|
| 183 |
+
fps: float,
|
| 184 |
+
) -> dict[int, list[float]]:
|
| 185 |
+
"""
|
| 186 |
+
Compute Kalman-filtered per-joint speed (px/s) for each frame.
|
| 187 |
+
|
| 188 |
+
Returns dict[joint_idx, [speed_frame0, speed_frame1, ...]] for all 17 COCO joints.
|
| 189 |
+
Missing/low-confidence keypoints yield speed=0.0 for that frame.
|
| 190 |
+
"""
|
| 191 |
+
dt = 1.0 / fps if fps > 0 else 1.0
|
| 192 |
+
filters: dict[int, SimpleKalmanFilter] = {j: SimpleKalmanFilter() for j in range(17)}
|
| 193 |
+
result: dict[int, list[float]] = {j: [] for j in range(17)}
|
| 194 |
+
|
| 195 |
+
for frame_kps in keypoints_per_frame:
|
| 196 |
+
for j in range(17):
|
| 197 |
+
kf = filters[j]
|
| 198 |
+
kp = frame_kps.get(j)
|
| 199 |
+
kf.predict(dt)
|
| 200 |
+
if kp and kp.get("conf", 0.0) >= CONF_THRESHOLD:
|
| 201 |
+
kf.update(kp["x"], kp["y"])
|
| 202 |
+
speed = kf.velocity_magnitude()
|
| 203 |
+
else:
|
| 204 |
+
speed = 0.0
|
| 205 |
+
result[j].append(speed)
|
| 206 |
+
|
| 207 |
+
return result
|
| 208 |
+
```
|
| 209 |
+
|
| 210 |
+
- [ ] **Step 4: Run tests**
|
| 211 |
+
|
| 212 |
+
```bash
|
| 213 |
+
pytest tests/test_visualizer.py::TestComputeJointVelocity -v
|
| 214 |
+
```
|
| 215 |
+
|
| 216 |
+
Expected: 4 PASS
|
| 217 |
+
|
| 218 |
+
- [ ] **Step 5: Commit**
|
| 219 |
+
|
| 220 |
+
```bash
|
| 221 |
+
git add formscout/agents/visualizer.py tests/test_visualizer.py
|
| 222 |
+
git commit -m "feat: SimpleKalmanFilter + compute_joint_velocity (4 tests pass)"
|
| 223 |
+
```
|
| 224 |
+
|
| 225 |
+
---
|
| 226 |
+
|
| 227 |
+
## Task 2: `PoseVisualizer._draw_skeleton`
|
| 228 |
+
|
| 229 |
+
**Files:**
|
| 230 |
+
- Modify: `formscout/agents/visualizer.py`
|
| 231 |
+
- Modify: `tests/test_visualizer.py`
|
| 232 |
+
|
| 233 |
+
- [ ] **Step 1: Write failing test**
|
| 234 |
+
|
| 235 |
+
Append to `tests/test_visualizer.py`:
|
| 236 |
+
|
| 237 |
+
```python
|
| 238 |
+
class TestDrawSkeleton:
|
| 239 |
+
def test_skeleton_draws_without_error(self):
|
| 240 |
+
from formscout.agents.visualizer import PoseVisualizer
|
| 241 |
+
vis = PoseVisualizer()
|
| 242 |
+
frame = np.zeros((480, 640, 3), dtype=np.uint8)
|
| 243 |
+
kps = {j: {"x": float(50 + j * 30), "y": float(100 + j * 20), "conf": 0.9}
|
| 244 |
+
for j in range(17)}
|
| 245 |
+
result = vis._draw_skeleton(frame.copy(), kps)
|
| 246 |
+
assert result.shape == frame.shape
|
| 247 |
+
# Frame must be modified (not all zeros after drawing)
|
| 248 |
+
assert not np.array_equal(result, frame)
|
| 249 |
+
|
| 250 |
+
def test_low_confidence_keypoints_not_drawn(self):
|
| 251 |
+
from formscout.agents.visualizer import PoseVisualizer
|
| 252 |
+
vis = PoseVisualizer()
|
| 253 |
+
frame = np.zeros((480, 640, 3), dtype=np.uint8)
|
| 254 |
+
# All keypoints below threshold
|
| 255 |
+
kps = {j: {"x": float(50 + j * 30), "y": 100.0, "conf": 0.1} for j in range(17)}
|
| 256 |
+
result = vis._draw_skeleton(frame.copy(), kps)
|
| 257 |
+
# Nothing drawn β frame stays all zeros
|
| 258 |
+
assert np.array_equal(result, frame)
|
| 259 |
+
```
|
| 260 |
+
|
| 261 |
+
- [ ] **Step 2: Run to confirm failure**
|
| 262 |
+
|
| 263 |
+
```bash
|
| 264 |
+
pytest tests/test_visualizer.py::TestDrawSkeleton -v
|
| 265 |
+
```
|
| 266 |
+
|
| 267 |
+
Expected: FAIL β `AttributeError: 'PoseVisualizer' object has no attribute '_draw_skeleton'`
|
| 268 |
+
|
| 269 |
+
- [ ] **Step 3: Add `PoseVisualizer` class with `_draw_skeleton` to `visualizer.py`**
|
| 270 |
+
|
| 271 |
+
Append after `compute_joint_velocity`:
|
| 272 |
+
|
| 273 |
+
```python
|
| 274 |
+
# ββ Helpers βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 275 |
+
|
| 276 |
+
def _conf_to_bgr(conf: float) -> tuple[int, int, int]:
|
| 277 |
+
"""Map confidence 0β1 to BGR color redβgreen via HSV."""
|
| 278 |
+
hue = conf * 120.0 / 360.0
|
| 279 |
+
r, g, b = colorsys.hsv_to_rgb(hue, 1.0, 1.0)
|
| 280 |
+
return (int(b * 255), int(g * 255), int(r * 255))
|
| 281 |
+
|
| 282 |
+
|
| 283 |
+
# ββ PoseVisualizer ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 284 |
+
|
| 285 |
+
class PoseVisualizer:
|
| 286 |
+
"""Renders skeleton, trails, and velocity arrows onto video frames."""
|
| 287 |
+
|
| 288 |
+
def __init__(self):
|
| 289 |
+
self.last_velocities: dict[int, list[float]] = {}
|
| 290 |
+
|
| 291 |
+
# ββ Skeleton ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 292 |
+
|
| 293 |
+
def _draw_skeleton(self, frame: np.ndarray, kps: dict) -> np.ndarray:
|
| 294 |
+
"""Draw COCO-17 bones (white) and joints (confidence-colored) onto frame."""
|
| 295 |
+
visible = {j: kp for j, kp in kps.items() if kp.get("conf", 0.0) >= CONF_THRESHOLD}
|
| 296 |
+
|
| 297 |
+
# Bones
|
| 298 |
+
for j1, j2 in COCO_SKELETON:
|
| 299 |
+
if j1 in visible and j2 in visible:
|
| 300 |
+
p1 = (int(visible[j1]["x"]), int(visible[j1]["y"]))
|
| 301 |
+
p2 = (int(visible[j2]["x"]), int(visible[j2]["y"]))
|
| 302 |
+
cv2.line(frame, p1, p2, (255, 255, 255), 2)
|
| 303 |
+
|
| 304 |
+
# Joints
|
| 305 |
+
for j, kp in visible.items():
|
| 306 |
+
pt = (int(kp["x"]), int(kp["y"]))
|
| 307 |
+
color = _conf_to_bgr(kp["conf"])
|
| 308 |
+
cv2.circle(frame, pt, 4, color, -1)
|
| 309 |
+
cv2.circle(frame, pt, 5, (255, 255, 255), 1)
|
| 310 |
+
|
| 311 |
+
return frame
|
| 312 |
+
```
|
| 313 |
+
|
| 314 |
+
- [ ] **Step 4: Run tests**
|
| 315 |
+
|
| 316 |
+
```bash
|
| 317 |
+
pytest tests/test_visualizer.py::TestDrawSkeleton -v
|
| 318 |
+
```
|
| 319 |
+
|
| 320 |
+
Expected: 2 PASS
|
| 321 |
+
|
| 322 |
+
- [ ] **Step 5: Commit**
|
| 323 |
+
|
| 324 |
+
```bash
|
| 325 |
+
git add formscout/agents/visualizer.py tests/test_visualizer.py
|
| 326 |
+
git commit -m "feat: PoseVisualizer._draw_skeleton with confidence-colored joints"
|
| 327 |
+
```
|
| 328 |
+
|
| 329 |
+
---
|
| 330 |
+
|
| 331 |
+
## Task 3: `PoseVisualizer._draw_trails`
|
| 332 |
+
|
| 333 |
+
**Files:**
|
| 334 |
+
- Modify: `formscout/agents/visualizer.py`
|
| 335 |
+
- Modify: `tests/test_visualizer.py`
|
| 336 |
+
|
| 337 |
+
- [ ] **Step 1: Write failing test**
|
| 338 |
+
|
| 339 |
+
Append to `tests/test_visualizer.py`:
|
| 340 |
+
|
| 341 |
+
```python
|
| 342 |
+
class TestDrawTrails:
|
| 343 |
+
def test_trails_draw_without_error(self):
|
| 344 |
+
from formscout.agents.visualizer import PoseVisualizer, TRAIL_LENGTH
|
| 345 |
+
from collections import deque
|
| 346 |
+
vis = PoseVisualizer()
|
| 347 |
+
frame = np.zeros((480, 640, 3), dtype=np.uint8)
|
| 348 |
+
# Build a trail history for joint 0 with 5 positions
|
| 349 |
+
trail_history = {
|
| 350 |
+
0: deque([(100 + i * 5, 200 + i * 3) for i in range(5)], maxlen=TRAIL_LENGTH)
|
| 351 |
+
}
|
| 352 |
+
result = vis._draw_trails(frame.copy(), trail_history)
|
| 353 |
+
assert result.shape == frame.shape
|
| 354 |
+
# Trail should modify at least some pixels
|
| 355 |
+
assert not np.array_equal(result, frame)
|
| 356 |
+
|
| 357 |
+
def test_short_trail_no_crash(self):
|
| 358 |
+
from formscout.agents.visualizer import PoseVisualizer, TRAIL_LENGTH
|
| 359 |
+
from collections import deque
|
| 360 |
+
vis = PoseVisualizer()
|
| 361 |
+
frame = np.zeros((480, 640, 3), dtype=np.uint8)
|
| 362 |
+
# Only one point β no line possible
|
| 363 |
+
trail_history = {0: deque([(100, 200)], maxlen=TRAIL_LENGTH)}
|
| 364 |
+
result = vis._draw_trails(frame.copy(), trail_history)
|
| 365 |
+
# No crash, frame unchanged (single point = no segment)
|
| 366 |
+
assert np.array_equal(result, frame)
|
| 367 |
+
```
|
| 368 |
+
|
| 369 |
+
- [ ] **Step 2: Run to confirm failure**
|
| 370 |
+
|
| 371 |
+
```bash
|
| 372 |
+
pytest tests/test_visualizer.py::TestDrawTrails -v
|
| 373 |
+
```
|
| 374 |
+
|
| 375 |
+
Expected: FAIL β `AttributeError: 'PoseVisualizer' object has no attribute '_draw_trails'`
|
| 376 |
+
|
| 377 |
+
- [ ] **Step 3: Add `_draw_trails` to `PoseVisualizer`**
|
| 378 |
+
|
| 379 |
+
Inside the `PoseVisualizer` class, after `_draw_skeleton`:
|
| 380 |
+
|
| 381 |
+
```python
|
| 382 |
+
# ββ Trails βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 383 |
+
|
| 384 |
+
def _draw_trails(self, frame: np.ndarray, trail_history: dict) -> np.ndarray:
|
| 385 |
+
"""Draw fading motion trails for each joint."""
|
| 386 |
+
for joint_idx, trail in trail_history.items():
|
| 387 |
+
pts = list(trail)
|
| 388 |
+
if len(pts) < 2:
|
| 389 |
+
continue
|
| 390 |
+
for i in range(1, len(pts)):
|
| 391 |
+
alpha = i / len(pts)
|
| 392 |
+
brightness = int(255 * alpha)
|
| 393 |
+
color = (brightness, brightness, brightness)
|
| 394 |
+
thickness = max(1, int(3 * alpha))
|
| 395 |
+
p1 = (int(pts[i - 1][0]), int(pts[i - 1][1]))
|
| 396 |
+
p2 = (int(pts[i][0]), int(pts[i][1]))
|
| 397 |
+
cv2.line(frame, p1, p2, color, thickness)
|
| 398 |
+
return frame
|
| 399 |
+
```
|
| 400 |
+
|
| 401 |
+
- [ ] **Step 4: Run tests**
|
| 402 |
+
|
| 403 |
+
```bash
|
| 404 |
+
pytest tests/test_visualizer.py::TestDrawTrails -v
|
| 405 |
+
```
|
| 406 |
+
|
| 407 |
+
Expected: 2 PASS
|
| 408 |
+
|
| 409 |
+
- [ ] **Step 5: Commit**
|
| 410 |
+
|
| 411 |
+
```bash
|
| 412 |
+
git add formscout/agents/visualizer.py tests/test_visualizer.py
|
| 413 |
+
git commit -m "feat: PoseVisualizer._draw_trails with fading alpha"
|
| 414 |
+
```
|
| 415 |
+
|
| 416 |
+
---
|
| 417 |
+
|
| 418 |
+
## Task 4: `PoseVisualizer._draw_velocity_arrows`
|
| 419 |
+
|
| 420 |
+
**Files:**
|
| 421 |
+
- Modify: `formscout/agents/visualizer.py`
|
| 422 |
+
- Modify: `tests/test_visualizer.py`
|
| 423 |
+
|
| 424 |
+
- [ ] **Step 1: Write failing test**
|
| 425 |
+
|
| 426 |
+
Append to `tests/test_visualizer.py`:
|
| 427 |
+
|
| 428 |
+
```python
|
| 429 |
+
class TestDrawVelocityArrows:
|
| 430 |
+
def test_arrows_draw_without_error(self):
|
| 431 |
+
from formscout.agents.visualizer import PoseVisualizer
|
| 432 |
+
vis = PoseVisualizer()
|
| 433 |
+
frame = np.zeros((480, 640, 3), dtype=np.uint8)
|
| 434 |
+
kps = {j: {"x": float(50 + j * 30), "y": float(100 + j * 20), "conf": 0.9}
|
| 435 |
+
for j in range(17)}
|
| 436 |
+
prev_kps = {j: {"x": float(48 + j * 30), "y": float(98 + j * 20), "conf": 0.9}
|
| 437 |
+
for j in range(17)}
|
| 438 |
+
# velocities: joint 5 moving fast
|
| 439 |
+
velocities = {j: [0.0] * 5 for j in range(17)}
|
| 440 |
+
velocities[5] = [0.0, 10.0, 50.0, 80.0, 120.0]
|
| 441 |
+
result = vis._draw_velocity_arrows(frame.copy(), kps, prev_kps, velocities, frame_idx=4)
|
| 442 |
+
assert result.shape == frame.shape
|
| 443 |
+
|
| 444 |
+
def test_no_prev_kps_no_crash(self):
|
| 445 |
+
from formscout.agents.visualizer import PoseVisualizer
|
| 446 |
+
vis = PoseVisualizer()
|
| 447 |
+
frame = np.zeros((480, 640, 3), dtype=np.uint8)
|
| 448 |
+
kps = {j: {"x": float(50 + j * 30), "y": 100.0, "conf": 0.9} for j in range(17)}
|
| 449 |
+
velocities = {j: [50.0] * 5 for j in range(17)}
|
| 450 |
+
# prev_kps is None β should skip without crash
|
| 451 |
+
result = vis._draw_velocity_arrows(frame.copy(), kps, None, velocities, frame_idx=0)
|
| 452 |
+
assert result.shape == frame.shape
|
| 453 |
+
```
|
| 454 |
+
|
| 455 |
+
- [ ] **Step 2: Run to confirm failure**
|
| 456 |
+
|
| 457 |
+
```bash
|
| 458 |
+
pytest tests/test_visualizer.py::TestDrawVelocityArrows -v
|
| 459 |
+
```
|
| 460 |
+
|
| 461 |
+
Expected: FAIL β `AttributeError: 'PoseVisualizer' object has no attribute '_draw_velocity_arrows'`
|
| 462 |
+
|
| 463 |
+
- [ ] **Step 3: Add `_draw_velocity_arrows` to `PoseVisualizer`**
|
| 464 |
+
|
| 465 |
+
Inside the `PoseVisualizer` class, after `_draw_trails`:
|
| 466 |
+
|
| 467 |
+
```python
|
| 468 |
+
# ββ Velocity arrows βββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 469 |
+
|
| 470 |
+
def _draw_velocity_arrows(
|
| 471 |
+
self,
|
| 472 |
+
frame: np.ndarray,
|
| 473 |
+
kps: dict,
|
| 474 |
+
prev_kps: dict | None,
|
| 475 |
+
velocities: dict[int, list[float]],
|
| 476 |
+
frame_idx: int,
|
| 477 |
+
) -> np.ndarray:
|
| 478 |
+
"""Draw per-joint velocity arrows scaled by speed."""
|
| 479 |
+
if prev_kps is None:
|
| 480 |
+
return frame
|
| 481 |
+
|
| 482 |
+
all_speeds = [velocities[j][frame_idx] for j in range(17) if frame_idx < len(velocities.get(j, []))]
|
| 483 |
+
peak = max(all_speeds) if all_speeds else 1.0
|
| 484 |
+
if peak == 0.0:
|
| 485 |
+
return frame
|
| 486 |
+
|
| 487 |
+
for j in range(17):
|
| 488 |
+
kp = kps.get(j)
|
| 489 |
+
pk = prev_kps.get(j)
|
| 490 |
+
if not kp or not pk:
|
| 491 |
+
continue
|
| 492 |
+
if kp.get("conf", 0.0) < CONF_THRESHOLD:
|
| 493 |
+
continue
|
| 494 |
+
speeds = velocities.get(j, [])
|
| 495 |
+
if frame_idx >= len(speeds):
|
| 496 |
+
continue
|
| 497 |
+
speed = speeds[frame_idx]
|
| 498 |
+
if speed == 0.0:
|
| 499 |
+
continue
|
| 500 |
+
|
| 501 |
+
dx = kp["x"] - pk["x"]
|
| 502 |
+
dy = kp["y"] - pk["y"]
|
| 503 |
+
mag = math.sqrt(dx * dx + dy * dy)
|
| 504 |
+
if mag < 1e-6:
|
| 505 |
+
continue
|
| 506 |
+
|
| 507 |
+
# Normalize direction, scale to arrow length
|
| 508 |
+
length = min(speed / peak * MAX_ARROW_PX, MAX_ARROW_PX)
|
| 509 |
+
nx, ny = dx / mag, dy / mag
|
| 510 |
+
start = (int(kp["x"]), int(kp["y"]))
|
| 511 |
+
end = (int(kp["x"] + nx * length), int(kp["y"] + ny * length))
|
| 512 |
+
|
| 513 |
+
ratio = speed / peak
|
| 514 |
+
if ratio < 0.33:
|
| 515 |
+
color = (0, 200, 0) # green
|
| 516 |
+
elif ratio < 0.66:
|
| 517 |
+
color = (0, 140, 255) # orange
|
| 518 |
+
else:
|
| 519 |
+
color = (0, 0, 255) # red
|
| 520 |
+
|
| 521 |
+
cv2.arrowedLine(frame, start, end, color, 2, tipLength=0.35)
|
| 522 |
+
|
| 523 |
+
return frame
|
| 524 |
+
```
|
| 525 |
+
|
| 526 |
+
- [ ] **Step 4: Run tests**
|
| 527 |
+
|
| 528 |
+
```bash
|
| 529 |
+
pytest tests/test_visualizer.py::TestDrawVelocityArrows -v
|
| 530 |
+
```
|
| 531 |
+
|
| 532 |
+
Expected: 2 PASS
|
| 533 |
+
|
| 534 |
+
- [ ] **Step 5: Commit**
|
| 535 |
+
|
| 536 |
+
```bash
|
| 537 |
+
git add formscout/agents/visualizer.py tests/test_visualizer.py
|
| 538 |
+
git commit -m "feat: PoseVisualizer._draw_velocity_arrows speed-colored"
|
| 539 |
+
```
|
| 540 |
+
|
| 541 |
+
---
|
| 542 |
+
|
| 543 |
+
## Task 5: `render_video` + `build_velocity_summary`
|
| 544 |
+
|
| 545 |
+
**Files:**
|
| 546 |
+
- Modify: `formscout/agents/visualizer.py`
|
| 547 |
+
- Modify: `tests/test_visualizer.py`
|
| 548 |
+
|
| 549 |
+
- [ ] **Step 1: Write failing tests**
|
| 550 |
+
|
| 551 |
+
Append to `tests/test_visualizer.py`:
|
| 552 |
+
|
| 553 |
+
```python
|
| 554 |
+
class TestRenderVideo:
|
| 555 |
+
def test_creates_mp4_file(self, tmp_path):
|
| 556 |
+
from formscout.agents.visualizer import PoseVisualizer
|
| 557 |
+
vis = PoseVisualizer()
|
| 558 |
+
ingest = _make_ingest(n=5)
|
| 559 |
+
pose = _make_pose(n=5)
|
| 560 |
+
out = str(tmp_path / "out.mp4")
|
| 561 |
+
result = vis.render_video(ingest, pose, {"skeleton"}, out)
|
| 562 |
+
assert result is not None
|
| 563 |
+
import os
|
| 564 |
+
assert os.path.exists(result)
|
| 565 |
+
assert os.path.getsize(result) > 0
|
| 566 |
+
|
| 567 |
+
def test_empty_layers_returns_none(self, tmp_path):
|
| 568 |
+
from formscout.agents.visualizer import PoseVisualizer
|
| 569 |
+
vis = PoseVisualizer()
|
| 570 |
+
out = str(tmp_path / "out.mp4")
|
| 571 |
+
result = vis.render_video(_make_ingest(), _make_pose(), set(), out)
|
| 572 |
+
assert result is None
|
| 573 |
+
|
| 574 |
+
def test_no_detections_returns_none(self, tmp_path):
|
| 575 |
+
from formscout.agents.visualizer import PoseVisualizer
|
| 576 |
+
vis = PoseVisualizer()
|
| 577 |
+
ingest = _make_ingest(n=5)
|
| 578 |
+
empty_pose = Pose2DResult(
|
| 579 |
+
keypoints=[{} for _ in range(5)], fps=30.0, confidence=0.0, notes=""
|
| 580 |
+
)
|
| 581 |
+
out = str(tmp_path / "out.mp4")
|
| 582 |
+
result = vis.render_video(ingest, empty_pose, {"skeleton"}, out)
|
| 583 |
+
assert result is None
|
| 584 |
+
|
| 585 |
+
def test_last_velocities_set_after_render(self, tmp_path):
|
| 586 |
+
from formscout.agents.visualizer import PoseVisualizer
|
| 587 |
+
vis = PoseVisualizer()
|
| 588 |
+
out = str(tmp_path / "out.mp4")
|
| 589 |
+
vis.render_video(_make_ingest(n=5), _make_pose(n=5), {"skeleton"}, out)
|
| 590 |
+
assert len(vis.last_velocities) == 17
|
| 591 |
+
|
| 592 |
+
|
| 593 |
+
class TestBuildVelocitySummary:
|
| 594 |
+
def test_returns_markdown_table(self):
|
| 595 |
+
from formscout.agents.visualizer import build_velocity_summary, compute_joint_velocity
|
| 596 |
+
pose = _make_pose(n=10)
|
| 597 |
+
vels = compute_joint_velocity(pose.keypoints, fps=30.0)
|
| 598 |
+
result = build_velocity_summary(pose.keypoints, vels)
|
| 599 |
+
assert "|" in result
|
| 600 |
+
# At least one COCO joint name appears
|
| 601 |
+
assert any(name in result for name in ["knee", "shoulder", "hip", "ankle"])
|
| 602 |
+
|
| 603 |
+
def test_empty_keypoints_returns_empty_string(self):
|
| 604 |
+
from formscout.agents.visualizer import build_velocity_summary
|
| 605 |
+
empty_kps = [{} for _ in range(5)]
|
| 606 |
+
vels = {j: [0.0] * 5 for j in range(17)}
|
| 607 |
+
result = build_velocity_summary(empty_kps, vels)
|
| 608 |
+
assert result == ""
|
| 609 |
+
```
|
| 610 |
+
|
| 611 |
+
- [ ] **Step 2: Run to confirm failure**
|
| 612 |
+
|
| 613 |
+
```bash
|
| 614 |
+
pytest tests/test_visualizer.py::TestRenderVideo tests/test_visualizer.py::TestBuildVelocitySummary -v
|
| 615 |
+
```
|
| 616 |
+
|
| 617 |
+
Expected: FAIL β `AttributeError: 'PoseVisualizer' object has no attribute 'render_video'`
|
| 618 |
+
|
| 619 |
+
- [ ] **Step 3: Add `render_video` to `PoseVisualizer`**
|
| 620 |
+
|
| 621 |
+
Inside the `PoseVisualizer` class, after `_draw_velocity_arrows`:
|
| 622 |
+
|
| 623 |
+
```python
|
| 624 |
+
# ββ Public ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 625 |
+
|
| 626 |
+
def render_video(
|
| 627 |
+
self,
|
| 628 |
+
ingest,
|
| 629 |
+
pose2d,
|
| 630 |
+
layers: set[str],
|
| 631 |
+
output_path: str,
|
| 632 |
+
) -> str | None:
|
| 633 |
+
"""
|
| 634 |
+
Render annotated video. Returns output_path on success, None otherwise.
|
| 635 |
+
layers: subset of {"skeleton", "trails", "velocity_arrows"}
|
| 636 |
+
"""
|
| 637 |
+
if not layers:
|
| 638 |
+
return None
|
| 639 |
+
|
| 640 |
+
# Require at least one detected frame
|
| 641 |
+
if not any(pose2d.keypoints):
|
| 642 |
+
return None
|
| 643 |
+
|
| 644 |
+
try:
|
| 645 |
+
velocities = compute_joint_velocity(pose2d.keypoints, ingest.fps)
|
| 646 |
+
self.last_velocities = velocities
|
| 647 |
+
|
| 648 |
+
frames = ingest.frames
|
| 649 |
+
h, w = frames[0].shape[:2]
|
| 650 |
+
fps = ingest.fps or 30.0
|
| 651 |
+
|
| 652 |
+
fourcc = cv2.VideoWriter_fourcc(*"mp4v")
|
| 653 |
+
writer = cv2.VideoWriter(output_path, fourcc, fps, (w, h))
|
| 654 |
+
if not writer.isOpened():
|
| 655 |
+
logger.warning("VideoWriter failed to open: %s", output_path)
|
| 656 |
+
return None
|
| 657 |
+
|
| 658 |
+
trail_history: dict[int, deque] = {j: deque(maxlen=TRAIL_LENGTH) for j in range(17)}
|
| 659 |
+
prev_kps: dict | None = None
|
| 660 |
+
|
| 661 |
+
for frame_idx, (frame, kps) in enumerate(zip(frames, pose2d.keypoints)):
|
| 662 |
+
out_frame = frame.copy()
|
| 663 |
+
|
| 664 |
+
if "trails" in layers:
|
| 665 |
+
# Update trail history before drawing
|
| 666 |
+
for j, kp in kps.items():
|
| 667 |
+
if kp.get("conf", 0.0) >= CONF_THRESHOLD:
|
| 668 |
+
trail_history[j].append((kp["x"], kp["y"]))
|
| 669 |
+
out_frame = self._draw_trails(out_frame, trail_history)
|
| 670 |
+
|
| 671 |
+
if "skeleton" in layers:
|
| 672 |
+
out_frame = self._draw_skeleton(out_frame, kps)
|
| 673 |
+
|
| 674 |
+
if "velocity_arrows" in layers:
|
| 675 |
+
out_frame = self._draw_velocity_arrows(
|
| 676 |
+
out_frame, kps, prev_kps, velocities, frame_idx
|
| 677 |
+
)
|
| 678 |
+
|
| 679 |
+
writer.write(out_frame)
|
| 680 |
+
prev_kps = kps
|
| 681 |
+
|
| 682 |
+
writer.release()
|
| 683 |
+
return output_path
|
| 684 |
+
|
| 685 |
+
except Exception as e:
|
| 686 |
+
logger.warning("render_video failed: %s", e)
|
| 687 |
+
return None
|
| 688 |
+
```
|
| 689 |
+
|
| 690 |
+
- [ ] **Step 4: Add `build_velocity_summary` after the class**
|
| 691 |
+
|
| 692 |
+
After the `PoseVisualizer` class definition, add:
|
| 693 |
+
|
| 694 |
+
```python
|
| 695 |
+
# ββ Velocity summary ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 696 |
+
|
| 697 |
+
def build_velocity_summary(
|
| 698 |
+
keypoints_per_frame: list[dict],
|
| 699 |
+
velocities: dict[int, list[float]],
|
| 700 |
+
) -> str:
|
| 701 |
+
"""Return markdown table of per-joint avg/peak velocity. Empty string if no valid joints."""
|
| 702 |
+
n_frames = len(keypoints_per_frame)
|
| 703 |
+
if n_frames == 0:
|
| 704 |
+
return ""
|
| 705 |
+
|
| 706 |
+
rows = []
|
| 707 |
+
for j in range(17):
|
| 708 |
+
# Count frames where this joint is detected
|
| 709 |
+
detected = sum(
|
| 710 |
+
1 for kps in keypoints_per_frame
|
| 711 |
+
if kps.get(j, {}).get("conf", 0.0) >= CONF_THRESHOLD
|
| 712 |
+
)
|
| 713 |
+
if detected < n_frames * 0.5:
|
| 714 |
+
continue # skip joints present in <50% of frames
|
| 715 |
+
|
| 716 |
+
speeds = velocities.get(j, [])
|
| 717 |
+
if not speeds:
|
| 718 |
+
continue
|
| 719 |
+
|
| 720 |
+
avg_speed = sum(speeds) / len(speeds)
|
| 721 |
+
peak_speed = max(speeds)
|
| 722 |
+
rows.append((COCO_KEYPOINTS[j], avg_speed, peak_speed))
|
| 723 |
+
|
| 724 |
+
if not rows:
|
| 725 |
+
return ""
|
| 726 |
+
|
| 727 |
+
rows.sort(key=lambda r: r[2], reverse=True) # sort by peak descending
|
| 728 |
+
lines = [
|
| 729 |
+
"| Joint | Avg (px/s) | Peak (px/s) |",
|
| 730 |
+
"|---|---|---|",
|
| 731 |
+
]
|
| 732 |
+
for name, avg, peak in rows:
|
| 733 |
+
lines.append(f"| {name} | {avg:.1f} | {peak:.1f} |")
|
| 734 |
+
return "\n".join(lines)
|
| 735 |
+
```
|
| 736 |
+
|
| 737 |
+
- [ ] **Step 5: Run all visualizer tests**
|
| 738 |
+
|
| 739 |
+
```bash
|
| 740 |
+
pytest tests/test_visualizer.py -v
|
| 741 |
+
```
|
| 742 |
+
|
| 743 |
+
Expected: all tests PASS (4 + 2 + 2 + 2 + 4 + 2 = 16 total)
|
| 744 |
+
|
| 745 |
+
- [ ] **Step 6: Commit**
|
| 746 |
+
|
| 747 |
+
```bash
|
| 748 |
+
git add formscout/agents/visualizer.py tests/test_visualizer.py
|
| 749 |
+
git commit -m "feat: PoseVisualizer.render_video + build_velocity_summary (16 tests pass)"
|
| 750 |
+
```
|
| 751 |
+
|
| 752 |
+
---
|
| 753 |
+
|
| 754 |
+
## Task 6: Wire `app.py`
|
| 755 |
+
|
| 756 |
+
**Files:**
|
| 757 |
+
- Modify: `app.py`
|
| 758 |
+
|
| 759 |
+
- [ ] **Step 1: Add `import tempfile` if not present and import visualizer in `process_video`**
|
| 760 |
+
|
| 761 |
+
Check the top of `app.py` for `import tempfile`. If missing, add it alongside the other stdlib imports. (Look at the existing import block and add `import tempfile` there.)
|
| 762 |
+
|
| 763 |
+
- [ ] **Step 2: Update `process_video()` signature and body**
|
| 764 |
+
|
| 765 |
+
Replace the existing `process_video` function (lines 46β83) with:
|
| 766 |
+
|
| 767 |
+
```python
|
| 768 |
+
def process_video(video_path: str, test_name: str, side: str, model_key: str, layers: list[str]):
|
| 769 |
+
"""Process an uploaded video through the FormScout pipeline."""
|
| 770 |
+
if not video_path:
|
| 771 |
+
return (
|
| 772 |
+
_render_empty_state(),
|
| 773 |
+
"Upload a video to begin analysis.",
|
| 774 |
+
"",
|
| 775 |
+
"",
|
| 776 |
+
None,
|
| 777 |
+
"",
|
| 778 |
+
)
|
| 779 |
+
|
| 780 |
+
director = Director()
|
| 781 |
+
state = director.run(video_path, test_name=test_name, side=side, model_key=model_key)
|
| 782 |
+
|
| 783 |
+
# βββ Score card βββ
|
| 784 |
+
score_html = _render_empty_state()
|
| 785 |
+
score_details = ""
|
| 786 |
+
|
| 787 |
+
if state.features:
|
| 788 |
+
result = score_test(state.features)
|
| 789 |
+
judge = state.judge
|
| 790 |
+
if judge and judge.score is not None:
|
| 791 |
+
score_html = _render_score_card(judge.score, judge.confidence, judge.needs_human)
|
| 792 |
+
score_details = _render_score_details_judge(judge, result, state.features)
|
| 793 |
+
elif judge and judge.needs_human:
|
| 794 |
+
score_html = _render_score_card(0, 0, True)
|
| 795 |
+
score_details = f"### Needs Clinician Review\n{judge.rationale}"
|
| 796 |
+
else:
|
| 797 |
+
score_html = _render_score_card(result.score, result.confidence, result.needs_human)
|
| 798 |
+
score_details = _render_score_details(result, state.features)
|
| 799 |
+
|
| 800 |
+
# βββ Pipeline info βββ
|
| 801 |
+
pipeline_md = _render_pipeline_status(state)
|
| 802 |
+
|
| 803 |
+
# βββ Warnings/errors βββ
|
| 804 |
+
alerts = _render_alerts(state)
|
| 805 |
+
|
| 806 |
+
# βββ Overlay video βββ
|
| 807 |
+
overlay_path = None
|
| 808 |
+
vel_summary = ""
|
| 809 |
+
layer_set = {lbl.lower().replace(" ", "_") for lbl in (layers or [])}
|
| 810 |
+
if layer_set and state.ingest and state.pose2d:
|
| 811 |
+
try:
|
| 812 |
+
from formscout.agents.visualizer import PoseVisualizer, build_velocity_summary
|
| 813 |
+
vis = PoseVisualizer()
|
| 814 |
+
with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as f:
|
| 815 |
+
out_path = f.name
|
| 816 |
+
overlay_path = vis.render_video(state.ingest, state.pose2d, layer_set, out_path)
|
| 817 |
+
if overlay_path:
|
| 818 |
+
vel_summary = build_velocity_summary(state.pose2d.keypoints, vis.last_velocities)
|
| 819 |
+
except Exception as e:
|
| 820 |
+
alerts = (alerts or "") + f"\nβ οΈ Visualizer error: {e}"
|
| 821 |
+
|
| 822 |
+
return score_html, pipeline_md, score_details, alerts, overlay_path, vel_summary
|
| 823 |
+
```
|
| 824 |
+
|
| 825 |
+
- [ ] **Step 3: Add `overlay_layers` CheckboxGroup in `build_app()`**
|
| 826 |
+
|
| 827 |
+
After the `pose_model_dropdown` block (around line 270), and before `submit_btn`:
|
| 828 |
+
|
| 829 |
+
```python
|
| 830 |
+
overlay_layers = gr.CheckboxGroup(
|
| 831 |
+
choices=["Skeleton", "Trails", "Velocity arrows"],
|
| 832 |
+
value=["Skeleton", "Trails"],
|
| 833 |
+
label="Overlay Layers",
|
| 834 |
+
)
|
| 835 |
+
```
|
| 836 |
+
|
| 837 |
+
- [ ] **Step 4: Add overlay tab in the results panel**
|
| 838 |
+
|
| 839 |
+
Inside the `with gr.Tabs():` block (after the `β οΈ Alerts` tab):
|
| 840 |
+
|
| 841 |
+
```python
|
| 842 |
+
with gr.TabItem("π¬ Overlay Video"):
|
| 843 |
+
overlay_video = gr.Video(label="Annotated Movement")
|
| 844 |
+
velocity_md = gr.Markdown("")
|
| 845 |
+
```
|
| 846 |
+
|
| 847 |
+
- [ ] **Step 5: Update `_map_inputs` and `submit_btn.click`**
|
| 848 |
+
|
| 849 |
+
Replace the `_map_inputs` closure and `submit_btn.click` call:
|
| 850 |
+
|
| 851 |
+
```python
|
| 852 |
+
def _map_inputs(video, test_display_name, side_display, pose_model_key, overlay_layers):
|
| 853 |
+
"""Map UI display values to internal values."""
|
| 854 |
+
test_map = {name: val for name, val in FMS_TESTS}
|
| 855 |
+
test_name = test_map.get(test_display_name, "deep_squat")
|
| 856 |
+
side = {"N/A": "na", "Left": "left", "Right": "right"}.get(side_display, "na")
|
| 857 |
+
return process_video(video, test_name, side, pose_model_key, overlay_layers)
|
| 858 |
+
|
| 859 |
+
submit_btn.click(
|
| 860 |
+
fn=_map_inputs,
|
| 861 |
+
inputs=[video_input, test_dropdown, side_dropdown, pose_model_dropdown, overlay_layers],
|
| 862 |
+
outputs=[score_html, pipeline_md, score_details, alerts_md, overlay_video, velocity_md],
|
| 863 |
+
)
|
| 864 |
+
```
|
| 865 |
+
|
| 866 |
+
- [ ] **Step 6: Smoke-test the app builds**
|
| 867 |
+
|
| 868 |
+
```bash
|
| 869 |
+
python3 -c "from app import build_app; build_app(); print('ok')"
|
| 870 |
+
```
|
| 871 |
+
|
| 872 |
+
Expected: `ok` (Gradio UserWarning about theme is fine, not an error)
|
| 873 |
+
|
| 874 |
+
- [ ] **Step 7: Run full test suite to check for regressions**
|
| 875 |
+
|
| 876 |
+
```bash
|
| 877 |
+
pytest tests/ -v --tb=short 2>&1 | tail -15
|
| 878 |
+
```
|
| 879 |
+
|
| 880 |
+
Expected: all previous tests still pass (62 passing, 1 pre-existing fail in biomechanics), plus 16 new visualizer tests = 78 passing.
|
| 881 |
+
|
| 882 |
+
- [ ] **Step 8: Commit**
|
| 883 |
+
|
| 884 |
+
```bash
|
| 885 |
+
git add app.py
|
| 886 |
+
git commit -m "feat: overlay video tab + velocity summary wired in Gradio UI"
|
| 887 |
+
```
|
| 888 |
+
|
| 889 |
+
---
|
| 890 |
+
|
| 891 |
+
## Self-review
|
| 892 |
+
|
| 893 |
+
**Spec coverage:**
|
| 894 |
+
- β
`SimpleKalmanFilter` 4-state (Task 1)
|
| 895 |
+
- β
`compute_joint_velocity` Kalman-filtered px/s (Task 1)
|
| 896 |
+
- β
`_draw_skeleton` COCO bones, confidence-colored joints (Task 2)
|
| 897 |
+
- β
`_draw_trails` fading deque-based trails (Task 3)
|
| 898 |
+
- β
`_draw_velocity_arrows` speed-colored, direction from consecutive frames (Task 4)
|
| 899 |
+
- β
`render_video` layer dispatch, trail history, VideoWriter (Task 5)
|
| 900 |
+
- β
`build_velocity_summary` markdown table, >50% detection filter (Task 5)
|
| 901 |
+
- β
`overlay_layers` CheckboxGroup in UI (Task 6)
|
| 902 |
+
- β
New `π¬ Overlay Video` tab with `gr.Video` + `gr.Markdown` (Task 6)
|
| 903 |
+
- β
`process_video` wired with layers param (Task 6)
|
| 904 |
+
- β
`vis.last_velocities` stored on instance after `render_video` (Task 5)
|
| 905 |
+
- β
Error handling: empty layers β None, empty detections β None, exception β alerts (Task 5 + 6)
|
| 906 |
+
- β
All 5 spec test cases covered across Tasks 1β5
|
| 907 |
+
|
| 908 |
+
**Placeholder scan:** None found. All code blocks are complete.
|
| 909 |
+
|
| 910 |
+
**Type consistency:**
|
| 911 |
+
- `compute_joint_velocity` returns `dict[int, list[float]]` β used identically in `render_video`, `_draw_velocity_arrows`, and `build_velocity_summary`. β
|
| 912 |
+
- `layers: set[str]` in `render_video`; converted from `list[str]` in `process_video` via set comprehension. β
|
| 913 |
+
- `vis.last_velocities` set in `render_video`, read in `process_video`. β
|
| 914 |
+
- `_draw_velocity_arrows(frame, kps, prev_kps, velocities, frame_idx)` β signature matches call in `render_video`. β
|
docs/superpowers/plans/2026-06-13-full-fms-session-pdf.md
ADDED
|
@@ -0,0 +1,1209 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Full FMS Session + PDF Report β Implementation Plan
|
| 2 |
+
|
| 3 |
+
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
| 4 |
+
|
| 5 |
+
**Goal:** Turn FormScout's one-clip scorer into a screening session that accumulates analyzed clips into a composite 0β21 report and exports a branded PDF with annotated worst-moment key-frame stills.
|
| 6 |
+
|
| 7 |
+
**Architecture:** A new `formscout/session.py` accumulates typed `SessionEntry` objects (one per analyzed clip), persisting each to a temp session dir. `PoseVisualizer.render_frame()` captures the governing frame (already computed by `BiomechanicsAgent` and stored in `features.timing`) as an annotated PNG. On "Finish", the existing `ReportAgent` computes composite + asymmetries, and a new `PdfReportAgent` renders a ReportLab PDF. The UI (`app.py`) gains `gr.State` session accumulation with "Analyse new clip" / "Finish & generate PDF" buttons.
|
| 8 |
+
|
| 9 |
+
**Tech Stack:** Python 3.13, ReportLab (new dep), OpenCV (existing), Gradio 5, pytest. No model downloads in tests.
|
| 10 |
+
|
| 11 |
+
---
|
| 12 |
+
|
| 13 |
+
## File Structure
|
| 14 |
+
|
| 15 |
+
- `requirements.txt` β add `reportlab`.
|
| 16 |
+
- `formscout/types.py` β add `SessionEntry` frozen dataclass.
|
| 17 |
+
- `formscout/agents/biomechanics.py` β add `max_sag_frame` to `trunk_stability_pushup` timing (rotary already has `peak_extension_frame`).
|
| 18 |
+
- `formscout/agents/visualizer.py` β add `PoseVisualizer.render_frame()`.
|
| 19 |
+
- `formscout/session.py` β **new**: session accumulator (new/add/finish + persistence + key-frame helpers).
|
| 20 |
+
- `formscout/agents/pdf_report.py` β **new**: `PdfReportAgent` (ReportLab).
|
| 21 |
+
- `app.py` β wire `gr.State`, two buttons, "Session so far" table, finish handler.
|
| 22 |
+
- `tests/test_session.py`, `tests/test_keyframe.py`, `tests/test_pdf_report.py` β **new**.
|
| 23 |
+
|
| 24 |
+
---
|
| 25 |
+
|
| 26 |
+
## Task 1: Add ReportLab dependency
|
| 27 |
+
|
| 28 |
+
**Files:**
|
| 29 |
+
- Modify: `requirements.txt`
|
| 30 |
+
|
| 31 |
+
- [ ] **Step 1: Add the dependency**
|
| 32 |
+
|
| 33 |
+
Add this line to `requirements.txt` (after `pillow>=10.3`):
|
| 34 |
+
|
| 35 |
+
```
|
| 36 |
+
reportlab>=4.0
|
| 37 |
+
```
|
| 38 |
+
|
| 39 |
+
- [ ] **Step 2: Install it**
|
| 40 |
+
|
| 41 |
+
Run: `pip install 'reportlab>=4.0'`
|
| 42 |
+
Expected: `Successfully installed reportlab-4.x.x`
|
| 43 |
+
|
| 44 |
+
- [ ] **Step 3: Verify import**
|
| 45 |
+
|
| 46 |
+
Run: `python3 -c "import reportlab; print(reportlab.Version)"`
|
| 47 |
+
Expected: prints a version like `4.x.x`
|
| 48 |
+
|
| 49 |
+
- [ ] **Step 4: Commit**
|
| 50 |
+
|
| 51 |
+
```bash
|
| 52 |
+
git add requirements.txt
|
| 53 |
+
git commit -m "build: add reportlab for PDF report generation"
|
| 54 |
+
```
|
| 55 |
+
|
| 56 |
+
---
|
| 57 |
+
|
| 58 |
+
## Task 2: Add `SessionEntry` dataclass
|
| 59 |
+
|
| 60 |
+
**Files:**
|
| 61 |
+
- Modify: `formscout/types.py` (after `ReportResult`, before `PipelineState`)
|
| 62 |
+
- Test: `tests/test_session.py`
|
| 63 |
+
|
| 64 |
+
- [ ] **Step 1: Write the failing test**
|
| 65 |
+
|
| 66 |
+
Create `tests/test_session.py` with:
|
| 67 |
+
|
| 68 |
+
```python
|
| 69 |
+
"""Tests for the FMS session accumulator β no GPU, no model downloads."""
|
| 70 |
+
import numpy as np
|
| 71 |
+
|
| 72 |
+
from formscout.types import (
|
| 73 |
+
IngestResult, Pose2DResult, BiomechFeatures, ScoreResult, JudgeResult,
|
| 74 |
+
MovementResult, SessionEntry,
|
| 75 |
+
)
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def test_session_entry_holds_typed_objects():
|
| 79 |
+
movement = MovementResult(test_name="deep_squat", side="na", confidence=1.0)
|
| 80 |
+
features = BiomechFeatures(
|
| 81 |
+
test_name="deep_squat", view="2d", side="na",
|
| 82 |
+
angles={"left_knee_flexion_deg": 95.0}, alignments={"knees_tracking_over_feet": True},
|
| 83 |
+
symmetry_delta=None, timing={"deepest_frame": 2}, confidence=0.9,
|
| 84 |
+
)
|
| 85 |
+
rubric = ScoreResult(score=2, rationale="ok", confidence=0.8)
|
| 86 |
+
judge = JudgeResult(score=2, rationale="ok", compensation_tags=["heels elevated"],
|
| 87 |
+
corrective_hint="ankle mobility", confidence=0.85)
|
| 88 |
+
entry = SessionEntry(
|
| 89 |
+
test_name="deep_squat", side="na", score=2, needs_human=False,
|
| 90 |
+
rationale="ok", compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
|
| 91 |
+
measurements={"left_knee_flexion_deg": 95.0}, confidence=0.85, view="2d",
|
| 92 |
+
keyframe_path=None, movement=movement, features=features,
|
| 93 |
+
rubric_score=rubric, judge=judge,
|
| 94 |
+
)
|
| 95 |
+
assert entry.score == 2
|
| 96 |
+
assert entry.movement.test_name == "deep_squat"
|
| 97 |
+
assert entry.rubric_score.score == 2
|
| 98 |
+
assert entry.judge.compensation_tags == ["heels elevated"]
|
| 99 |
+
```
|
| 100 |
+
|
| 101 |
+
- [ ] **Step 2: Run test to verify it fails**
|
| 102 |
+
|
| 103 |
+
Run: `pytest tests/test_session.py::test_session_entry_holds_typed_objects -v`
|
| 104 |
+
Expected: FAIL with `ImportError: cannot import name 'SessionEntry'`
|
| 105 |
+
|
| 106 |
+
- [ ] **Step 3: Add the dataclass**
|
| 107 |
+
|
| 108 |
+
In `formscout/types.py`, insert after the `ReportResult` class (line ~142) and before `PipelineState`:
|
| 109 |
+
|
| 110 |
+
```python
|
| 111 |
+
@dataclass(frozen=True)
|
| 112 |
+
class SessionEntry:
|
| 113 |
+
"""One accumulated analysis in a screening session.
|
| 114 |
+
|
| 115 |
+
Display fields (test_nameβ¦keyframe_path) feed the PDF/JSON/MD artifacts;
|
| 116 |
+
the trailing typed objects (movementβ¦judge) feed ReportAgent.run().
|
| 117 |
+
"""
|
| 118 |
+
test_name: str
|
| 119 |
+
side: str
|
| 120 |
+
score: int | None
|
| 121 |
+
needs_human: bool
|
| 122 |
+
rationale: str
|
| 123 |
+
compensation_tags: list
|
| 124 |
+
corrective_hint: str
|
| 125 |
+
measurements: dict
|
| 126 |
+
confidence: float
|
| 127 |
+
view: str
|
| 128 |
+
keyframe_path: str | None
|
| 129 |
+
movement: MovementResult
|
| 130 |
+
features: BiomechFeatures
|
| 131 |
+
rubric_score: ScoreResult
|
| 132 |
+
judge: JudgeResult | None
|
| 133 |
+
```
|
| 134 |
+
|
| 135 |
+
- [ ] **Step 4: Run test to verify it passes**
|
| 136 |
+
|
| 137 |
+
Run: `pytest tests/test_session.py::test_session_entry_holds_typed_objects -v`
|
| 138 |
+
Expected: PASS
|
| 139 |
+
|
| 140 |
+
- [ ] **Step 5: Commit**
|
| 141 |
+
|
| 142 |
+
```bash
|
| 143 |
+
git add formscout/types.py tests/test_session.py
|
| 144 |
+
git commit -m "feat: add SessionEntry typed contract for screening sessions"
|
| 145 |
+
```
|
| 146 |
+
|
| 147 |
+
---
|
| 148 |
+
|
| 149 |
+
## Task 3: Add governing-frame index to push-up biomechanics
|
| 150 |
+
|
| 151 |
+
**Files:**
|
| 152 |
+
- Modify: `formscout/agents/biomechanics.py:468-529` (`_trunk_stability_pushup`)
|
| 153 |
+
- Test: `tests/test_biomechanics.py` (append a test)
|
| 154 |
+
|
| 155 |
+
The other six tests already store a governing frame index in `features.timing`
|
| 156 |
+
(`deepest_frame`, `peak_step_frame`, `deepest_lunge_frame`, `measure_frame`,
|
| 157 |
+
`peak_raise_frame`, `peak_extension_frame`). Only `trunk_stability_pushup` is missing one.
|
| 158 |
+
|
| 159 |
+
- [ ] **Step 1: Write the failing test**
|
| 160 |
+
|
| 161 |
+
Append to `tests/test_biomechanics.py`:
|
| 162 |
+
|
| 163 |
+
```python
|
| 164 |
+
def test_pushup_timing_has_max_sag_frame():
|
| 165 |
+
from formscout.agents.biomechanics import BiomechanicsAgent
|
| 166 |
+
from formscout.types import Pose2DResult, Body3DResult, MovementResult
|
| 167 |
+
|
| 168 |
+
# 4 frames; frame 2 has the largest hip sag (hip far below shoulder/ankle midline)
|
| 169 |
+
def kps(hip_y):
|
| 170 |
+
base = {
|
| 171 |
+
5: {"x": 200, "y": 200, "conf": 0.9}, # L shoulder
|
| 172 |
+
6: {"x": 220, "y": 200, "conf": 0.9}, # R shoulder
|
| 173 |
+
11: {"x": 300, "y": hip_y, "conf": 0.9}, # L hip
|
| 174 |
+
12: {"x": 320, "y": hip_y, "conf": 0.9}, # R hip
|
| 175 |
+
15: {"x": 400, "y": 200, "conf": 0.9}, # L ankle
|
| 176 |
+
16: {"x": 420, "y": 200, "conf": 0.9}, # R ankle
|
| 177 |
+
}
|
| 178 |
+
return base
|
| 179 |
+
|
| 180 |
+
frames = [kps(200), kps(210), kps(260), kps(205)]
|
| 181 |
+
pose = Pose2DResult(keypoints=frames, fps=30.0, confidence=0.9)
|
| 182 |
+
body3d = Body3DResult(used=False, joints_3d=[])
|
| 183 |
+
movement = MovementResult(test_name="trunk_stability_pushup", side="na", confidence=1.0)
|
| 184 |
+
|
| 185 |
+
feats = BiomechanicsAgent().run(pose, body3d, movement)
|
| 186 |
+
assert "max_sag_frame" in feats.timing
|
| 187 |
+
assert feats.timing["max_sag_frame"] == 2
|
| 188 |
+
```
|
| 189 |
+
|
| 190 |
+
- [ ] **Step 2: Run test to verify it fails**
|
| 191 |
+
|
| 192 |
+
Run: `pytest tests/test_biomechanics.py::test_pushup_timing_has_max_sag_frame -v`
|
| 193 |
+
Expected: FAIL with `assert 'max_sag_frame' in {...}` (KeyError-style assertion failure)
|
| 194 |
+
|
| 195 |
+
- [ ] **Step 3: Track the max-sag frame index**
|
| 196 |
+
|
| 197 |
+
In `formscout/agents/biomechanics.py`, replace the body of `_trunk_stability_pushup` from the
|
| 198 |
+
`trunk_angles_over_time = []` loop through the `if trunk_angles_over_time:` block. Replace:
|
| 199 |
+
|
| 200 |
+
```python
|
| 201 |
+
# Analyze multiple frames to detect sag/lag
|
| 202 |
+
trunk_angles_over_time = []
|
| 203 |
+
for i, kps in enumerate(pose2d.keypoints):
|
| 204 |
+
```
|
| 205 |
+
|
| 206 |
+
β¦down to and including the `alignments["no_sag"] = max_sag < 30` line, with:
|
| 207 |
+
|
| 208 |
+
```python
|
| 209 |
+
# Analyze multiple frames to detect sag/lag
|
| 210 |
+
trunk_sags: list[tuple[int, float]] = [] # (frame_idx, sag_px)
|
| 211 |
+
for i, kps in enumerate(pose2d.keypoints):
|
| 212 |
+
l_sh = _get_joint(kps, L_SHOULDER)
|
| 213 |
+
r_sh = _get_joint(kps, R_SHOULDER)
|
| 214 |
+
l_hip = _get_joint(kps, L_HIP)
|
| 215 |
+
r_hip = _get_joint(kps, R_HIP)
|
| 216 |
+
l_ankle = _get_joint(kps, L_ANKLE)
|
| 217 |
+
r_ankle = _get_joint(kps, R_ANKLE)
|
| 218 |
+
|
| 219 |
+
if l_sh and r_sh and l_hip and r_hip and l_ankle and r_ankle:
|
| 220 |
+
sh_y = (l_sh[1] + r_sh[1]) / 2
|
| 221 |
+
hip_y = (l_hip[1] + r_hip[1]) / 2
|
| 222 |
+
ankle_y = (l_ankle[1] + r_ankle[1]) / 2
|
| 223 |
+
expected_hip_y = (sh_y + ankle_y) / 2
|
| 224 |
+
sag_px = hip_y - expected_hip_y
|
| 225 |
+
trunk_sags.append((i, sag_px))
|
| 226 |
+
|
| 227 |
+
max_sag_frame = 0
|
| 228 |
+
if trunk_sags:
|
| 229 |
+
sags = [s for _, s in trunk_sags]
|
| 230 |
+
max_sag_frame = max(trunk_sags, key=lambda t: t[1])[0]
|
| 231 |
+
mean = sum(sags) / len(sags)
|
| 232 |
+
variance = (sum((x - mean) ** 2 for x in sags) / len(sags)) ** 0.5
|
| 233 |
+
max_sag = max(sags)
|
| 234 |
+
angles["max_sag_px"] = max_sag
|
| 235 |
+
angles["trunk_variance_px"] = variance
|
| 236 |
+
alignments["body_rigid"] = max_sag < 30 and variance < 15
|
| 237 |
+
alignments["no_sag"] = max_sag < 30
|
| 238 |
+
else:
|
| 239 |
+
notes_parts.append("insufficient landmarks for trunk analysis")
|
| 240 |
+
```
|
| 241 |
+
|
| 242 |
+
Then update the `return BiomechFeatures(...)` `timing=` argument at the end of the method from:
|
| 243 |
+
|
| 244 |
+
```python
|
| 245 |
+
timing={"n_frames_analyzed": len(trunk_angles_over_time)},
|
| 246 |
+
```
|
| 247 |
+
|
| 248 |
+
to:
|
| 249 |
+
|
| 250 |
+
```python
|
| 251 |
+
timing={"n_frames_analyzed": len(trunk_sags), "max_sag_frame": max_sag_frame},
|
| 252 |
+
```
|
| 253 |
+
|
| 254 |
+
- [ ] **Step 4: Run test to verify it passes**
|
| 255 |
+
|
| 256 |
+
Run: `pytest tests/test_biomechanics.py::test_pushup_timing_has_max_sag_frame -v`
|
| 257 |
+
Expected: PASS
|
| 258 |
+
|
| 259 |
+
- [ ] **Step 5: Run the full biomechanics suite (no regressions)**
|
| 260 |
+
|
| 261 |
+
Run: `pytest tests/test_biomechanics.py -v`
|
| 262 |
+
Expected: all previously-passing tests still pass (the pre-existing `test_unimplemented_test_returns_low_confidence` known-failure may remain failing β that is unrelated and documented in CLAUDE.md).
|
| 263 |
+
|
| 264 |
+
- [ ] **Step 6: Commit**
|
| 265 |
+
|
| 266 |
+
```bash
|
| 267 |
+
git add formscout/agents/biomechanics.py tests/test_biomechanics.py
|
| 268 |
+
git commit -m "feat: track max-sag frame index in push-up biomechanics for key-frame capture"
|
| 269 |
+
```
|
| 270 |
+
|
| 271 |
+
---
|
| 272 |
+
|
| 273 |
+
## Task 4: Add `PoseVisualizer.render_frame()`
|
| 274 |
+
|
| 275 |
+
**Files:**
|
| 276 |
+
- Modify: `formscout/agents/visualizer.py` (add method to `PoseVisualizer`, after `render_video`)
|
| 277 |
+
- Test: `tests/test_keyframe.py`
|
| 278 |
+
|
| 279 |
+
- [ ] **Step 1: Write the failing test**
|
| 280 |
+
|
| 281 |
+
Create `tests/test_keyframe.py`:
|
| 282 |
+
|
| 283 |
+
```python
|
| 284 |
+
"""Tests for PoseVisualizer.render_frame β single annotated still."""
|
| 285 |
+
import os
|
| 286 |
+
import numpy as np
|
| 287 |
+
|
| 288 |
+
from formscout.types import IngestResult, Pose2DResult
|
| 289 |
+
|
| 290 |
+
|
| 291 |
+
def _ingest(n=5, h=480, w=640):
|
| 292 |
+
frames = [np.zeros((h, w, 3), dtype=np.uint8) for _ in range(n)]
|
| 293 |
+
return IngestResult(frames=frames, fps=30.0, duration=n / 30.0, n_people=1, width=w, height=h)
|
| 294 |
+
|
| 295 |
+
|
| 296 |
+
def _pose(n=5):
|
| 297 |
+
kps = []
|
| 298 |
+
for i in range(n):
|
| 299 |
+
kps.append({j: {"x": float(50 + j * 25), "y": float(80 + j * 18), "conf": 0.9}
|
| 300 |
+
for j in range(17)})
|
| 301 |
+
return Pose2DResult(keypoints=kps, fps=30.0, confidence=0.9)
|
| 302 |
+
|
| 303 |
+
|
| 304 |
+
def test_render_frame_writes_png(tmp_path):
|
| 305 |
+
from formscout.agents.visualizer import PoseVisualizer
|
| 306 |
+
out = str(tmp_path / "key.png")
|
| 307 |
+
path = PoseVisualizer().render_frame(_ingest(), _pose(), frame_idx=2,
|
| 308 |
+
layers={"skeleton"}, caption="Deep Squat β heels elevated",
|
| 309 |
+
out_png=out)
|
| 310 |
+
assert path == out
|
| 311 |
+
assert os.path.exists(out)
|
| 312 |
+
assert os.path.getsize(out) > 0
|
| 313 |
+
|
| 314 |
+
|
| 315 |
+
def test_render_frame_bad_index_returns_none(tmp_path):
|
| 316 |
+
from formscout.agents.visualizer import PoseVisualizer
|
| 317 |
+
out = str(tmp_path / "key.png")
|
| 318 |
+
path = PoseVisualizer().render_frame(_ingest(n=3), _pose(n=3), frame_idx=99,
|
| 319 |
+
layers={"skeleton"}, caption="", out_png=out)
|
| 320 |
+
assert path is None
|
| 321 |
+
```
|
| 322 |
+
|
| 323 |
+
- [ ] **Step 2: Run test to verify it fails**
|
| 324 |
+
|
| 325 |
+
Run: `pytest tests/test_keyframe.py -v`
|
| 326 |
+
Expected: FAIL with `AttributeError: 'PoseVisualizer' object has no attribute 'render_frame'`
|
| 327 |
+
|
| 328 |
+
- [ ] **Step 3: Add the method**
|
| 329 |
+
|
| 330 |
+
In `formscout/agents/visualizer.py`, inside the `PoseVisualizer` class, add this method
|
| 331 |
+
immediately after `render_video` (before the closing of the class / the module-level
|
| 332 |
+
`build_velocity_summary`):
|
| 333 |
+
|
| 334 |
+
```python
|
| 335 |
+
def render_frame(
|
| 336 |
+
self,
|
| 337 |
+
ingest,
|
| 338 |
+
pose2d,
|
| 339 |
+
frame_idx: int,
|
| 340 |
+
layers: set[str],
|
| 341 |
+
caption: str = "",
|
| 342 |
+
out_png: str | None = None,
|
| 343 |
+
) -> str | None:
|
| 344 |
+
"""Render a single annotated still (skeleton + optional trails + caption).
|
| 345 |
+
|
| 346 |
+
frame_idx is typically the governing frame from BiomechFeatures.timing.
|
| 347 |
+
Returns the PNG path on success, None on any failure. Never raises.
|
| 348 |
+
"""
|
| 349 |
+
try:
|
| 350 |
+
if not (0 <= frame_idx < len(ingest.frames)) or frame_idx >= len(pose2d.keypoints):
|
| 351 |
+
return None
|
| 352 |
+
|
| 353 |
+
frame = ingest.frames[frame_idx].copy()
|
| 354 |
+
kps = pose2d.keypoints[frame_idx]
|
| 355 |
+
|
| 356 |
+
if "trails" in layers:
|
| 357 |
+
trail: dict[int, deque] = {j: deque(maxlen=TRAIL_LENGTH) for j in range(17)}
|
| 358 |
+
start = max(0, frame_idx - TRAIL_LENGTH)
|
| 359 |
+
for fi in range(start, frame_idx + 1):
|
| 360 |
+
for j, kp in pose2d.keypoints[fi].items():
|
| 361 |
+
if kp.get("conf", 0.0) >= CONF_THRESHOLD:
|
| 362 |
+
trail[j].append((kp["x"], kp["y"]))
|
| 363 |
+
frame = self._draw_trails(frame, trail)
|
| 364 |
+
|
| 365 |
+
if "skeleton" in layers:
|
| 366 |
+
frame = self._draw_skeleton(frame, kps)
|
| 367 |
+
|
| 368 |
+
if caption:
|
| 369 |
+
cv2.rectangle(frame, (0, 0), (frame.shape[1], 28), (0, 0, 0), -1)
|
| 370 |
+
cv2.putText(frame, caption[:80], (8, 20), cv2.FONT_HERSHEY_SIMPLEX,
|
| 371 |
+
0.55, (255, 255, 255), 1, cv2.LINE_AA)
|
| 372 |
+
|
| 373 |
+
if out_png is None:
|
| 374 |
+
out_png = tempfile.NamedTemporaryFile(suffix=".png", delete=False).name
|
| 375 |
+
|
| 376 |
+
ok = cv2.imwrite(out_png, frame)
|
| 377 |
+
return out_png if ok else None
|
| 378 |
+
except Exception as e:
|
| 379 |
+
logger.warning("render_frame failed: %s", e)
|
| 380 |
+
return None
|
| 381 |
+
```
|
| 382 |
+
|
| 383 |
+
(`deque`, `cv2`, `tempfile`, `logger`, `TRAIL_LENGTH`, `CONF_THRESHOLD` are all already imported at the top of this file.)
|
| 384 |
+
|
| 385 |
+
- [ ] **Step 4: Run test to verify it passes**
|
| 386 |
+
|
| 387 |
+
Run: `pytest tests/test_keyframe.py -v`
|
| 388 |
+
Expected: both tests PASS
|
| 389 |
+
|
| 390 |
+
- [ ] **Step 5: Commit**
|
| 391 |
+
|
| 392 |
+
```bash
|
| 393 |
+
git add formscout/agents/visualizer.py tests/test_keyframe.py
|
| 394 |
+
git commit -m "feat: add PoseVisualizer.render_frame for annotated key-frame stills"
|
| 395 |
+
```
|
| 396 |
+
|
| 397 |
+
---
|
| 398 |
+
|
| 399 |
+
## Task 5: Create the session accumulator
|
| 400 |
+
|
| 401 |
+
**Files:**
|
| 402 |
+
- Create: `formscout/session.py`
|
| 403 |
+
- Test: `tests/test_session.py` (append tests)
|
| 404 |
+
|
| 405 |
+
- [ ] **Step 1: Write the failing tests**
|
| 406 |
+
|
| 407 |
+
Append to `tests/test_session.py`:
|
| 408 |
+
|
| 409 |
+
```python
|
| 410 |
+
def _ingest(n=5, h=480, w=640):
|
| 411 |
+
frames = [np.zeros((h, w, 3), dtype=np.uint8) for _ in range(n)]
|
| 412 |
+
return IngestResult(frames=frames, fps=30.0, duration=n / 30.0, n_people=1, width=w, height=h)
|
| 413 |
+
|
| 414 |
+
|
| 415 |
+
def _pose(n=5):
|
| 416 |
+
kps = []
|
| 417 |
+
for i in range(n):
|
| 418 |
+
kps.append({j: {"x": float(50 + j * 25), "y": float(80 + j * 18), "conf": 0.9}
|
| 419 |
+
for j in range(17)})
|
| 420 |
+
return Pose2DResult(keypoints=kps, fps=30.0, confidence=0.9)
|
| 421 |
+
|
| 422 |
+
|
| 423 |
+
def _features(test_name="deep_squat", side="na", frame_key="deepest_frame"):
|
| 424 |
+
return BiomechFeatures(
|
| 425 |
+
test_name=test_name, view="2d", side=side,
|
| 426 |
+
angles={"left_knee_flexion_deg": 95.0},
|
| 427 |
+
alignments={"knees_tracking_over_feet": False},
|
| 428 |
+
symmetry_delta=None, timing={frame_key: 2}, confidence=0.9,
|
| 429 |
+
)
|
| 430 |
+
|
| 431 |
+
|
| 432 |
+
def _judge(score=2, needs_human=False):
|
| 433 |
+
return JudgeResult(
|
| 434 |
+
score=None if needs_human else score, rationale="r",
|
| 435 |
+
compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
|
| 436 |
+
confidence=0.85, needs_human=needs_human,
|
| 437 |
+
)
|
| 438 |
+
|
| 439 |
+
|
| 440 |
+
def test_add_analysis_appends_entry_and_writes_files():
|
| 441 |
+
import os
|
| 442 |
+
from formscout import session as S
|
| 443 |
+
sess = S.new_session()
|
| 444 |
+
entry = S.add_analysis(sess, ingest=_ingest(), pose2d=_pose(),
|
| 445 |
+
features=_features(), judge=_judge(), test_name="deep_squat", side="na")
|
| 446 |
+
assert len(sess.entries) == 1
|
| 447 |
+
assert entry.score == 2
|
| 448 |
+
assert os.path.exists(os.path.join(sess.session_dir, "session.json"))
|
| 449 |
+
assert os.path.exists(os.path.join(sess.session_dir, "analysis.md"))
|
| 450 |
+
# key-frame still written (deepest_frame=2 is valid)
|
| 451 |
+
assert entry.keyframe_path and os.path.exists(entry.keyframe_path)
|
| 452 |
+
|
| 453 |
+
|
| 454 |
+
def test_finish_composite_null_when_needs_human():
|
| 455 |
+
from formscout import session as S
|
| 456 |
+
sess = S.new_session()
|
| 457 |
+
S.add_analysis(sess, ingest=_ingest(), pose2d=_pose(), features=_features(),
|
| 458 |
+
judge=_judge(score=3), test_name="deep_squat", side="na")
|
| 459 |
+
S.add_analysis(sess, ingest=_ingest(), pose2d=_pose(),
|
| 460 |
+
features=_features("trunk_stability_pushup", frame_key="max_sag_frame"),
|
| 461 |
+
judge=_judge(needs_human=True), test_name="trunk_stability_pushup", side="na")
|
| 462 |
+
report, pdf_path = S.finish_session(sess)
|
| 463 |
+
assert report is not None
|
| 464 |
+
assert report.composite is None # one test needs_human
|
| 465 |
+
|
| 466 |
+
|
| 467 |
+
def test_finish_empty_session_returns_none():
|
| 468 |
+
from formscout import session as S
|
| 469 |
+
sess = S.new_session()
|
| 470 |
+
report, pdf_path = S.finish_session(sess)
|
| 471 |
+
assert report is None and pdf_path is None
|
| 472 |
+
```
|
| 473 |
+
|
| 474 |
+
- [ ] **Step 2: Run tests to verify they fail**
|
| 475 |
+
|
| 476 |
+
Run: `pytest tests/test_session.py -v`
|
| 477 |
+
Expected: the three new tests FAIL with `ModuleNotFoundError: No module named 'formscout.session'`
|
| 478 |
+
|
| 479 |
+
- [ ] **Step 3: Create the module**
|
| 480 |
+
|
| 481 |
+
Create `formscout/session.py`:
|
| 482 |
+
|
| 483 |
+
```python
|
| 484 |
+
"""
|
| 485 |
+
Screening-session accumulator.
|
| 486 |
+
|
| 487 |
+
Accumulates one SessionEntry per analyzed clip, persists each to a temp session
|
| 488 |
+
dir (session.json + analysis.md + key-frame PNGs), and on finish builds a
|
| 489 |
+
ReportResult (via ReportAgent) + a PDF (via PdfReportAgent).
|
| 490 |
+
|
| 491 |
+
Pure orchestration β no Gradio imports. Disk writes tolerate failure with a
|
| 492 |
+
logged warning and never block scoring.
|
| 493 |
+
"""
|
| 494 |
+
from __future__ import annotations
|
| 495 |
+
|
| 496 |
+
import json
|
| 497 |
+
import logging
|
| 498 |
+
import os
|
| 499 |
+
import tempfile
|
| 500 |
+
import uuid
|
| 501 |
+
from dataclasses import dataclass, replace
|
| 502 |
+
|
| 503 |
+
from formscout.rubric import score_test
|
| 504 |
+
from formscout.types import MovementResult, ReportResult, SessionEntry
|
| 505 |
+
|
| 506 |
+
logger = logging.getLogger(__name__)
|
| 507 |
+
|
| 508 |
+
# Maps each test to the BiomechFeatures.timing key holding its governing frame.
|
| 509 |
+
TIMING_KEY = {
|
| 510 |
+
"deep_squat": "deepest_frame",
|
| 511 |
+
"hurdle_step": "peak_step_frame",
|
| 512 |
+
"inline_lunge": "deepest_lunge_frame",
|
| 513 |
+
"shoulder_mobility": "measure_frame",
|
| 514 |
+
"active_slr": "peak_raise_frame",
|
| 515 |
+
"trunk_stability_pushup": "max_sag_frame",
|
| 516 |
+
"rotary_stability": "peak_extension_frame",
|
| 517 |
+
}
|
| 518 |
+
|
| 519 |
+
|
| 520 |
+
@dataclass
|
| 521 |
+
class Session:
|
| 522 |
+
"""Mutable session: an id, its temp dir, and accumulated entries."""
|
| 523 |
+
session_id: str
|
| 524 |
+
session_dir: str
|
| 525 |
+
entries: list # list[SessionEntry]
|
| 526 |
+
|
| 527 |
+
|
| 528 |
+
def new_session() -> Session:
|
| 529 |
+
sid = uuid.uuid4().hex[:12]
|
| 530 |
+
base = os.path.join(tempfile.gettempdir(), "formscout_sessions", sid)
|
| 531 |
+
try:
|
| 532 |
+
os.makedirs(os.path.join(base, "keyframes"), exist_ok=True)
|
| 533 |
+
except Exception as e:
|
| 534 |
+
logger.warning("session dir create failed: %s", e)
|
| 535 |
+
return Session(session_id=sid, session_dir=base, entries=[])
|
| 536 |
+
|
| 537 |
+
|
| 538 |
+
def governing_frame_index(features) -> int | None:
|
| 539 |
+
"""Return the governing frame index for this test, or None."""
|
| 540 |
+
key = TIMING_KEY.get(features.test_name)
|
| 541 |
+
if key is None:
|
| 542 |
+
return None
|
| 543 |
+
idx = features.timing.get(key)
|
| 544 |
+
return int(idx) if isinstance(idx, (int, float)) else None
|
| 545 |
+
|
| 546 |
+
|
| 547 |
+
def worst_compensation_caption(judge, features) -> str:
|
| 548 |
+
"""Short caption naming the worst compensation for the key-frame still."""
|
| 549 |
+
if judge and getattr(judge, "compensation_tags", None):
|
| 550 |
+
return ", ".join(judge.compensation_tags)
|
| 551 |
+
failed = [k.replace("_", " ") for k, v in features.alignments.items() if v is False]
|
| 552 |
+
return ("compensation: " + ", ".join(failed)) if failed else "key position"
|
| 553 |
+
|
| 554 |
+
|
| 555 |
+
def add_analysis(session, *, ingest, pose2d, features, judge, test_name, side,
|
| 556 |
+
draw_trails: bool = False) -> SessionEntry:
|
| 557 |
+
"""Build a SessionEntry from a completed analysis, render its key-frame,
|
| 558 |
+
persist the session, append, and return the entry."""
|
| 559 |
+
movement = MovementResult(test_name=test_name, side=side, confidence=1.0)
|
| 560 |
+
rubric = score_test(features)
|
| 561 |
+
|
| 562 |
+
needs_human = bool((judge and judge.needs_human) or rubric.needs_human)
|
| 563 |
+
if needs_human:
|
| 564 |
+
score = None
|
| 565 |
+
elif judge and judge.score is not None:
|
| 566 |
+
score = judge.score
|
| 567 |
+
else:
|
| 568 |
+
score = rubric.score
|
| 569 |
+
|
| 570 |
+
keyframe_path = None
|
| 571 |
+
idx = governing_frame_index(features)
|
| 572 |
+
if idx is not None and 0 <= idx < len(pose2d.keypoints):
|
| 573 |
+
from formscout.agents.visualizer import PoseVisualizer
|
| 574 |
+
caption = (f"{test_name.replace('_', ' ').title()} "
|
| 575 |
+
f"({side}) β {worst_compensation_caption(judge, features)}")
|
| 576 |
+
layers = {"skeleton", "trails"} if draw_trails else {"skeleton"}
|
| 577 |
+
out_png = os.path.join(session.session_dir, "keyframes", f"{test_name}_{side}.png")
|
| 578 |
+
try:
|
| 579 |
+
keyframe_path = PoseVisualizer().render_frame(ingest, pose2d, idx, layers, caption, out_png)
|
| 580 |
+
except Exception as e:
|
| 581 |
+
logger.warning("keyframe render failed: %s", e)
|
| 582 |
+
|
| 583 |
+
measurements = {}
|
| 584 |
+
measurements.update(features.angles)
|
| 585 |
+
measurements.update(features.alignments)
|
| 586 |
+
|
| 587 |
+
entry = SessionEntry(
|
| 588 |
+
test_name=test_name, side=side, score=score, needs_human=needs_human,
|
| 589 |
+
rationale=(judge.rationale if judge else rubric.rationale),
|
| 590 |
+
compensation_tags=list(judge.compensation_tags) if judge else [],
|
| 591 |
+
corrective_hint=(judge.corrective_hint if judge else ""),
|
| 592 |
+
measurements=measurements,
|
| 593 |
+
confidence=(judge.confidence if judge else rubric.confidence),
|
| 594 |
+
view=features.view,
|
| 595 |
+
keyframe_path=keyframe_path,
|
| 596 |
+
movement=movement, features=features, rubric_score=rubric, judge=judge,
|
| 597 |
+
)
|
| 598 |
+
session.entries.append(entry)
|
| 599 |
+
_persist(session)
|
| 600 |
+
return entry
|
| 601 |
+
|
| 602 |
+
|
| 603 |
+
def finish_session(session) -> tuple[ReportResult | None, str | None]:
|
| 604 |
+
"""Build the composite report + PDF. Returns (report, pdf_path).
|
| 605 |
+
Returns (None, None) for an empty session."""
|
| 606 |
+
if not session.entries:
|
| 607 |
+
return None, None
|
| 608 |
+
|
| 609 |
+
from formscout.agents.report import ReportAgent
|
| 610 |
+
report_inputs = [{
|
| 611 |
+
"movement": e.movement, "features": e.features,
|
| 612 |
+
"rubric_score": e.rubric_score, "judge": e.judge, "side": e.side,
|
| 613 |
+
} for e in session.entries]
|
| 614 |
+
report = ReportAgent().run(report_inputs)
|
| 615 |
+
|
| 616 |
+
pdf_path = None
|
| 617 |
+
try:
|
| 618 |
+
from formscout.agents.pdf_report import PdfReportAgent
|
| 619 |
+
pdf_path = PdfReportAgent().run(report, session.entries, session.session_dir)
|
| 620 |
+
except Exception as e:
|
| 621 |
+
logger.warning("pdf generation failed: %s", e)
|
| 622 |
+
|
| 623 |
+
report = replace(report, pdf_path=pdf_path)
|
| 624 |
+
return report, pdf_path
|
| 625 |
+
|
| 626 |
+
|
| 627 |
+
# ββ Persistence βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 628 |
+
|
| 629 |
+
def _jsonable(d: dict) -> dict:
|
| 630 |
+
out = {}
|
| 631 |
+
for k, v in d.items():
|
| 632 |
+
if isinstance(v, float):
|
| 633 |
+
out[k] = round(v, 2)
|
| 634 |
+
elif isinstance(v, (int, str, bool)) or v is None:
|
| 635 |
+
out[k] = v
|
| 636 |
+
else:
|
| 637 |
+
out[k] = str(v)
|
| 638 |
+
return out
|
| 639 |
+
|
| 640 |
+
|
| 641 |
+
def _entry_display(e: SessionEntry) -> dict:
|
| 642 |
+
return {
|
| 643 |
+
"test_name": e.test_name, "side": e.side, "score": e.score,
|
| 644 |
+
"needs_human": e.needs_human, "rationale": e.rationale,
|
| 645 |
+
"compensation_tags": list(e.compensation_tags), "corrective_hint": e.corrective_hint,
|
| 646 |
+
"measurements": _jsonable(e.measurements), "confidence": round(e.confidence, 2),
|
| 647 |
+
"view": e.view, "keyframe_path": e.keyframe_path,
|
| 648 |
+
}
|
| 649 |
+
|
| 650 |
+
|
| 651 |
+
def _render_markdown(session: Session) -> str:
|
| 652 |
+
lines = ["# FormScout β Session Log", ""]
|
| 653 |
+
for e in session.entries:
|
| 654 |
+
title = e.test_name.replace("_", " ").title()
|
| 655 |
+
if e.side in ("left", "right"):
|
| 656 |
+
title += f" ({e.side})"
|
| 657 |
+
score = "Clinician review required" if e.needs_human else f"{e.score}/3"
|
| 658 |
+
lines.append(f"## {title} β {score}")
|
| 659 |
+
lines.append(e.rationale or "")
|
| 660 |
+
if e.compensation_tags:
|
| 661 |
+
lines.append(f"- Compensations: {', '.join(e.compensation_tags)}")
|
| 662 |
+
if e.corrective_hint:
|
| 663 |
+
lines.append(f"- Corrective: {e.corrective_hint}")
|
| 664 |
+
if e.keyframe_path:
|
| 665 |
+
lines.append(f"- Key frame: `{e.keyframe_path}`")
|
| 666 |
+
lines.append("")
|
| 667 |
+
return "\n".join(lines)
|
| 668 |
+
|
| 669 |
+
|
| 670 |
+
def _persist(session: Session) -> None:
|
| 671 |
+
try:
|
| 672 |
+
with open(os.path.join(session.session_dir, "session.json"), "w") as f:
|
| 673 |
+
json.dump([_entry_display(e) for e in session.entries], f, indent=2)
|
| 674 |
+
with open(os.path.join(session.session_dir, "analysis.md"), "w") as f:
|
| 675 |
+
f.write(_render_markdown(session))
|
| 676 |
+
except Exception as e:
|
| 677 |
+
logger.warning("session persist failed: %s", e)
|
| 678 |
+
```
|
| 679 |
+
|
| 680 |
+
- [ ] **Step 4: Run tests to verify they pass**
|
| 681 |
+
|
| 682 |
+
Run: `pytest tests/test_session.py -v`
|
| 683 |
+
Expected: all session tests PASS (Task 6 provides `PdfReportAgent`; `finish_session` tolerates its
|
| 684 |
+
absence via the try/except, so these pass now β `pdf_path` may be `None` until Task 6).
|
| 685 |
+
|
| 686 |
+
- [ ] **Step 5: Commit**
|
| 687 |
+
|
| 688 |
+
```bash
|
| 689 |
+
git add formscout/session.py tests/test_session.py
|
| 690 |
+
git commit -m "feat: add screening-session accumulator with key-frame capture and persistence"
|
| 691 |
+
```
|
| 692 |
+
|
| 693 |
+
---
|
| 694 |
+
|
| 695 |
+
## Task 6: Create `PdfReportAgent`
|
| 696 |
+
|
| 697 |
+
**Files:**
|
| 698 |
+
- Create: `formscout/agents/pdf_report.py`
|
| 699 |
+
- Test: `tests/test_pdf_report.py`
|
| 700 |
+
|
| 701 |
+
- [ ] **Step 1: Write the failing test**
|
| 702 |
+
|
| 703 |
+
Create `tests/test_pdf_report.py`:
|
| 704 |
+
|
| 705 |
+
```python
|
| 706 |
+
"""Tests for PdfReportAgent β no GPU, no model downloads."""
|
| 707 |
+
import os
|
| 708 |
+
|
| 709 |
+
from formscout.types import (
|
| 710 |
+
ReportResult, SessionEntry, MovementResult, BiomechFeatures, ScoreResult, JudgeResult,
|
| 711 |
+
)
|
| 712 |
+
|
| 713 |
+
|
| 714 |
+
def _entry(test_name="deep_squat", score=2, needs_human=False):
|
| 715 |
+
movement = MovementResult(test_name=test_name, side="na", confidence=1.0)
|
| 716 |
+
features = BiomechFeatures(
|
| 717 |
+
test_name=test_name, view="2d", side="na",
|
| 718 |
+
angles={"left_knee_flexion_deg": 95.0}, alignments={"knees_tracking_over_feet": False},
|
| 719 |
+
symmetry_delta=None, timing={"deepest_frame": 1}, confidence=0.9,
|
| 720 |
+
)
|
| 721 |
+
rubric = ScoreResult(score=2, rationale="rubric ok", confidence=0.8)
|
| 722 |
+
judge = JudgeResult(score=None if needs_human else score, rationale="judge rationale",
|
| 723 |
+
compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
|
| 724 |
+
confidence=0.85, needs_human=needs_human)
|
| 725 |
+
return SessionEntry(
|
| 726 |
+
test_name=test_name, side="na", score=None if needs_human else score,
|
| 727 |
+
needs_human=needs_human, rationale="judge rationale",
|
| 728 |
+
compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
|
| 729 |
+
measurements={"left_knee_flexion_deg": 95.0, "knees_tracking_over_feet": False},
|
| 730 |
+
confidence=0.85, view="2d", keyframe_path=None,
|
| 731 |
+
movement=movement, features=features, rubric_score=rubric, judge=judge,
|
| 732 |
+
)
|
| 733 |
+
|
| 734 |
+
|
| 735 |
+
def _report(composite=2):
|
| 736 |
+
return ReportResult(
|
| 737 |
+
per_test=[], composite=composite, asymmetries=[],
|
| 738 |
+
overlay_video_path=None, pdf_path=None,
|
| 739 |
+
low_confidence_flags=[], disagreement_flags=[],
|
| 740 |
+
)
|
| 741 |
+
|
| 742 |
+
|
| 743 |
+
def test_pdf_is_created(tmp_path):
|
| 744 |
+
from formscout.agents.pdf_report import PdfReportAgent
|
| 745 |
+
path = PdfReportAgent().run(_report(2), [_entry()], str(tmp_path))
|
| 746 |
+
assert path is not None
|
| 747 |
+
assert os.path.exists(path)
|
| 748 |
+
assert os.path.getsize(path) > 1000 # a real PDF, not an empty file
|
| 749 |
+
with open(path, "rb") as f:
|
| 750 |
+
assert f.read(5) == b"%PDF-"
|
| 751 |
+
|
| 752 |
+
|
| 753 |
+
def test_pdf_handles_incomplete_composite(tmp_path):
|
| 754 |
+
from formscout.agents.pdf_report import PdfReportAgent
|
| 755 |
+
path = PdfReportAgent().run(_report(None), [_entry(needs_human=True)], str(tmp_path))
|
| 756 |
+
assert path is not None and os.path.exists(path)
|
| 757 |
+
```
|
| 758 |
+
|
| 759 |
+
- [ ] **Step 2: Run test to verify it fails**
|
| 760 |
+
|
| 761 |
+
Run: `pytest tests/test_pdf_report.py -v`
|
| 762 |
+
Expected: FAIL with `ModuleNotFoundError: No module named 'formscout.agents.pdf_report'`
|
| 763 |
+
|
| 764 |
+
- [ ] **Step 3: Create the agent**
|
| 765 |
+
|
| 766 |
+
Create `formscout/agents/pdf_report.py`:
|
| 767 |
+
|
| 768 |
+
```python
|
| 769 |
+
"""
|
| 770 |
+
PdfReportAgent β renders a ReportResult + session entries to a branded PDF.
|
| 771 |
+
|
| 772 |
+
Input: ReportResult, list[SessionEntry], session_dir (str)
|
| 773 |
+
Output: path to the written PDF (str), or None on failure.
|
| 774 |
+
Failure: returns None, never raises.
|
| 775 |
+
Params: 0 (pure rendering β no model).
|
| 776 |
+
License: n/a.
|
| 777 |
+
Gated: no.
|
| 778 |
+
"""
|
| 779 |
+
from __future__ import annotations
|
| 780 |
+
|
| 781 |
+
import logging
|
| 782 |
+
import os
|
| 783 |
+
|
| 784 |
+
from formscout.types import ReportResult
|
| 785 |
+
|
| 786 |
+
logger = logging.getLogger(__name__)
|
| 787 |
+
|
| 788 |
+
DISCLAIMER = "Screening aid β not a diagnosis. Pain or clearing tests require a clinician."
|
| 789 |
+
|
| 790 |
+
|
| 791 |
+
class PdfReportAgent:
|
| 792 |
+
"""Assembles the screening-session PDF via ReportLab."""
|
| 793 |
+
|
| 794 |
+
def run(self, report: ReportResult, entries: list, session_dir: str) -> str | None:
|
| 795 |
+
try:
|
| 796 |
+
from reportlab.lib import colors
|
| 797 |
+
from reportlab.lib.pagesizes import LETTER
|
| 798 |
+
from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
|
| 799 |
+
from reportlab.lib.units import inch
|
| 800 |
+
from reportlab.platypus import (
|
| 801 |
+
Image, Paragraph, SimpleDocTemplate, Spacer, Table, TableStyle,
|
| 802 |
+
)
|
| 803 |
+
except Exception as e:
|
| 804 |
+
logger.warning("reportlab unavailable: %s", e)
|
| 805 |
+
return None
|
| 806 |
+
|
| 807 |
+
out_path = os.path.join(session_dir, "formscout_report.pdf")
|
| 808 |
+
try:
|
| 809 |
+
styles = getSampleStyleSheet()
|
| 810 |
+
banner = ParagraphStyle(
|
| 811 |
+
"banner", parent=styles["Normal"], fontSize=9, textColor=colors.white,
|
| 812 |
+
backColor=colors.HexColor("#b45309"), alignment=1, borderPadding=6, spaceAfter=12,
|
| 813 |
+
)
|
| 814 |
+
story = []
|
| 815 |
+
story.append(Paragraph(f"<b>⚠ {DISCLAIMER}</b>", banner))
|
| 816 |
+
story.append(Paragraph("FormScout β FMS Screening Report", styles["Title"]))
|
| 817 |
+
|
| 818 |
+
if report.composite is not None:
|
| 819 |
+
comp = f"Composite: <b>{report.composite} / 21</b>"
|
| 820 |
+
else:
|
| 821 |
+
comp = f"Composite: <b>Incomplete</b> β {len(entries)}/7 tests scored"
|
| 822 |
+
story.append(Paragraph(comp, styles["Heading2"]))
|
| 823 |
+
story.append(Spacer(1, 0.2 * inch))
|
| 824 |
+
|
| 825 |
+
for e in entries:
|
| 826 |
+
title = e.test_name.replace("_", " ").title()
|
| 827 |
+
if e.side in ("left", "right"):
|
| 828 |
+
title += f" ({e.side})"
|
| 829 |
+
score_txt = "Clinician review required" if e.needs_human else f"Score: {e.score}/3"
|
| 830 |
+
story.append(Paragraph(f"<b>{title}</b> β {score_txt}", styles["Heading3"]))
|
| 831 |
+
if e.rationale:
|
| 832 |
+
story.append(Paragraph(e.rationale, styles["Normal"]))
|
| 833 |
+
if e.compensation_tags:
|
| 834 |
+
story.append(Paragraph("Compensations: " + ", ".join(e.compensation_tags),
|
| 835 |
+
styles["Normal"]))
|
| 836 |
+
if e.corrective_hint:
|
| 837 |
+
story.append(Paragraph("Corrective: " + e.corrective_hint, styles["Normal"]))
|
| 838 |
+
|
| 839 |
+
items = list(e.measurements.items())[:6]
|
| 840 |
+
if items:
|
| 841 |
+
rows = [[k.replace("_", " "),
|
| 842 |
+
(f"{v:.1f}" if isinstance(v, float) else str(v))] for k, v in items]
|
| 843 |
+
tbl = Table(rows, colWidths=[3 * inch, 1.5 * inch])
|
| 844 |
+
tbl.setStyle(TableStyle([
|
| 845 |
+
("FONTSIZE", (0, 0), (-1, -1), 8),
|
| 846 |
+
("TEXTCOLOR", (0, 0), (-1, -1), colors.HexColor("#334155")),
|
| 847 |
+
]))
|
| 848 |
+
story.append(tbl)
|
| 849 |
+
|
| 850 |
+
if e.keyframe_path and os.path.exists(e.keyframe_path):
|
| 851 |
+
try:
|
| 852 |
+
story.append(Image(e.keyframe_path, width=3.0 * inch, height=2.25 * inch))
|
| 853 |
+
except Exception:
|
| 854 |
+
story.append(Paragraph("<i>(key-frame image unavailable)</i>", styles["Normal"]))
|
| 855 |
+
else:
|
| 856 |
+
story.append(Paragraph("<i>(key-frame image unavailable)</i>", styles["Normal"]))
|
| 857 |
+
|
| 858 |
+
story.append(Spacer(1, 0.2 * inch))
|
| 859 |
+
|
| 860 |
+
if report.asymmetries:
|
| 861 |
+
story.append(Paragraph("Asymmetries", styles["Heading2"]))
|
| 862 |
+
for a in report.asymmetries:
|
| 863 |
+
story.append(Paragraph(
|
| 864 |
+
f"{a['test'].replace('_', ' ').title()}: "
|
| 865 |
+
f"L={a['left_score']} R={a['right_score']} (Δ {a['delta']})",
|
| 866 |
+
styles["Normal"]))
|
| 867 |
+
|
| 868 |
+
flags = list(report.low_confidence_flags) + list(report.disagreement_flags)
|
| 869 |
+
if flags:
|
| 870 |
+
story.append(Paragraph("Flags", styles["Heading2"]))
|
| 871 |
+
for fl in flags:
|
| 872 |
+
story.append(Paragraph(fl, styles["Normal"]))
|
| 873 |
+
|
| 874 |
+
story.append(Spacer(1, 0.3 * inch))
|
| 875 |
+
story.append(Paragraph(f"<b>⚠ {DISCLAIMER}</b>", banner))
|
| 876 |
+
|
| 877 |
+
doc = SimpleDocTemplate(out_path, pagesize=LETTER,
|
| 878 |
+
topMargin=0.6 * inch, bottomMargin=0.6 * inch)
|
| 879 |
+
doc.build(story)
|
| 880 |
+
return out_path
|
| 881 |
+
except Exception as e:
|
| 882 |
+
logger.warning("pdf build failed: %s", e)
|
| 883 |
+
return None
|
| 884 |
+
```
|
| 885 |
+
|
| 886 |
+
- [ ] **Step 4: Run test to verify it passes**
|
| 887 |
+
|
| 888 |
+
Run: `pytest tests/test_pdf_report.py -v`
|
| 889 |
+
Expected: both tests PASS
|
| 890 |
+
|
| 891 |
+
- [ ] **Step 5: Re-run the session suite (pdf_path now populated)**
|
| 892 |
+
|
| 893 |
+
Run: `pytest tests/test_session.py -v`
|
| 894 |
+
Expected: all PASS (now `finish_session` returns a real `pdf_path`).
|
| 895 |
+
|
| 896 |
+
- [ ] **Step 6: Commit**
|
| 897 |
+
|
| 898 |
+
```bash
|
| 899 |
+
git add formscout/agents/pdf_report.py tests/test_pdf_report.py
|
| 900 |
+
git commit -m "feat: add PdfReportAgent β branded ReportLab session PDF"
|
| 901 |
+
```
|
| 902 |
+
|
| 903 |
+
---
|
| 904 |
+
|
| 905 |
+
## Task 7: Wire the session UI in `app.py`
|
| 906 |
+
|
| 907 |
+
**Files:**
|
| 908 |
+
- Modify: `app.py` (`process_video`, `build_app`, event wiring)
|
| 909 |
+
|
| 910 |
+
This task is verified by running the app (Gradio event wiring is not unit-tested; the
|
| 911 |
+
orchestration it calls is already covered by `tests/test_session.py`).
|
| 912 |
+
|
| 913 |
+
- [ ] **Step 1: Import the session module**
|
| 914 |
+
|
| 915 |
+
In `app.py`, add to the imports block (after `from formscout.startup import ensure_checkpoints`):
|
| 916 |
+
|
| 917 |
+
```python
|
| 918 |
+
from formscout import session as session_mod
|
| 919 |
+
```
|
| 920 |
+
|
| 921 |
+
- [ ] **Step 2: Refactor `process_video` to accumulate into a session**
|
| 922 |
+
|
| 923 |
+
Replace the entire `process_video` function (lines ~51-105) with a version that takes and
|
| 924 |
+
returns the session, appends an entry on success, and builds the "Session so far" table.
|
| 925 |
+
Replace from `def process_video(` through its final `return ...` with:
|
| 926 |
+
|
| 927 |
+
```python
|
| 928 |
+
def process_video(video_path: str, test_name: str, side: str, model_key: str,
|
| 929 |
+
layers: list[str], session_state):
|
| 930 |
+
"""Analyse one clip and accumulate it into the screening session."""
|
| 931 |
+
if not video_path:
|
| 932 |
+
return (
|
| 933 |
+
session_state, _render_empty_state(), "Upload a video to begin analysis.",
|
| 934 |
+
"", "", None, "", _render_session_table(session_state),
|
| 935 |
+
gr.update(visible=False), gr.update(visible=False),
|
| 936 |
+
)
|
| 937 |
+
|
| 938 |
+
if session_state is None:
|
| 939 |
+
session_state = session_mod.new_session()
|
| 940 |
+
|
| 941 |
+
director = Director()
|
| 942 |
+
state = director.run(video_path, test_name=test_name, side=side, model_key=model_key)
|
| 943 |
+
|
| 944 |
+
score_html = _render_empty_state()
|
| 945 |
+
score_details = ""
|
| 946 |
+
|
| 947 |
+
if state.features:
|
| 948 |
+
result = score_test(state.features)
|
| 949 |
+
judge = state.judge
|
| 950 |
+
if judge and judge.score is not None:
|
| 951 |
+
score_html = _render_score_card(judge.score, judge.confidence, judge.needs_human)
|
| 952 |
+
score_details = _render_score_details_judge(judge, result, state.features)
|
| 953 |
+
elif judge and judge.needs_human:
|
| 954 |
+
score_html = _render_score_card(0, 0, True)
|
| 955 |
+
score_details = f"### Needs Clinician Review\n{judge.rationale}"
|
| 956 |
+
else:
|
| 957 |
+
score_html = _render_score_card(result.score, result.confidence, result.needs_human)
|
| 958 |
+
score_details = _render_score_details(result, state.features)
|
| 959 |
+
|
| 960 |
+
# Accumulate into the session (only when we have a real analysis)
|
| 961 |
+
if state.ingest and state.pose2d and state.judge:
|
| 962 |
+
draw_trails = "trails" in {lbl.lower().replace(" ", "_") for lbl in (layers or [])}
|
| 963 |
+
try:
|
| 964 |
+
session_mod.add_analysis(
|
| 965 |
+
session_state, ingest=state.ingest, pose2d=state.pose2d,
|
| 966 |
+
features=state.features, judge=state.judge,
|
| 967 |
+
test_name=test_name, side=side, draw_trails=draw_trails,
|
| 968 |
+
)
|
| 969 |
+
except Exception as e:
|
| 970 |
+
state.warnings.append(f"session accumulation failed: {e}")
|
| 971 |
+
|
| 972 |
+
pipeline_md = _render_pipeline_status(state)
|
| 973 |
+
alerts = _render_alerts(state)
|
| 974 |
+
|
| 975 |
+
overlay_path = None
|
| 976 |
+
vel_summary = ""
|
| 977 |
+
layer_set = {lbl.lower().replace(" ", "_") for lbl in (layers or [])}
|
| 978 |
+
if layer_set and state.ingest and state.pose2d:
|
| 979 |
+
try:
|
| 980 |
+
from formscout.agents.visualizer import PoseVisualizer, build_velocity_summary
|
| 981 |
+
vis = PoseVisualizer()
|
| 982 |
+
with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as f:
|
| 983 |
+
out_path = f.name
|
| 984 |
+
overlay_path = vis.render_video(state.ingest, state.pose2d, layer_set, out_path)
|
| 985 |
+
if overlay_path:
|
| 986 |
+
vel_summary = build_velocity_summary(state.pose2d.keypoints, vis.last_velocities)
|
| 987 |
+
except Exception as e:
|
| 988 |
+
alerts = (alerts or "") + f"\nβ οΈ Visualizer error: {e}"
|
| 989 |
+
|
| 990 |
+
has_entries = bool(session_state and session_state.entries)
|
| 991 |
+
return (
|
| 992 |
+
session_state, score_html, pipeline_md, score_details, alerts,
|
| 993 |
+
overlay_path, vel_summary, _render_session_table(session_state),
|
| 994 |
+
gr.update(visible=has_entries), gr.update(visible=has_entries),
|
| 995 |
+
)
|
| 996 |
+
```
|
| 997 |
+
|
| 998 |
+
- [ ] **Step 3: Add the session-table renderer and finish handler**
|
| 999 |
+
|
| 1000 |
+
In `app.py`, add these two functions just before `def build_app()`:
|
| 1001 |
+
|
| 1002 |
+
```python
|
| 1003 |
+
def _render_session_table(session_state) -> str:
|
| 1004 |
+
"""Render the accumulated 'Session so far' table as markdown."""
|
| 1005 |
+
if not session_state or not session_state.entries:
|
| 1006 |
+
return "*No clips analysed yet.*"
|
| 1007 |
+
lines = ["| Test | Side | Score | Status |", "|---|---|---|---|"]
|
| 1008 |
+
for e in session_state.entries:
|
| 1009 |
+
test = e.test_name.replace("_", " ").title()
|
| 1010 |
+
side = e.side if e.side in ("left", "right") else "β"
|
| 1011 |
+
if e.needs_human:
|
| 1012 |
+
score, status = "β", "β οΈ Clinician review"
|
| 1013 |
+
else:
|
| 1014 |
+
score, status = f"{e.score}/3", "β scored"
|
| 1015 |
+
lines.append(f"| {test} | {side} | {score} | {status} |")
|
| 1016 |
+
return "\n".join(lines)
|
| 1017 |
+
|
| 1018 |
+
|
| 1019 |
+
def _finish_session(session_state):
|
| 1020 |
+
"""Build the composite report + PDF for the whole session."""
|
| 1021 |
+
if not session_state or not session_state.entries:
|
| 1022 |
+
return ("β οΈ No clips analysed yet β analyse at least one clip first.",
|
| 1023 |
+
None, None)
|
| 1024 |
+
|
| 1025 |
+
report, pdf_path = session_mod.finish_session(session_state)
|
| 1026 |
+
if report is None:
|
| 1027 |
+
return ("β οΈ Nothing to report.", None, None)
|
| 1028 |
+
|
| 1029 |
+
if report.composite is not None:
|
| 1030 |
+
summary = [f"## Composite: {report.composite} / 21"]
|
| 1031 |
+
else:
|
| 1032 |
+
n = len(session_state.entries)
|
| 1033 |
+
summary = [f"## Composite: Incomplete β {n}/7 tests scored",
|
| 1034 |
+
"*(One or more tests need clinician review or were unscored.)*"]
|
| 1035 |
+
|
| 1036 |
+
if report.asymmetries:
|
| 1037 |
+
summary.append("\n### Asymmetries")
|
| 1038 |
+
for a in report.asymmetries:
|
| 1039 |
+
test = a["test"].replace("_", " ").title()
|
| 1040 |
+
summary.append(f"- **{test}:** L={a['left_score']} R={a['right_score']} (Ξ {a['delta']})")
|
| 1041 |
+
|
| 1042 |
+
flags = list(report.low_confidence_flags) + list(report.disagreement_flags)
|
| 1043 |
+
if flags:
|
| 1044 |
+
summary.append("\n### Flags")
|
| 1045 |
+
for fl in flags:
|
| 1046 |
+
summary.append(f"- {fl}")
|
| 1047 |
+
|
| 1048 |
+
md_path = os.path.join(session_state.session_dir, "analysis.md")
|
| 1049 |
+
md_out = md_path if os.path.exists(md_path) else None
|
| 1050 |
+
return "\n".join(summary), pdf_path, md_out
|
| 1051 |
+
```
|
| 1052 |
+
|
| 1053 |
+
Also add `import os` to the top of `app.py` if not already present (it currently imports only
|
| 1054 |
+
`tempfile` and `gradio`). Add after `import tempfile`:
|
| 1055 |
+
|
| 1056 |
+
```python
|
| 1057 |
+
import os
|
| 1058 |
+
```
|
| 1059 |
+
|
| 1060 |
+
- [ ] **Step 4: Add the session state, buttons, and outputs to `build_app`**
|
| 1061 |
+
|
| 1062 |
+
In `build_app`, inside the `with gr.Blocks(...) as app:` block, immediately after the line
|
| 1063 |
+
`with gr.Blocks(title="FormScout β FMS Screening Aid") as app:` add:
|
| 1064 |
+
|
| 1065 |
+
```python
|
| 1066 |
+
session_state = gr.State(None)
|
| 1067 |
+
```
|
| 1068 |
+
|
| 1069 |
+
Then, in the left input column, replace the single submit button block:
|
| 1070 |
+
|
| 1071 |
+
```python
|
| 1072 |
+
submit_btn = gr.Button(
|
| 1073 |
+
"π― Score Movement",
|
| 1074 |
+
variant="primary",
|
| 1075 |
+
size="lg",
|
| 1076 |
+
)
|
| 1077 |
+
```
|
| 1078 |
+
|
| 1079 |
+
with:
|
| 1080 |
+
|
| 1081 |
+
```python
|
| 1082 |
+
submit_btn = gr.Button(
|
| 1083 |
+
"π― Score Movement",
|
| 1084 |
+
variant="primary",
|
| 1085 |
+
size="lg",
|
| 1086 |
+
)
|
| 1087 |
+
with gr.Row():
|
| 1088 |
+
new_clip_btn = gr.Button("β Analyse new clip", visible=False)
|
| 1089 |
+
finish_btn = gr.Button("β
Finish & generate PDF",
|
| 1090 |
+
variant="primary", visible=False)
|
| 1091 |
+
```
|
| 1092 |
+
|
| 1093 |
+
In the right results column, add a "Session" tab and a finish-output area. Inside `with gr.Tabs():`
|
| 1094 |
+
add a new tab after the "π¬ Overlay Video" tab:
|
| 1095 |
+
|
| 1096 |
+
```python
|
| 1097 |
+
with gr.TabItem("ποΈ Session"):
|
| 1098 |
+
session_table = gr.Markdown("*No clips analysed yet.*")
|
| 1099 |
+
finish_summary = gr.Markdown("")
|
| 1100 |
+
pdf_file = gr.File(label="Screening Report (PDF)", visible=True)
|
| 1101 |
+
md_file = gr.File(label="Analysis Log (Markdown)", visible=True)
|
| 1102 |
+
```
|
| 1103 |
+
|
| 1104 |
+
- [ ] **Step 5: Update event wiring**
|
| 1105 |
+
|
| 1106 |
+
Replace the `_map_inputs` function and `submit_btn.click(...)` block at the bottom of `build_app`
|
| 1107 |
+
with:
|
| 1108 |
+
|
| 1109 |
+
```python
|
| 1110 |
+
def _map_inputs(video, test_display_name, side_display, pose_model_key, overlay_layers, sess):
|
| 1111 |
+
"""Map UI display values to internal values and accumulate into the session."""
|
| 1112 |
+
test_map = {name: val for name, val in FMS_TESTS}
|
| 1113 |
+
test_name = test_map.get(test_display_name, "deep_squat")
|
| 1114 |
+
side = {"N/A": "na", "Left": "left", "Right": "right"}.get(side_display, "na")
|
| 1115 |
+
return process_video(video, test_name, side, pose_model_key, overlay_layers, sess)
|
| 1116 |
+
|
| 1117 |
+
submit_btn.click(
|
| 1118 |
+
fn=_map_inputs,
|
| 1119 |
+
inputs=[video_input, test_dropdown, side_dropdown, pose_model_dropdown,
|
| 1120 |
+
overlay_layers, session_state],
|
| 1121 |
+
outputs=[session_state, score_html, pipeline_md, score_details, alerts_md,
|
| 1122 |
+
overlay_video, velocity_md, session_table, new_clip_btn, finish_btn],
|
| 1123 |
+
)
|
| 1124 |
+
|
| 1125 |
+
def _new_clip():
|
| 1126 |
+
"""Clear inputs for the next clip; keep the session intact."""
|
| 1127 |
+
return None, _render_empty_state(), ""
|
| 1128 |
+
|
| 1129 |
+
new_clip_btn.click(
|
| 1130 |
+
fn=_new_clip,
|
| 1131 |
+
inputs=[],
|
| 1132 |
+
outputs=[video_input, score_html, score_details],
|
| 1133 |
+
)
|
| 1134 |
+
|
| 1135 |
+
finish_btn.click(
|
| 1136 |
+
fn=_finish_session,
|
| 1137 |
+
inputs=[session_state],
|
| 1138 |
+
outputs=[finish_summary, pdf_file, md_file],
|
| 1139 |
+
)
|
| 1140 |
+
```
|
| 1141 |
+
|
| 1142 |
+
- [ ] **Step 6: Verify the full test suite still passes**
|
| 1143 |
+
|
| 1144 |
+
Run: `pytest tests/ -q`
|
| 1145 |
+
Expected: all tests pass except the single pre-existing known failure documented in CLAUDE.md
|
| 1146 |
+
(`test_unimplemented_test_returns_low_confidence`). No new failures.
|
| 1147 |
+
|
| 1148 |
+
- [ ] **Step 7: Manually verify the app**
|
| 1149 |
+
|
| 1150 |
+
Run: `python3 app.py`
|
| 1151 |
+
Then in the browser:
|
| 1152 |
+
1. Upload a clip, pick a test, click **Score Movement** β score card appears; the **Session** tab
|
| 1153 |
+
shows one row; the two new buttons appear.
|
| 1154 |
+
2. Click **β Analyse new clip** β the video input clears, the session row persists.
|
| 1155 |
+
3. Analyse a second test β a second row appears.
|
| 1156 |
+
4. Click **β
Finish & generate PDF** β the Session tab shows the composite summary and a
|
| 1157 |
+
downloadable PDF (open it: disclaimer top + bottom, per-test blocks with key-frame images,
|
| 1158 |
+
composite or "Incomplete"). The Markdown log is also downloadable.
|
| 1159 |
+
|
| 1160 |
+
Expected: all four steps work; PDF opens and contains the disclaimer, composite, and per-test sections.
|
| 1161 |
+
|
| 1162 |
+
- [ ] **Step 8: Commit**
|
| 1163 |
+
|
| 1164 |
+
```bash
|
| 1165 |
+
git add app.py
|
| 1166 |
+
git commit -m "feat: accumulate FMS clips into a session with composite report + PDF export"
|
| 1167 |
+
```
|
| 1168 |
+
|
| 1169 |
+
---
|
| 1170 |
+
|
| 1171 |
+
## Task 8: Update docs
|
| 1172 |
+
|
| 1173 |
+
**Files:**
|
| 1174 |
+
- Modify: `CLAUDE.md` (Build phases / status)
|
| 1175 |
+
- Modify: `MODEL_BUDGET.md` (no param change β note PDF agent adds 0 params, for completeness)
|
| 1176 |
+
|
| 1177 |
+
- [ ] **Step 1: Update the Phase 4 line in CLAUDE.md**
|
| 1178 |
+
|
| 1179 |
+
In `CLAUDE.md`, in the "Build phases" section, update the Phase 4 line from:
|
| 1180 |
+
|
| 1181 |
+
```
|
| 1182 |
+
4. **Phase 4 β Polish + ship:** Custom Svelte UI components, PDF export, agent trace to Hub, blog post. (Overlay video already done via `PoseVisualizer`.)
|
| 1183 |
+
```
|
| 1184 |
+
|
| 1185 |
+
to:
|
| 1186 |
+
|
| 1187 |
+
```
|
| 1188 |
+
4. **Phase 4 β Polish + ship:** Custom Svelte UI components, agent trace to Hub, blog post. (Overlay video done via `PoseVisualizer`; full 7-test session + PDF export done via `formscout/session.py` + `PdfReportAgent`.)
|
| 1189 |
+
```
|
| 1190 |
+
|
| 1191 |
+
- [ ] **Step 2: Note the PDF agent in the architecture section**
|
| 1192 |
+
|
| 1193 |
+
In `CLAUDE.md`, under "### Rubric scorers" or near the ReportAgent description, this is optional
|
| 1194 |
+
context; no required change. Skip if no natural home.
|
| 1195 |
+
|
| 1196 |
+
- [ ] **Step 3: Commit**
|
| 1197 |
+
|
| 1198 |
+
```bash
|
| 1199 |
+
git add CLAUDE.md
|
| 1200 |
+
git commit -m "docs: mark full FMS session + PDF export complete in build phases"
|
| 1201 |
+
```
|
| 1202 |
+
|
| 1203 |
+
---
|
| 1204 |
+
|
| 1205 |
+
## Self-Review Notes (already applied)
|
| 1206 |
+
|
| 1207 |
+
- **Spec coverage:** session accumulation (Task 5), two-button UX (Task 7), on-disk MD/JSON/keyframes (Task 5), key-frame from `features.timing` (Tasks 3β5), ReportLab PDF top/bottom disclaimer + composite + per-test + asymmetry + flags (Task 6), `SessionEntry` type (Task 2), `ReportAgent` reuse (Task 5 `finish_session`), composite-null-on-needs-human (Task 5 test), error tolerance / never-raise (Tasks 4β6). All covered.
|
| 1208 |
+
- **Type consistency:** `SessionEntry` field names are identical across Tasks 2, 5, 6, 7. `finish_session` returns `(ReportResult | None, str | None)` and is consumed that way in Task 7. `render_frame(ingest, pose2d, frame_idx, layers, caption, out_png)` signature matches its callers.
|
| 1209 |
+
- **No placeholders:** every code step shows complete code; every run step states the exact command + expected outcome.
|
docs/superpowers/specs/2026-06-09-pose-model-selector-design.md
ADDED
|
@@ -0,0 +1,171 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Pose Model Selector β Design Spec
|
| 2 |
+
|
| 3 |
+
**Date:** 2026-06-09
|
| 4 |
+
**Status:** Approved
|
| 5 |
+
|
| 6 |
+
## Goal
|
| 7 |
+
|
| 8 |
+
Expose all available pose estimation models as a selectable dropdown in the Gradio UI, replacing the hard-coded YOLO26l default. Supported families: MediaPipe (Qualcomm HF/ONNX), YOLO26 nβx (local), Sapiens2 0.4Bβ5B (HF/transformers).
|
| 9 |
+
|
| 10 |
+
---
|
| 11 |
+
|
| 12 |
+
## Architecture
|
| 13 |
+
|
| 14 |
+
### Unified model registry (`config.py`)
|
| 15 |
+
|
| 16 |
+
Replace `YOLO_POSE_MODELS` with a single `POSE_MODELS` dict. Each entry:
|
| 17 |
+
|
| 18 |
+
```python
|
| 19 |
+
{
|
| 20 |
+
"backend": "yolo" | "mediapipe" | "sapiens2",
|
| 21 |
+
"path": str, # yolo only β absolute path to local .pt
|
| 22 |
+
"hf_id": str, # mediapipe + sapiens2 β HuggingFace repo id
|
| 23 |
+
"params_m": float, # millions of parameters
|
| 24 |
+
}
|
| 25 |
+
```
|
| 26 |
+
|
| 27 |
+
Ordered as displayed in the UI:
|
| 28 |
+
|
| 29 |
+
| Label | backend | source |
|
| 30 |
+
|---|---|---|
|
| 31 |
+
| `MediaPipe-Pose β¬ ~16 MB, CPU-friendly` | mediapipe | `qualcomm/MediaPipe-Pose-Estimation` |
|
| 32 |
+
| `YOLO26n β nano (0.7M, fastest)` β
default | yolo | local checkpoint |
|
| 33 |
+
| `YOLO26s β small (3.5M)` | yolo | local checkpoint |
|
| 34 |
+
| `YOLO26m β medium (9M)` | yolo | local checkpoint |
|
| 35 |
+
| `YOLO26l β large (25.9M)` | yolo | local checkpoint |
|
| 36 |
+
| `YOLO26x β extra-large (57.6M)` | yolo | local checkpoint |
|
| 37 |
+
| `Sapiens2-0.4B β¬ ~1.6 GB` | sapiens2 | `facebook/sapiens2-pose-0.4b` |
|
| 38 |
+
| `Sapiens2-0.8B β¬ ~3.2 GB` | sapiens2 | `facebook/sapiens2-pose-0.8b` |
|
| 39 |
+
| `Sapiens2-1B β¬ ~4 GB` | sapiens2 | `facebook/sapiens2-pose-1b` |
|
| 40 |
+
| `Sapiens2-5B β¬ ~20 GB, large GPU` | sapiens2 | `facebook/sapiens2-pose-5b` |
|
| 41 |
+
|
| 42 |
+
```python
|
| 43 |
+
DEFAULT_POSE_MODEL = "YOLO26n β nano (0.7M, fastest)"
|
| 44 |
+
```
|
| 45 |
+
|
| 46 |
+
Keep `YOLO_POSE_MODEL` and `YOLO_POSE_MODEL_HQ` as string aliases for backward compat with any direct references outside the agent.
|
| 47 |
+
|
| 48 |
+
---
|
| 49 |
+
|
| 50 |
+
### Pose2DAgent (`formscout/agents/pose2d.py`)
|
| 51 |
+
|
| 52 |
+
Three private sub-runners, all returning `list[dict[int, dict]]` (COCO 17 keypoints per frame, same format as today):
|
| 53 |
+
|
| 54 |
+
#### `_run_yolo(frames, path) -> list[dict]`
|
| 55 |
+
Existing logic, lifted into a named function. Model cached in `_model_cache[path]`.
|
| 56 |
+
|
| 57 |
+
#### `_run_mediapipe(frames, hf_id) -> list[dict]`
|
| 58 |
+
- Download repo snapshot via `huggingface_hub.snapshot_download(hf_id)`
|
| 59 |
+
- Locate the pose landmark `.onnx` file in the snapshot
|
| 60 |
+
- Load with `onnxruntime.InferenceSession`
|
| 61 |
+
- Preprocess each frame: resize to 256Γ256, normalize
|
| 62 |
+
- Run inference β 33 BlazePose landmarks
|
| 63 |
+
- Map BlazePose 33 β COCO 17 via fixed index table:
|
| 64 |
+
```
|
| 65 |
+
COCO 0=nose β BlazePose 0
|
| 66 |
+
COCO 1=left_eye β BlazePose 2
|
| 67 |
+
COCO 2=right_eye β BlazePose 5
|
| 68 |
+
COCO 3=left_ear β BlazePose 7
|
| 69 |
+
COCO 4=right_ear β BlazePose 8
|
| 70 |
+
COCO 5=left_shld β BlazePose 11
|
| 71 |
+
COCO 6=right_shld β BlazePose 12
|
| 72 |
+
COCO 7=left_elbow β BlazePose 13
|
| 73 |
+
COCO 8=right_elbow β BlazePose 14
|
| 74 |
+
COCO 9=left_wrist β BlazePose 15
|
| 75 |
+
COCO 10=right_wrist β BlazePose 16
|
| 76 |
+
COCO 11=left_hip β BlazePose 23
|
| 77 |
+
COCO 12=right_hip β BlazePose 24
|
| 78 |
+
COCO 13=left_knee β BlazePose 25
|
| 79 |
+
COCO 14=right_knee β BlazePose 26
|
| 80 |
+
COCO 15=left_ankle β BlazePose 27
|
| 81 |
+
COCO 16=right_ankle β BlazePose 28
|
| 82 |
+
```
|
| 83 |
+
- Session cached in `_model_cache[hf_id]`
|
| 84 |
+
|
| 85 |
+
#### `_run_sapiens2(frames, hf_id) -> list[dict]`
|
| 86 |
+
- Load via `transformers.pipeline("pose-estimation", model=hf_id)`
|
| 87 |
+
- Sapiens2 outputs 308 whole-body keypoints; map first 17 (indices 0β16) to COCO 17 β Sapiens2 preserves COCO ordering for the body subset
|
| 88 |
+
- Pipeline cached in `_model_cache[hf_id]`
|
| 89 |
+
|
| 90 |
+
#### `Pose2DAgent.run(ingest, model_key)`
|
| 91 |
+
- `model_key: str` replaces `model_path: str` (old param)
|
| 92 |
+
- Looks up `config.POSE_MODELS[model_key]` (falls back to `DEFAULT_POSE_MODEL` if key missing)
|
| 93 |
+
- Dispatches to the appropriate sub-runner
|
| 94 |
+
- Returns `Pose2DResult` β identical contract as today
|
| 95 |
+
|
| 96 |
+
---
|
| 97 |
+
|
| 98 |
+
### UI (`app.py`)
|
| 99 |
+
|
| 100 |
+
Add `gr.Dropdown` for pose model in the input column, below the test/side row:
|
| 101 |
+
|
| 102 |
+
```python
|
| 103 |
+
pose_model_dropdown = gr.Dropdown(
|
| 104 |
+
choices=list(config.POSE_MODELS.keys()),
|
| 105 |
+
value=config.DEFAULT_POSE_MODEL,
|
| 106 |
+
label="Pose Model",
|
| 107 |
+
)
|
| 108 |
+
```
|
| 109 |
+
|
| 110 |
+
Update `_map_inputs` to accept and forward `pose_model_key`:
|
| 111 |
+
|
| 112 |
+
```python
|
| 113 |
+
def _map_inputs(video, test_display_name, side_display, pose_model_key):
|
| 114 |
+
...
|
| 115 |
+
return process_video(video, test_name, side, pose_model_key)
|
| 116 |
+
```
|
| 117 |
+
|
| 118 |
+
Update `submit_btn.click` inputs to include `pose_model_dropdown`.
|
| 119 |
+
|
| 120 |
+
`process_video(video_path, test_name, side, pose_model_key)` passes `pose_model_key` through to `director.run()`, which passes it to `Pose2DAgent.run()`. Remove the old `YOLO_POSE_MODELS.get()` lookup from `process_video`.
|
| 121 |
+
|
| 122 |
+
---
|
| 123 |
+
|
| 124 |
+
## Data flow
|
| 125 |
+
|
| 126 |
+
```
|
| 127 |
+
UI dropdown (pose_model_key: str)
|
| 128 |
+
β process_video()
|
| 129 |
+
β Director.run(pose_model_key=...)
|
| 130 |
+
β Pose2DAgent.run(ingest, model_key=pose_model_key)
|
| 131 |
+
β config.POSE_MODELS[model_key] β {backend, path|hf_id}
|
| 132 |
+
β _run_yolo / _run_mediapipe / _run_sapiens2
|
| 133 |
+
β list[dict[int, {x, y, conf}]] (COCO 17, same contract)
|
| 134 |
+
β Pose2DResult
|
| 135 |
+
```
|
| 136 |
+
|
| 137 |
+
---
|
| 138 |
+
|
| 139 |
+
## Error handling
|
| 140 |
+
|
| 141 |
+
- Unknown `model_key`: log warning, fall back to `DEFAULT_POSE_MODEL`
|
| 142 |
+
- ONNX file not found in MediaPipe snapshot: `Pose2DResult(confidence=0.0, notes="mediapipe onnx not found")`
|
| 143 |
+
- Sapiens2 / MediaPipe download failure: `Pose2DResult(confidence=0.0, notes=str(e))`
|
| 144 |
+
- All failures are non-fatal; pipeline continues with 0-confidence result and surfaces alert in UI
|
| 145 |
+
|
| 146 |
+
---
|
| 147 |
+
|
| 148 |
+
## Dependencies to add (`requirements.txt`)
|
| 149 |
+
|
| 150 |
+
- `onnxruntime` β MediaPipe ONNX inference
|
| 151 |
+
- `huggingface_hub` β snapshot download for MediaPipe (already likely present via transformers)
|
| 152 |
+
|
| 153 |
+
Sapiens2 uses `transformers`, already a dependency.
|
| 154 |
+
|
| 155 |
+
---
|
| 156 |
+
|
| 157 |
+
## Testing
|
| 158 |
+
|
| 159 |
+
Each new backend gets a pytest in `tests/test_pose2d.py` that:
|
| 160 |
+
- Mocks the model load (no actual HF download in CI)
|
| 161 |
+
- Passes a 3-frame synthetic IngestResult
|
| 162 |
+
- Asserts `Pose2DResult.keypoints` has 3 entries, each a dict with at most 17 int keys
|
| 163 |
+
- Asserts `confidence` is a float in [0, 1]
|
| 164 |
+
|
| 165 |
+
---
|
| 166 |
+
|
| 167 |
+
## Out of scope
|
| 168 |
+
|
| 169 |
+
- Sapiens2 / MediaPipe accuracy benchmarking
|
| 170 |
+
- Automatic backend selection based on hardware
|
| 171 |
+
- Downloading Sapiens2/MediaPipe checkpoints to local `checkpoints/` directory
|
docs/superpowers/specs/2026-06-09-pose-visualizer-design.md
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Pose Overlay Visualizer β Design Spec
|
| 2 |
+
|
| 3 |
+
**Date:** 2026-06-09
|
| 4 |
+
**Status:** Approved
|
| 5 |
+
|
| 6 |
+
## Goal
|
| 7 |
+
|
| 8 |
+
Add an annotated overlay video output to the FormScout UI showing skeleton, motion trails, and velocity arrows on top of the original footage, alongside a per-joint velocity summary table. Overlay layers are user-selectable via checkboxes. Adapted from the Laban Movement Analysis project.
|
| 9 |
+
|
| 10 |
+
---
|
| 11 |
+
|
| 12 |
+
## Architecture
|
| 13 |
+
|
| 14 |
+
Three files change or are created. No changes to `pipeline.py`, `types.py`, or any existing agent.
|
| 15 |
+
|
| 16 |
+
```
|
| 17 |
+
formscout/agents/visualizer.py β new
|
| 18 |
+
tests/test_visualizer.py β new
|
| 19 |
+
app.py β overlay_layers checkbox, new tab, wiring
|
| 20 |
+
```
|
| 21 |
+
|
| 22 |
+
The visualizer runs **after** `director.run()` returns in `process_video()` β it is a pure post-processing step, never on the critical scoring path.
|
| 23 |
+
|
| 24 |
+
---
|
| 25 |
+
|
| 26 |
+
## Module: `formscout/agents/visualizer.py`
|
| 27 |
+
|
| 28 |
+
### `compute_joint_velocity(keypoints_per_frame, fps) β dict[int, list[float]]`
|
| 29 |
+
|
| 30 |
+
- Input: `list[dict[int, {x, y, conf}]]` (COCO-17 pixel coords per frame), `fps: float`
|
| 31 |
+
- Output: `dict[int, list[float]]` β per-joint per-frame speed in **px/s**
|
| 32 |
+
- Method: for each joint index, run a `SimpleKalmanFilter` (1D per axis, constant-velocity model, same structure as Laban's engine) over the (x, y) series. Speed = `sqrt(vxΒ² + vyΒ²)` from the filter's velocity state.
|
| 33 |
+
- Missing keypoints (conf < 0.3 or absent) β speed = 0.0 for that frame, filter state held.
|
| 34 |
+
|
| 35 |
+
### `SimpleKalmanFilter`
|
| 36 |
+
|
| 37 |
+
Minimal 4-state Kalman (x, y, vx, vy), identical in structure to the Laban `SimpleKalmanFilter`:
|
| 38 |
+
- Transition: constant-velocity model
|
| 39 |
+
- Measurement: position only (x, y)
|
| 40 |
+
- One instance per joint per video run
|
| 41 |
+
|
| 42 |
+
### `PoseVisualizer`
|
| 43 |
+
|
| 44 |
+
#### Constants
|
| 45 |
+
```python
|
| 46 |
+
COCO_SKELETON = [
|
| 47 |
+
(0,1),(0,2),(1,3),(2,4), # face
|
| 48 |
+
(5,6),(5,7),(7,9),(6,8),(8,10), # arms
|
| 49 |
+
(5,11),(6,12),(11,12), # torso
|
| 50 |
+
(11,13),(13,15),(12,14),(14,16), # legs
|
| 51 |
+
]
|
| 52 |
+
TRAIL_LENGTH = 10 # frames of trail history
|
| 53 |
+
MAX_ARROW_PX = 40 # arrow scaled so peak velocity β 40px length
|
| 54 |
+
CONF_THRESHOLD = 0.3 # min confidence to draw a keypoint
|
| 55 |
+
```
|
| 56 |
+
|
| 57 |
+
#### Private methods
|
| 58 |
+
|
| 59 |
+
**`_draw_skeleton(frame, kps)`**
|
| 60 |
+
- Draw each COCO bone as a line if both endpoints have conf > CONF_THRESHOLD
|
| 61 |
+
- Joint dots: color greenβred by confidence using HSV (same as Laban `_confidence_to_color`)
|
| 62 |
+
- Bone color: white
|
| 63 |
+
|
| 64 |
+
**`_draw_trails(frame, trail_history, frame_idx)`**
|
| 65 |
+
- `trail_history: dict[int, deque(maxlen=TRAIL_LENGTH)]` keyed by joint index
|
| 66 |
+
- Each deque holds `(x, y)` pixel positions from previous frames
|
| 67 |
+
- Draw fading line segments: alpha = segment_position / TRAIL_LENGTH, color white
|
| 68 |
+
|
| 69 |
+
**`_draw_velocity_arrows(frame, kps, velocities, frame_idx)`**
|
| 70 |
+
- `velocities: dict[int, list[float]]` β speeds per joint per frame
|
| 71 |
+
- Direction vector from consecutive keypoint positions (x[t] - x[t-1], y[t] - y[t-1])
|
| 72 |
+
- Arrow length = `speed / peak_speed * MAX_ARROW_PX` (clamped)
|
| 73 |
+
- Drawn only for joints with conf > CONF_THRESHOLD and speed > 0
|
| 74 |
+
- Color: green=slow, orange=medium, red=fast (same thresholds as Laban intensity)
|
| 75 |
+
|
| 76 |
+
#### Public method
|
| 77 |
+
|
| 78 |
+
**`render_video(ingest, pose2d, layers: set[str], output_path: str) β str | None`**
|
| 79 |
+
- `layers`: subset of `{"skeleton", "trails", "velocity_arrows"}`
|
| 80 |
+
- If `layers` is empty β return `None` immediately
|
| 81 |
+
- Pre-computes `compute_joint_velocity(pose2d.keypoints, ingest.fps)`
|
| 82 |
+
- Iterates frames, updates `trail_history`, calls selected `_draw_*` methods
|
| 83 |
+
- Writes output via `cv2.VideoWriter` (codec: `mp4v`, same fps as ingest)
|
| 84 |
+
- Returns output path on success; `None` on any exception (logs warning)
|
| 85 |
+
|
| 86 |
+
#### Velocity summary
|
| 87 |
+
|
| 88 |
+
**`build_velocity_summary(keypoints_per_frame, velocities) β str`**
|
| 89 |
+
- For each joint with conf > 0.3 in >50% of frames:
|
| 90 |
+
- Compute avg and peak speed (px/s)
|
| 91 |
+
- Return markdown table sorted by peak speed descending:
|
| 92 |
+
```
|
| 93 |
+
| Joint | Avg (px/s) | Peak (px/s) |
|
| 94 |
+
|---------------|-----------|-------------|
|
| 95 |
+
| left_knee | 42.3 | 118.7 |
|
| 96 |
+
```
|
| 97 |
+
- Returns empty string if no valid joints
|
| 98 |
+
|
| 99 |
+
---
|
| 100 |
+
|
| 101 |
+
## UI changes: `app.py`
|
| 102 |
+
|
| 103 |
+
### Input column β overlay layer checkboxes
|
| 104 |
+
|
| 105 |
+
Below `pose_model_dropdown`, add:
|
| 106 |
+
|
| 107 |
+
```python
|
| 108 |
+
overlay_layers = gr.CheckboxGroup(
|
| 109 |
+
choices=["Skeleton", "Trails", "Velocity arrows"],
|
| 110 |
+
value=["Skeleton", "Trails"],
|
| 111 |
+
label="Overlay Layers",
|
| 112 |
+
)
|
| 113 |
+
```
|
| 114 |
+
|
| 115 |
+
### Results panel β new tab
|
| 116 |
+
|
| 117 |
+
Inside the existing `gr.Tabs()` block, add a fourth tab:
|
| 118 |
+
|
| 119 |
+
```python
|
| 120 |
+
with gr.TabItem("π¬ Overlay Video"):
|
| 121 |
+
overlay_video = gr.Video(label="Annotated Movement")
|
| 122 |
+
velocity_md = gr.Markdown("")
|
| 123 |
+
```
|
| 124 |
+
|
| 125 |
+
### `process_video()` signature
|
| 126 |
+
|
| 127 |
+
```python
|
| 128 |
+
def process_video(video_path, test_name, side, model_key, layers: list[str]):
|
| 129 |
+
```
|
| 130 |
+
|
| 131 |
+
After `director.run()`:
|
| 132 |
+
```python
|
| 133 |
+
from formscout.agents.visualizer import PoseVisualizer, build_velocity_summary
|
| 134 |
+
layer_set = {l.lower().replace(" ", "_") for l in layers}
|
| 135 |
+
# map UI labels to internal names:
|
| 136 |
+
# "Skeleton" β "skeleton", "Trails" β "trails", "Velocity arrows" β "velocity_arrows"
|
| 137 |
+
overlay_path = None
|
| 138 |
+
vel_summary = ""
|
| 139 |
+
if layer_set and state.ingest and state.pose2d:
|
| 140 |
+
try:
|
| 141 |
+
vis = PoseVisualizer()
|
| 142 |
+
with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as f:
|
| 143 |
+
out_path = f.name
|
| 144 |
+
overlay_path = vis.render_video(state.ingest, state.pose2d, layer_set, out_path)
|
| 145 |
+
if overlay_path:
|
| 146 |
+
vel_summary = build_velocity_summary(state.pose2d.keypoints, vis.last_velocities)
|
| 147 |
+
except Exception as e:
|
| 148 |
+
alerts += f"\nβ οΈ Visualizer error: {e}"
|
| 149 |
+
return score_html, pipeline_md, score_details, alerts, overlay_path, vel_summary
|
| 150 |
+
```
|
| 151 |
+
|
| 152 |
+
`vis.last_velocities` is stored on the instance after `render_video()` to avoid recomputing.
|
| 153 |
+
|
| 154 |
+
### Event wiring
|
| 155 |
+
|
| 156 |
+
```python
|
| 157 |
+
submit_btn.click(
|
| 158 |
+
fn=_map_inputs,
|
| 159 |
+
inputs=[video_input, test_dropdown, side_dropdown, pose_model_dropdown, overlay_layers],
|
| 160 |
+
outputs=[score_html, pipeline_md, score_details, alerts_md, overlay_video, velocity_md],
|
| 161 |
+
)
|
| 162 |
+
```
|
| 163 |
+
|
| 164 |
+
`_map_inputs` gains `overlay_layers` as fifth parameter.
|
| 165 |
+
|
| 166 |
+
---
|
| 167 |
+
|
| 168 |
+
## Error handling
|
| 169 |
+
|
| 170 |
+
| Failure | Behaviour |
|
| 171 |
+
|---|---|
|
| 172 |
+
| All frames have no detections | `render_video()` returns `None`, tab empty, no crash |
|
| 173 |
+
| `cv2.VideoWriter` fails | logs warning, returns `None` |
|
| 174 |
+
| Any exception in visualizer | caught in `process_video()`, appended to alerts, `overlay_path = None` |
|
| 175 |
+
| `layers` is empty | returns `None` immediately, no processing |
|
| 176 |
+
|
| 177 |
+
The score is always returned regardless of visualizer outcome.
|
| 178 |
+
|
| 179 |
+
---
|
| 180 |
+
|
| 181 |
+
## Testing: `tests/test_visualizer.py`
|
| 182 |
+
|
| 183 |
+
- Synthetic `IngestResult`: 5 blank 480Γ640 BGR frames, fps=30
|
| 184 |
+
- Synthetic `Pose2DResult`: 17 keypoints per frame at fixed positions with conf=0.9
|
| 185 |
+
- `test_render_video_creates_file`: assert output `.mp4` exists and size > 0
|
| 186 |
+
- `test_compute_joint_velocity_shape`: assert 17-key dict, each list length == 5
|
| 187 |
+
- `test_empty_layers_returns_none`: assert `render_video(..., layers=set())` returns `None`
|
| 188 |
+
- `test_no_detections_returns_none`: all-empty keypoints β `None`
|
| 189 |
+
- `test_velocity_summary_markdown`: assert output contains `|` (table) and at least one joint name
|
| 190 |
+
|
| 191 |
+
---
|
| 192 |
+
|
| 193 |
+
## Out of scope
|
| 194 |
+
|
| 195 |
+
- Frame-by-frame metrics synced to video playback (Phase 4 / custom Svelte)
|
| 196 |
+
- Multi-person tracking
|
| 197 |
+
- Saving overlay video to Hugging Face Hub (tracing feature, Phase 4)
|
docs/superpowers/specs/2026-06-13-full-fms-session-pdf-design.md
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Full FMS Session + PDF Report β Design
|
| 2 |
+
|
| 3 |
+
**Date:** 2026-06-13
|
| 4 |
+
**Status:** Approved (brainstorming) β pending implementation plan
|
| 5 |
+
**Owner:** FormScout
|
| 6 |
+
**Related:** `formscout/agents/report.py`, `formscout/agents/visualizer.py`, `formscout/agents/biomechanics.py`, `app.py`, `formscout/types.py`
|
| 7 |
+
|
| 8 |
+
## Problem
|
| 9 |
+
|
| 10 |
+
FormScout today scores **one** FMS test per upload. A real Functional Movement Screen is **all 7 tests** producing a single **composite 0β21** with asymmetry flags. `ReportAgent` and `ReportResult.composite` already support a multi-test report, but the UI never accumulates more than one test, and `ReportResult.pdf_path` is a hardcoded `None` stub.
|
| 11 |
+
|
| 12 |
+
This feature turns the one-clip scorer into a **screening session**: each analyzed clip accumulates; a "Finish" action produces the composite report plus a downloadable, brand-consistent **PDF**. Each clip's worst-moment frame is captured as an annotated still embedded in both an on-disk log and the PDF.
|
| 13 |
+
|
| 14 |
+
## Goals
|
| 15 |
+
|
| 16 |
+
- Accumulate multiple analyzed clips into one session, then emit a composite 0β21 report.
|
| 17 |
+
- Generate a clinician/client-facing **PDF** handout (ReportLab) with scores, rationale, asymmetries, key-frame images, and the safety disclaimer.
|
| 18 |
+
- Capture and annotate the **worst-moment frame** per test (the governing/peak frame already computed by `BiomechanicsAgent`).
|
| 19 |
+
- Persist each analysis incrementally to disk (`session.json`, `analysis.md`, key-frame PNGs) until "Finish" is clicked.
|
| 20 |
+
|
| 21 |
+
## Non-goals (YAGNI)
|
| 22 |
+
|
| 23 |
+
- No cross-restart session reload β the session lives in `gr.State` + a temp dir for the browser session.
|
| 24 |
+
- No PDF styling beyond a clean branded layout (no HTML/CSS engine; ReportLab only).
|
| 25 |
+
- No RAG / exemplar-clip citations (separate future spec).
|
| 26 |
+
- No changes to the scoring pipeline, rubric functions, or Director flow.
|
| 27 |
+
|
| 28 |
+
## UX
|
| 29 |
+
|
| 30 |
+
The current one-clip-at-a-time flow is preserved. Two new buttons appear after an analysis completes:
|
| 31 |
+
|
| 32 |
+
- **β Analyse new clip** β clears the video/test inputs for the next upload; **keeps** the session.
|
| 33 |
+
- **β
Finish & generate PDF** β runs the report + PDF over everything accumulated so far.
|
| 34 |
+
|
| 35 |
+
After each analysis, a **"Session so far"** table updates: `test Β· side Β· score Β· status`. Finish renders an on-screen composite scorecard + asymmetry summary and exposes the PDF (and `analysis.md`) via `gr.File` for download.
|
| 36 |
+
|
| 37 |
+
Guard: Finish with zero analyses β warning, no PDF.
|
| 38 |
+
|
| 39 |
+
## Components
|
| 40 |
+
|
| 41 |
+
### 1. Session state + on-disk store
|
| 42 |
+
|
| 43 |
+
A per-session temp directory `<tmpdir>/formscout_sessions/<session_id>/`:
|
| 44 |
+
|
| 45 |
+
- `session.json` β structured list of entries; **source of truth** for the PDF.
|
| 46 |
+
- `analysis.md` β human-readable log, appended after each clip.
|
| 47 |
+
- `keyframes/<test_name>_<side>.png` β annotated worst-frame stills.
|
| 48 |
+
|
| 49 |
+
Session identity lives in a `gr.State`. Each entry carries:
|
| 50 |
+
|
| 51 |
+
- `test_name`, `side`, `score` (judge score, else rubric), `needs_human`
|
| 52 |
+
- `rationale`, `compensation_tags`, `corrective_hint`
|
| 53 |
+
- key measurements (selected `angles` / `alignments`)
|
| 54 |
+
- `confidence`, `view` (`"2d"`/`"3d"`)
|
| 55 |
+
- `keyframe_path`
|
| 56 |
+
- the `movement` / `features` / `rubric_score` / `judge` objects that `ReportAgent.run()` consumes
|
| 57 |
+
|
| 58 |
+
Persistence lasts until Finish; files are kept afterward for download. Cross-restart cleanup is best-effort and out of scope.
|
| 59 |
+
|
| 60 |
+
### 2. Key-frame capture
|
| 61 |
+
|
| 62 |
+
New method on `PoseVisualizer`:
|
| 63 |
+
|
| 64 |
+
```python
|
| 65 |
+
def render_frame(self, ingest, pose2d, frame_idx: int,
|
| 66 |
+
layers: set[str], caption: str, out_png: str) -> str | None
|
| 67 |
+
```
|
| 68 |
+
|
| 69 |
+
- `frame_idx` comes from `features.timing`, which already stores the governing frame per test:
|
| 70 |
+
`deep_squat β deepest_frame`, `hurdle_step β peak_step_frame`,
|
| 71 |
+
`inline_lunge β deepest_lunge_frame`, `shoulder_mobility β measure_frame`,
|
| 72 |
+
`active_slr β peak_raise_frame`.
|
| 73 |
+
- `trunk_stability_pushup` and `rotary_stability` currently store only counts in `timing`. Add the worst-sag-frame and peak-extension-frame index to their `timing` dicts (one-line change in each `BiomechanicsAgent` method).
|
| 74 |
+
- Reuses `_draw_skeleton` (+ optional `_draw_trails`) on the single frame, overlays a caption naming the worst compensation, writes a PNG.
|
| 75 |
+
- Returns `None` on any failure β never raises, never blocks the entry.
|
| 76 |
+
|
| 77 |
+
The "worst compensation" caption is derived from `judge.compensation_tags` (preferred) or the failed `alignments` (fallback).
|
| 78 |
+
|
| 79 |
+
### 3. PDF generator
|
| 80 |
+
|
| 81 |
+
New module `formscout/agents/pdf_report.py`:
|
| 82 |
+
|
| 83 |
+
```python
|
| 84 |
+
class PdfReportAgent:
|
| 85 |
+
def run(self, report_result: ReportResult,
|
| 86 |
+
entries: list[SessionEntry], session_dir: str) -> str | None
|
| 87 |
+
```
|
| 88 |
+
|
| 89 |
+
Uses **ReportLab** (pure-Python, no system deps β safe on HF Spaces/ZeroGPU). Layout:
|
| 90 |
+
|
| 91 |
+
- Safety disclaimer banner at **top and bottom** (mirrors the UI invariant).
|
| 92 |
+
- Title/brand header + date.
|
| 93 |
+
- Composite **0β21** badge, or "Incomplete β N/7 tests scored" when `composite is None`.
|
| 94 |
+
- Per-test block: score, rationale, key measurements, compensation tags, corrective hint, the annotated key-frame image, asymmetry delta (bilateral).
|
| 95 |
+
- Flags section: low-confidence, rubricβjudge disagreement, needs-human.
|
| 96 |
+
- Populates `ReportResult.pdf_path`.
|
| 97 |
+
|
| 98 |
+
Returns the PDF path, or `None` on failure (UI surfaces the error and keeps the session for retry). Image embedding tolerates a missing/`None` `keyframe_path` with a placeholder line.
|
| 99 |
+
|
| 100 |
+
### 4. ReportAgent reuse
|
| 101 |
+
|
| 102 |
+
At Finish, build the entry list and call the existing `ReportAgent.run()` for composite + asymmetries + flags. The bilateral lower-score + asymmetry-delta logic and the null-composite rule already exist and are not rewritten. A small adapter converts `SessionEntry` objects to the dict schema `ReportAgent.run()` expects (or `ReportAgent` gains overload tolerance β implementer's choice, keep it minimal).
|
| 103 |
+
|
| 104 |
+
### 5. Types
|
| 105 |
+
|
| 106 |
+
Add a `SessionEntry` frozen dataclass to `formscout/types.py` (consistent with the "every agent I/O is a typed dataclass" standard), including `keyframe_path: str | None`. Populate the existing `ReportResult.pdf_path` (and optionally `overlay_video_path`). No other type changes.
|
| 107 |
+
|
| 108 |
+
### 6. UI (`app.py`)
|
| 109 |
+
|
| 110 |
+
- Add a `gr.State` holding the session (id + entries).
|
| 111 |
+
- After each analysis: render the scorecard as today, append the entry, write `session.json`/`analysis.md`/keyframe PNG, refresh the "Session so far" table, and reveal the two buttons.
|
| 112 |
+
- **Analyse new clip**: reset the video/test/side inputs; keep session state.
|
| 113 |
+
- **Finish & generate PDF**: `ReportAgent.run` β `PdfReportAgent.run` β display composite + asymmetry summary + `gr.File` downloads (PDF + `analysis.md`).
|
| 114 |
+
- Guard: Finish with zero analyses β warning.
|
| 115 |
+
|
| 116 |
+
## Data flow
|
| 117 |
+
|
| 118 |
+
```
|
| 119 |
+
upload β Director.run β score
|
| 120 |
+
β build SessionEntry (+ render_frame keyframe png)
|
| 121 |
+
β append to gr.State + write session.json / analysis.md / keyframe png
|
| 122 |
+
β refresh "Session so far" table
|
| 123 |
+
|
| 124 |
+
Finish β ReportAgent.run(entries) β composite / asymmetries / flags
|
| 125 |
+
β PdfReportAgent.run(...) β pdf_path
|
| 126 |
+
β on-screen composite + gr.File (PDF, analysis.md)
|
| 127 |
+
```
|
| 128 |
+
|
| 129 |
+
## Error handling
|
| 130 |
+
|
| 131 |
+
- Key-frame render fails β entry still saved; PDF shows an image placeholder.
|
| 132 |
+
- PDF generation fails β surface the error, keep the session intact for retry.
|
| 133 |
+
- `needs_human` entry β no numeric score; PDF shows "Clinician review required"; composite null.
|
| 134 |
+
- Composite is `None` whenever any test is unscored or needs human review (existing rule β never show a partial 0β21 as complete).
|
| 135 |
+
- All disk writes tolerate failure with a logged warning; a write failure degrades the artifact but never blocks scoring.
|
| 136 |
+
|
| 137 |
+
## Testing (must run without model downloads)
|
| 138 |
+
|
| 139 |
+
- `tests/test_pdf_report.py` β synthetic `ReportResult` + entries β PDF file created, non-zero size, contains the disclaimer text and composite line.
|
| 140 |
+
- `tests/test_session.py` β accumulation; composite math; bilateral lower-score + asymmetry delta; null composite when one entry `needs_human`.
|
| 141 |
+
- `tests/test_keyframe.py` β `render_frame` returns a real PNG path (file exists) for a synthetic frame; returns `None` gracefully on bad input.
|
| 142 |
+
|
| 143 |
+
## Invariants preserved
|
| 144 |
+
|
| 145 |
+
- Pipeline stays headless β no Gradio imports in agent files (`PdfReportAgent` is a pure agent; key-frame capture stays in `visualizer.py`, the existing UI-layer component).
|
| 146 |
+
- Safety disclaimer present top and bottom of the PDF, mirroring the UI.
|
| 147 |
+
- Pain / clearing / needs-human is never auto-scored; composite null when any test unscored.
|
| 148 |
+
- New code follows the engineering standards: one public entrypoint per agent, typed dataclass I/O, `confidence`/`notes` where applicable, module docstring stating purpose/inputs/outputs/failure/params/license/gated.
|
| 149 |
+
|
| 150 |
+
## Open implementation choices (left to the plan)
|
| 151 |
+
|
| 152 |
+
- Exact `SessionEntry` β `ReportAgent` dict adapter shape.
|
| 153 |
+
- Which measurements to surface per test in the PDF (a curated subset, not the full `angles` dump).
|
| 154 |
+
- PDF assertion strategy in tests (text extraction vs. size/smoke).
|
formscout/__init__.py
ADDED
|
File without changes
|
formscout/agents/__init__.py
ADDED
|
File without changes
|
formscout/agents/biomechanics.py
ADDED
|
@@ -0,0 +1,608 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
BiomechanicsAgent β extracts named, documented, unit-bearing measurements from pose data.
|
| 3 |
+
|
| 4 |
+
Input: Pose2DResult (or Body3DResult if used), MovementResult
|
| 5 |
+
Output: BiomechFeatures(test_name, view, angles, alignments, ...)
|
| 6 |
+
Failure: returns BiomechFeatures with confidence=0.0 and notes.
|
| 7 |
+
Params: 0 (pure computation β no model).
|
| 8 |
+
License: n/a.
|
| 9 |
+
Gated: no.
|
| 10 |
+
|
| 11 |
+
This module is MEASUREMENT ONLY β no scoring happens here.
|
| 12 |
+
Scoring is done by the rubric functions in formscout/rubric/.
|
| 13 |
+
"""
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import math
|
| 17 |
+
from typing import Any
|
| 18 |
+
|
| 19 |
+
from formscout.types import (
|
| 20 |
+
Pose2DResult, Body3DResult, MovementResult, BiomechFeatures,
|
| 21 |
+
)
|
| 22 |
+
from formscout import config
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def _angle_between_points(a: tuple, b: tuple, c: tuple) -> float:
|
| 26 |
+
"""
|
| 27 |
+
Compute angle at point b formed by segments ba and bc.
|
| 28 |
+
Returns degrees. Returns NaN if any point is missing.
|
| 29 |
+
"""
|
| 30 |
+
try:
|
| 31 |
+
ba = (a[0] - b[0], a[1] - b[1])
|
| 32 |
+
bc = (c[0] - b[0], c[1] - b[1])
|
| 33 |
+
dot = ba[0] * bc[0] + ba[1] * bc[1]
|
| 34 |
+
mag_ba = math.sqrt(ba[0] ** 2 + ba[1] ** 2)
|
| 35 |
+
mag_bc = math.sqrt(bc[0] ** 2 + bc[1] ** 2)
|
| 36 |
+
if mag_ba == 0 or mag_bc == 0:
|
| 37 |
+
return float("nan")
|
| 38 |
+
cos_angle = max(-1.0, min(1.0, dot / (mag_ba * mag_bc)))
|
| 39 |
+
return math.degrees(math.acos(cos_angle))
|
| 40 |
+
except (TypeError, IndexError, ZeroDivisionError):
|
| 41 |
+
return float("nan")
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def _get_joint(keypoints: dict, joint_id: int) -> tuple | None:
|
| 45 |
+
"""Extract (x, y) for a joint, or None if missing/low-confidence."""
|
| 46 |
+
j = keypoints.get(joint_id)
|
| 47 |
+
if j is None:
|
| 48 |
+
return None
|
| 49 |
+
if j.get("conf", 0) < config.POSE_CONF_THRESHOLD:
|
| 50 |
+
return None
|
| 51 |
+
return (j["x"], j["y"])
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
# COCO joint indices
|
| 55 |
+
NOSE, L_EYE, R_EYE, L_EAR, R_EAR = 0, 1, 2, 3, 4
|
| 56 |
+
L_SHOULDER, R_SHOULDER = 5, 6
|
| 57 |
+
L_ELBOW, R_ELBOW = 7, 8
|
| 58 |
+
L_WRIST, R_WRIST = 9, 10
|
| 59 |
+
L_HIP, R_HIP = 11, 12
|
| 60 |
+
L_KNEE, R_KNEE = 13, 14
|
| 61 |
+
L_ANKLE, R_ANKLE = 15, 16
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
class BiomechanicsAgent:
|
| 65 |
+
"""Pure-function biomechanics measurement β no model calls."""
|
| 66 |
+
|
| 67 |
+
def run(
|
| 68 |
+
self,
|
| 69 |
+
pose2d: Pose2DResult,
|
| 70 |
+
body3d: Body3DResult,
|
| 71 |
+
movement: MovementResult,
|
| 72 |
+
) -> BiomechFeatures:
|
| 73 |
+
if not pose2d.keypoints:
|
| 74 |
+
return BiomechFeatures(
|
| 75 |
+
test_name=movement.test_name,
|
| 76 |
+
view="2d",
|
| 77 |
+
side=movement.side,
|
| 78 |
+
angles={}, alignments={},
|
| 79 |
+
symmetry_delta=None, timing={},
|
| 80 |
+
confidence=0.0,
|
| 81 |
+
notes="no keypoints available",
|
| 82 |
+
)
|
| 83 |
+
|
| 84 |
+
view = "3d" if body3d.used else "2d"
|
| 85 |
+
|
| 86 |
+
dispatch = {
|
| 87 |
+
"deep_squat": self._deep_squat,
|
| 88 |
+
"hurdle_step": self._hurdle_step,
|
| 89 |
+
"inline_lunge": self._inline_lunge,
|
| 90 |
+
"shoulder_mobility": self._shoulder_mobility,
|
| 91 |
+
"active_slr": self._active_slr,
|
| 92 |
+
"trunk_stability_pushup": self._trunk_stability_pushup,
|
| 93 |
+
"rotary_stability": self._rotary_stability,
|
| 94 |
+
}
|
| 95 |
+
fn = dispatch.get(movement.test_name)
|
| 96 |
+
if fn is None:
|
| 97 |
+
return BiomechFeatures(
|
| 98 |
+
test_name=movement.test_name, view=view, side=movement.side,
|
| 99 |
+
angles={}, alignments={}, symmetry_delta=None, timing={},
|
| 100 |
+
confidence=0.0, notes=f"unknown test: {movement.test_name}",
|
| 101 |
+
)
|
| 102 |
+
return fn(pose2d, view, movement.side)
|
| 103 |
+
|
| 104 |
+
def _deep_squat(self, pose2d: Pose2DResult, view: str, side: str) -> BiomechFeatures:
|
| 105 |
+
"""Extract deep squat biomechanics from the deepest frame."""
|
| 106 |
+
# Find the frame with lowest hip Y (deepest squat position)
|
| 107 |
+
best_frame_idx = 0
|
| 108 |
+
lowest_hip_y = -1.0
|
| 109 |
+
for i, kps in enumerate(pose2d.keypoints):
|
| 110 |
+
l_hip = _get_joint(kps, L_HIP)
|
| 111 |
+
r_hip = _get_joint(kps, R_HIP)
|
| 112 |
+
if l_hip and r_hip:
|
| 113 |
+
mid_hip_y = (l_hip[1] + r_hip[1]) / 2
|
| 114 |
+
if mid_hip_y > lowest_hip_y: # higher Y = lower in image
|
| 115 |
+
lowest_hip_y = mid_hip_y
|
| 116 |
+
best_frame_idx = i
|
| 117 |
+
|
| 118 |
+
kps = pose2d.keypoints[best_frame_idx]
|
| 119 |
+
notes_parts: list[str] = []
|
| 120 |
+
|
| 121 |
+
# Extract joints
|
| 122 |
+
l_hip = _get_joint(kps, L_HIP)
|
| 123 |
+
r_hip = _get_joint(kps, R_HIP)
|
| 124 |
+
l_knee = _get_joint(kps, L_KNEE)
|
| 125 |
+
r_knee = _get_joint(kps, R_KNEE)
|
| 126 |
+
l_ankle = _get_joint(kps, L_ANKLE)
|
| 127 |
+
r_ankle = _get_joint(kps, R_ANKLE)
|
| 128 |
+
l_shoulder = _get_joint(kps, L_SHOULDER)
|
| 129 |
+
r_shoulder = _get_joint(kps, R_SHOULDER)
|
| 130 |
+
|
| 131 |
+
# Compute angles
|
| 132 |
+
angles: dict[str, float] = {}
|
| 133 |
+
|
| 134 |
+
# Hip-knee-ankle angle (knee flexion) β average of both sides
|
| 135 |
+
l_knee_angle = _angle_between_points(l_hip, l_knee, l_ankle) if all([l_hip, l_knee, l_ankle]) else float("nan")
|
| 136 |
+
r_knee_angle = _angle_between_points(r_hip, r_knee, r_ankle) if all([r_hip, r_knee, r_ankle]) else float("nan")
|
| 137 |
+
|
| 138 |
+
if not math.isnan(l_knee_angle):
|
| 139 |
+
angles["left_knee_flexion_deg"] = l_knee_angle
|
| 140 |
+
else:
|
| 141 |
+
notes_parts.append("left knee angle unavailable")
|
| 142 |
+
|
| 143 |
+
if not math.isnan(r_knee_angle):
|
| 144 |
+
angles["right_knee_flexion_deg"] = r_knee_angle
|
| 145 |
+
else:
|
| 146 |
+
notes_parts.append("right knee angle unavailable")
|
| 147 |
+
|
| 148 |
+
# Femur angle from horizontal
|
| 149 |
+
# Femur = hip to knee. Angle from horizontal = atan2(dy, dx)
|
| 150 |
+
if l_hip and l_knee:
|
| 151 |
+
dy = l_knee[1] - l_hip[1]
|
| 152 |
+
dx = l_knee[0] - l_hip[0]
|
| 153 |
+
angles["left_femur_from_horizontal_deg"] = abs(math.degrees(math.atan2(dy, dx)))
|
| 154 |
+
if r_hip and r_knee:
|
| 155 |
+
dy = r_knee[1] - r_hip[1]
|
| 156 |
+
dx = r_knee[0] - r_hip[0]
|
| 157 |
+
angles["right_femur_from_horizontal_deg"] = abs(math.degrees(math.atan2(dy, dx)))
|
| 158 |
+
|
| 159 |
+
# Torso-tibia angle (torso parallel to tibia = score 3 criterion)
|
| 160 |
+
if l_shoulder and l_hip and l_knee and l_ankle:
|
| 161 |
+
torso_angle = math.degrees(math.atan2(l_hip[1] - l_shoulder[1], l_hip[0] - l_shoulder[0]))
|
| 162 |
+
tibia_angle = math.degrees(math.atan2(l_ankle[1] - l_knee[1], l_ankle[0] - l_knee[0]))
|
| 163 |
+
angles["torso_tibia_angle_deg"] = abs(torso_angle - tibia_angle)
|
| 164 |
+
|
| 165 |
+
# Alignments
|
| 166 |
+
alignments: dict[str, Any] = {}
|
| 167 |
+
|
| 168 |
+
# Knee valgus check: are knees inside the ankle line?
|
| 169 |
+
if l_knee and r_knee and l_ankle and r_ankle:
|
| 170 |
+
knee_width = abs(l_knee[0] - r_knee[0])
|
| 171 |
+
ankle_width = abs(l_ankle[0] - r_ankle[0])
|
| 172 |
+
alignments["knees_tracking_over_feet"] = knee_width >= (ankle_width - config.DEEP_SQUAT_KNEE_TRACKING_MARGIN_PX)
|
| 173 |
+
alignments["knee_valgus_deg"] = 0.0 # placeholder for actual valgus angle
|
| 174 |
+
|
| 175 |
+
# Heels elevated detection (approximation: ankle Y relative to frame bottom)
|
| 176 |
+
# This is a rough heuristic β proper detection needs foot keypoints or depth
|
| 177 |
+
alignments["heels_elevated"] = False # default; refine with better detection
|
| 178 |
+
|
| 179 |
+
# Dowel position (need wrist positions relative to feet)
|
| 180 |
+
if l_wrist := _get_joint(kps, L_WRIST):
|
| 181 |
+
if r_wrist := _get_joint(kps, R_WRIST):
|
| 182 |
+
if l_ankle and r_ankle:
|
| 183 |
+
mid_wrist_x = (l_wrist[0] + r_wrist[0]) / 2
|
| 184 |
+
mid_ankle_x = (l_ankle[0] + r_ankle[0]) / 2
|
| 185 |
+
alignments["dowel_over_feet"] = abs(mid_wrist_x - mid_ankle_x) < 50
|
| 186 |
+
alignments["dowel_feet_offset_px"] = mid_wrist_x - mid_ankle_x
|
| 187 |
+
|
| 188 |
+
# Confidence based on how many measurements we got
|
| 189 |
+
n_expected = 6 # main measurements
|
| 190 |
+
n_got = len(angles) + len([v for v in alignments.values() if v is not None])
|
| 191 |
+
confidence = min(1.0, n_got / n_expected) * pose2d.confidence
|
| 192 |
+
|
| 193 |
+
return BiomechFeatures(
|
| 194 |
+
test_name="deep_squat",
|
| 195 |
+
view=view,
|
| 196 |
+
side="na",
|
| 197 |
+
angles=angles,
|
| 198 |
+
alignments=alignments,
|
| 199 |
+
symmetry_delta=None,
|
| 200 |
+
timing={"deepest_frame": best_frame_idx},
|
| 201 |
+
confidence=confidence,
|
| 202 |
+
notes="; ".join(notes_parts) if notes_parts else "",
|
| 203 |
+
)
|
| 204 |
+
|
| 205 |
+
# βββ Helper: find peak frame by joint Y βββββββββββββββββββββββββββββββββ
|
| 206 |
+
|
| 207 |
+
def _find_peak_frame(self, pose2d: Pose2DResult, joint_id: int, maximize: bool = True) -> int:
|
| 208 |
+
"""Find frame where a joint reaches its extreme Y position."""
|
| 209 |
+
best_idx, best_val = 0, -1.0 if maximize else float("inf")
|
| 210 |
+
for i, kps in enumerate(pose2d.keypoints):
|
| 211 |
+
j = _get_joint(kps, joint_id)
|
| 212 |
+
if j:
|
| 213 |
+
if (maximize and j[1] > best_val) or (not maximize and j[1] < best_val):
|
| 214 |
+
best_val = j[1]
|
| 215 |
+
best_idx = i
|
| 216 |
+
return best_idx
|
| 217 |
+
|
| 218 |
+
def _bilateral_features(
|
| 219 |
+
self, pose2d: Pose2DResult, view: str, side: str, test_name: str,
|
| 220 |
+
extractor,
|
| 221 |
+
) -> BiomechFeatures:
|
| 222 |
+
"""Run a bilateral test: compute both sides, report the specified side + symmetry_delta."""
|
| 223 |
+
left = extractor(pose2d, "left")
|
| 224 |
+
right = extractor(pose2d, "right")
|
| 225 |
+
|
| 226 |
+
# Pick the requested side as primary
|
| 227 |
+
primary = left if side == "left" else right if side == "right" else left
|
| 228 |
+
other = right if side == "left" else left if side == "right" else right
|
| 229 |
+
|
| 230 |
+
# Merge angles with side prefix for the primary
|
| 231 |
+
angles = primary.get("angles", {})
|
| 232 |
+
alignments = primary.get("alignments", {})
|
| 233 |
+
timing = primary.get("timing", {})
|
| 234 |
+
|
| 235 |
+
# Compute symmetry delta from the main measurement
|
| 236 |
+
main_key = primary.get("main_measure_key")
|
| 237 |
+
sym_delta = None
|
| 238 |
+
if main_key and main_key in left.get("angles", {}) and main_key in right.get("angles", {}):
|
| 239 |
+
sym_delta = abs(left["angles"][main_key] - right["angles"][main_key])
|
| 240 |
+
|
| 241 |
+
n_got = len(angles) + len([v for v in alignments.values() if v is not None])
|
| 242 |
+
confidence = min(1.0, n_got / max(primary.get("expected", 3), 1)) * pose2d.confidence
|
| 243 |
+
|
| 244 |
+
return BiomechFeatures(
|
| 245 |
+
test_name=test_name, view=view, side=side,
|
| 246 |
+
angles=angles, alignments=alignments,
|
| 247 |
+
symmetry_delta=sym_delta, timing=timing,
|
| 248 |
+
confidence=confidence,
|
| 249 |
+
notes=primary.get("notes", ""),
|
| 250 |
+
)
|
| 251 |
+
|
| 252 |
+
# βββ Hurdle Step βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 253 |
+
|
| 254 |
+
def _hurdle_step(self, pose2d: Pose2DResult, view: str, side: str) -> BiomechFeatures:
|
| 255 |
+
"""Hurdle Step: hip/knee flexion of stepping leg, stance stability."""
|
| 256 |
+
def extract(p2d: Pose2DResult, s: str) -> dict:
|
| 257 |
+
hip_id = L_HIP if s == "left" else R_HIP
|
| 258 |
+
knee_id = L_KNEE if s == "left" else R_KNEE
|
| 259 |
+
ankle_id = L_ANKLE if s == "left" else R_ANKLE
|
| 260 |
+
# Stance side is opposite
|
| 261 |
+
stance_hip = R_HIP if s == "left" else L_HIP
|
| 262 |
+
stance_knee = R_KNEE if s == "left" else L_KNEE
|
| 263 |
+
stance_ankle = R_ANKLE if s == "left" else L_ANKLE
|
| 264 |
+
|
| 265 |
+
# Peak = frame where stepping knee is highest (lowest Y in image)
|
| 266 |
+
peak_idx = self._find_peak_frame(p2d, knee_id, maximize=False)
|
| 267 |
+
kps = p2d.keypoints[peak_idx]
|
| 268 |
+
|
| 269 |
+
hip = _get_joint(kps, hip_id)
|
| 270 |
+
knee = _get_joint(kps, knee_id)
|
| 271 |
+
ankle = _get_joint(kps, ankle_id)
|
| 272 |
+
s_hip = _get_joint(kps, stance_hip)
|
| 273 |
+
s_knee = _get_joint(kps, stance_knee)
|
| 274 |
+
s_ankle = _get_joint(kps, stance_ankle)
|
| 275 |
+
|
| 276 |
+
angles = {}
|
| 277 |
+
alignments = {}
|
| 278 |
+
notes_parts = []
|
| 279 |
+
|
| 280 |
+
# Hip flexion of stepping leg
|
| 281 |
+
if all([hip, knee, ankle]):
|
| 282 |
+
angles["step_knee_flexion_deg"] = _angle_between_points(hip, knee, ankle)
|
| 283 |
+
# Hip angle (torso-femur)
|
| 284 |
+
shoulder_id = L_SHOULDER if s == "left" else R_SHOULDER
|
| 285 |
+
shoulder = _get_joint(kps, shoulder_id)
|
| 286 |
+
if all([shoulder, hip, knee]):
|
| 287 |
+
angles["step_hip_flexion_deg"] = _angle_between_points(shoulder, hip, knee)
|
| 288 |
+
|
| 289 |
+
# Stance knee should stay extended
|
| 290 |
+
if all([s_hip, s_knee, s_ankle]):
|
| 291 |
+
angles["stance_knee_angle_deg"] = _angle_between_points(s_hip, s_knee, s_ankle)
|
| 292 |
+
alignments["stance_knee_extended"] = angles["stance_knee_angle_deg"] > 160
|
| 293 |
+
|
| 294 |
+
# Lateral trunk lean: shoulders should be level
|
| 295 |
+
l_sh = _get_joint(kps, L_SHOULDER)
|
| 296 |
+
r_sh = _get_joint(kps, R_SHOULDER)
|
| 297 |
+
if l_sh and r_sh:
|
| 298 |
+
angles["shoulder_tilt_deg"] = abs(math.degrees(
|
| 299 |
+
math.atan2(r_sh[1] - l_sh[1], r_sh[0] - l_sh[0])
|
| 300 |
+
))
|
| 301 |
+
alignments["trunk_stable"] = angles["shoulder_tilt_deg"] < 10
|
| 302 |
+
|
| 303 |
+
return {
|
| 304 |
+
"angles": angles, "alignments": alignments,
|
| 305 |
+
"timing": {"peak_step_frame": peak_idx},
|
| 306 |
+
"main_measure_key": "step_hip_flexion_deg",
|
| 307 |
+
"expected": 4, "notes": "; ".join(notes_parts),
|
| 308 |
+
}
|
| 309 |
+
|
| 310 |
+
return self._bilateral_features(pose2d, view, side, "hurdle_step", extract)
|
| 311 |
+
|
| 312 |
+
# βββ In-Line Lunge βββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 313 |
+
|
| 314 |
+
def _inline_lunge(self, pose2d: Pose2DResult, view: str, side: str) -> BiomechFeatures:
|
| 315 |
+
"""In-Line Lunge: knee flexion depth, trunk upright, balance."""
|
| 316 |
+
def extract(p2d: Pose2DResult, s: str) -> dict:
|
| 317 |
+
# Front leg is the assessed side
|
| 318 |
+
hip_id = L_HIP if s == "left" else R_HIP
|
| 319 |
+
knee_id = L_KNEE if s == "left" else R_KNEE
|
| 320 |
+
ankle_id = L_ANKLE if s == "left" else R_ANKLE
|
| 321 |
+
rear_knee_id = R_KNEE if s == "left" else L_KNEE
|
| 322 |
+
|
| 323 |
+
# Deepest lunge = front knee lowest
|
| 324 |
+
peak_idx = self._find_peak_frame(p2d, knee_id, maximize=True)
|
| 325 |
+
kps = p2d.keypoints[peak_idx]
|
| 326 |
+
|
| 327 |
+
hip = _get_joint(kps, hip_id)
|
| 328 |
+
knee = _get_joint(kps, knee_id)
|
| 329 |
+
ankle = _get_joint(kps, ankle_id)
|
| 330 |
+
l_sh = _get_joint(kps, L_SHOULDER)
|
| 331 |
+
r_sh = _get_joint(kps, R_SHOULDER)
|
| 332 |
+
l_hip = _get_joint(kps, L_HIP)
|
| 333 |
+
r_hip = _get_joint(kps, R_HIP)
|
| 334 |
+
|
| 335 |
+
angles = {}
|
| 336 |
+
alignments = {}
|
| 337 |
+
|
| 338 |
+
# Front knee flexion
|
| 339 |
+
if all([hip, knee, ankle]):
|
| 340 |
+
angles["front_knee_flexion_deg"] = _angle_between_points(hip, knee, ankle)
|
| 341 |
+
|
| 342 |
+
# Trunk upright: midline shoulder-to-hip angle from vertical
|
| 343 |
+
if l_sh and r_sh and l_hip and r_hip:
|
| 344 |
+
mid_sh = ((l_sh[0] + r_sh[0]) / 2, (l_sh[1] + r_sh[1]) / 2)
|
| 345 |
+
mid_hip = ((l_hip[0] + r_hip[0]) / 2, (l_hip[1] + r_hip[1]) / 2)
|
| 346 |
+
trunk_from_vert = abs(math.degrees(
|
| 347 |
+
math.atan2(mid_hip[0] - mid_sh[0], mid_sh[1] - mid_hip[1])
|
| 348 |
+
))
|
| 349 |
+
angles["trunk_lean_from_vertical_deg"] = trunk_from_vert
|
| 350 |
+
alignments["trunk_upright"] = trunk_from_vert < 15
|
| 351 |
+
|
| 352 |
+
# Knee over ankle alignment
|
| 353 |
+
if knee and ankle:
|
| 354 |
+
alignments["knee_over_ankle"] = abs(knee[0] - ankle[0]) < 40
|
| 355 |
+
|
| 356 |
+
return {
|
| 357 |
+
"angles": angles, "alignments": alignments,
|
| 358 |
+
"timing": {"deepest_lunge_frame": peak_idx},
|
| 359 |
+
"main_measure_key": "front_knee_flexion_deg",
|
| 360 |
+
"expected": 3, "notes": "",
|
| 361 |
+
}
|
| 362 |
+
|
| 363 |
+
return self._bilateral_features(pose2d, view, side, "inline_lunge", extract)
|
| 364 |
+
|
| 365 |
+
# βββ Shoulder Mobility βββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 366 |
+
|
| 367 |
+
def _shoulder_mobility(self, pose2d: Pose2DResult, view: str, side: str) -> BiomechFeatures:
|
| 368 |
+
"""Shoulder Mobility: inter-fist distance normalized to hand length."""
|
| 369 |
+
def extract(p2d: Pose2DResult, s: str) -> dict:
|
| 370 |
+
# "side" = the hand reaching over (top hand)
|
| 371 |
+
top_wrist = L_WRIST if s == "left" else R_WRIST
|
| 372 |
+
bot_wrist = R_WRIST if s == "left" else L_WRIST
|
| 373 |
+
|
| 374 |
+
# Use mid-sequence frame (static hold)
|
| 375 |
+
mid_idx = len(p2d.keypoints) // 2
|
| 376 |
+
kps = p2d.keypoints[mid_idx]
|
| 377 |
+
|
| 378 |
+
top_w = _get_joint(kps, top_wrist)
|
| 379 |
+
bot_w = _get_joint(kps, bot_wrist)
|
| 380 |
+
|
| 381 |
+
angles = {}
|
| 382 |
+
alignments = {}
|
| 383 |
+
|
| 384 |
+
if top_w and bot_w:
|
| 385 |
+
# Vertical distance between fists (normalized by torso length)
|
| 386 |
+
fist_dist_px = math.sqrt((top_w[0] - bot_w[0])**2 + (top_w[1] - bot_w[1])**2)
|
| 387 |
+
angles["inter_fist_distance_px"] = fist_dist_px
|
| 388 |
+
|
| 389 |
+
# Normalize by torso length (shoulder to hip)
|
| 390 |
+
sh_id = L_SHOULDER if s == "left" else R_SHOULDER
|
| 391 |
+
hip_id = L_HIP if s == "left" else R_HIP
|
| 392 |
+
sh = _get_joint(kps, sh_id)
|
| 393 |
+
hip = _get_joint(kps, hip_id)
|
| 394 |
+
if sh and hip:
|
| 395 |
+
torso_len = math.sqrt((sh[0] - hip[0])**2 + (sh[1] - hip[1])**2)
|
| 396 |
+
if torso_len > 0:
|
| 397 |
+
norm_dist = fist_dist_px / torso_len
|
| 398 |
+
angles["inter_fist_normalized"] = norm_dist
|
| 399 |
+
# Score 3: fists within 1 hand-length (~0.3 torso)
|
| 400 |
+
# Score 2: within 1.5 hand-lengths
|
| 401 |
+
alignments["fists_within_one_hand"] = norm_dist < 0.35
|
| 402 |
+
alignments["fists_within_1_5_hand"] = norm_dist < 0.55
|
| 403 |
+
|
| 404 |
+
return {
|
| 405 |
+
"angles": angles, "alignments": alignments,
|
| 406 |
+
"timing": {"measure_frame": mid_idx},
|
| 407 |
+
"main_measure_key": "inter_fist_normalized",
|
| 408 |
+
"expected": 2, "notes": "",
|
| 409 |
+
}
|
| 410 |
+
|
| 411 |
+
return self._bilateral_features(pose2d, view, side, "shoulder_mobility", extract)
|
| 412 |
+
|
| 413 |
+
# βββ Active Straight-Leg Raise βββββββββββββββββββββββββββββββββββββββββββ
|
| 414 |
+
|
| 415 |
+
def _active_slr(self, pose2d: Pose2DResult, view: str, side: str) -> BiomechFeatures:
|
| 416 |
+
"""ASLR: hip flexion angle of raised leg; down-leg stays flat."""
|
| 417 |
+
def extract(p2d: Pose2DResult, s: str) -> dict:
|
| 418 |
+
hip_id = L_HIP if s == "left" else R_HIP
|
| 419 |
+
knee_id = L_KNEE if s == "left" else R_KNEE
|
| 420 |
+
ankle_id = L_ANKLE if s == "left" else R_ANKLE
|
| 421 |
+
# Down leg
|
| 422 |
+
d_hip_id = R_HIP if s == "left" else L_HIP
|
| 423 |
+
d_knee_id = R_KNEE if s == "left" else L_KNEE
|
| 424 |
+
d_ankle_id = R_ANKLE if s == "left" else L_ANKLE
|
| 425 |
+
|
| 426 |
+
# Peak = raised ankle at highest point (lowest Y)
|
| 427 |
+
peak_idx = self._find_peak_frame(p2d, ankle_id, maximize=False)
|
| 428 |
+
kps = p2d.keypoints[peak_idx]
|
| 429 |
+
|
| 430 |
+
hip = _get_joint(kps, hip_id)
|
| 431 |
+
knee = _get_joint(kps, knee_id)
|
| 432 |
+
ankle = _get_joint(kps, ankle_id)
|
| 433 |
+
d_hip = _get_joint(kps, d_hip_id)
|
| 434 |
+
d_knee = _get_joint(kps, d_knee_id)
|
| 435 |
+
d_ankle = _get_joint(kps, d_ankle_id)
|
| 436 |
+
|
| 437 |
+
angles = {}
|
| 438 |
+
alignments = {}
|
| 439 |
+
|
| 440 |
+
# Raised leg hip flexion: angle of femur from horizontal
|
| 441 |
+
if hip and ankle:
|
| 442 |
+
dy = hip[1] - ankle[1] # positive = ankle above hip
|
| 443 |
+
dx = ankle[0] - hip[0]
|
| 444 |
+
hip_flex = math.degrees(math.atan2(dy, abs(dx) if abs(dx) > 1 else 1))
|
| 445 |
+
angles["raised_leg_angle_deg"] = max(0, hip_flex)
|
| 446 |
+
# Score 3: malleolus past contralateral knee (>70Β°)
|
| 447 |
+
# Score 2: between contralateral knee and mid-thigh (45-70Β°)
|
| 448 |
+
alignments["past_contralateral_knee"] = hip_flex > 70
|
| 449 |
+
alignments["past_mid_thigh"] = hip_flex > 45
|
| 450 |
+
|
| 451 |
+
# Down leg: should stay flat (knee angle ~180)
|
| 452 |
+
if all([d_hip, d_knee, d_ankle]):
|
| 453 |
+
down_knee_angle = _angle_between_points(d_hip, d_knee, d_ankle)
|
| 454 |
+
angles["down_leg_knee_angle_deg"] = down_knee_angle
|
| 455 |
+
alignments["down_leg_flat"] = down_knee_angle > 160
|
| 456 |
+
|
| 457 |
+
return {
|
| 458 |
+
"angles": angles, "alignments": alignments,
|
| 459 |
+
"timing": {"peak_raise_frame": peak_idx},
|
| 460 |
+
"main_measure_key": "raised_leg_angle_deg",
|
| 461 |
+
"expected": 3, "notes": "",
|
| 462 |
+
}
|
| 463 |
+
|
| 464 |
+
return self._bilateral_features(pose2d, view, side, "active_slr", extract)
|
| 465 |
+
|
| 466 |
+
# βββ Trunk Stability Push-Up βββββββββββββββββββββββββββββββββββββββββββββ
|
| 467 |
+
|
| 468 |
+
def _trunk_stability_pushup(self, pose2d: Pose2DResult, view: str, side: str) -> BiomechFeatures:
|
| 469 |
+
"""Trunk Stability Push-Up: body rigidity through the press."""
|
| 470 |
+
angles = {}
|
| 471 |
+
alignments = {}
|
| 472 |
+
notes_parts = []
|
| 473 |
+
|
| 474 |
+
# Analyze multiple frames to detect sag/lag
|
| 475 |
+
trunk_sags: list[tuple[int, float]] = [] # (frame_idx, sag_px)
|
| 476 |
+
for i, kps in enumerate(pose2d.keypoints):
|
| 477 |
+
l_sh = _get_joint(kps, L_SHOULDER)
|
| 478 |
+
r_sh = _get_joint(kps, R_SHOULDER)
|
| 479 |
+
l_hip = _get_joint(kps, L_HIP)
|
| 480 |
+
r_hip = _get_joint(kps, R_HIP)
|
| 481 |
+
l_ankle = _get_joint(kps, L_ANKLE)
|
| 482 |
+
r_ankle = _get_joint(kps, R_ANKLE)
|
| 483 |
+
|
| 484 |
+
if l_sh and r_sh and l_hip and r_hip and l_ankle and r_ankle:
|
| 485 |
+
# Sag = hip drops below shoulder-ankle line
|
| 486 |
+
sh_y = (l_sh[1] + r_sh[1]) / 2
|
| 487 |
+
hip_y = (l_hip[1] + r_hip[1]) / 2
|
| 488 |
+
ankle_y = (l_ankle[1] + r_ankle[1]) / 2
|
| 489 |
+
# In image coords: sag = hip_y > midpoint of shoulder-ankle Y
|
| 490 |
+
expected_hip_y = (sh_y + ankle_y) / 2
|
| 491 |
+
sag_px = hip_y - expected_hip_y
|
| 492 |
+
trunk_sags.append((i, sag_px))
|
| 493 |
+
|
| 494 |
+
max_sag_frame = 0
|
| 495 |
+
if trunk_sags:
|
| 496 |
+
sags = [s for _, s in trunk_sags]
|
| 497 |
+
max_sag_frame = max(trunk_sags, key=lambda t: t[1])[0]
|
| 498 |
+
mean = sum(sags) / len(sags)
|
| 499 |
+
variance = (sum((x - mean) ** 2 for x in sags) / len(sags)) ** 0.5
|
| 500 |
+
max_sag = max(sags)
|
| 501 |
+
angles["max_sag_px"] = max_sag
|
| 502 |
+
angles["trunk_variance_px"] = variance
|
| 503 |
+
alignments["body_rigid"] = max_sag < 30 and variance < 15
|
| 504 |
+
alignments["no_sag"] = max_sag < 30
|
| 505 |
+
else:
|
| 506 |
+
notes_parts.append("insufficient landmarks for trunk analysis")
|
| 507 |
+
|
| 508 |
+
# Hand position (near head = harder = score 3 position)
|
| 509 |
+
if pose2d.keypoints:
|
| 510 |
+
mid_kps = pose2d.keypoints[0]
|
| 511 |
+
nose = _get_joint(mid_kps, NOSE)
|
| 512 |
+
l_w = _get_joint(mid_kps, L_WRIST)
|
| 513 |
+
r_w = _get_joint(mid_kps, R_WRIST)
|
| 514 |
+
if nose and l_w and r_w:
|
| 515 |
+
avg_wrist_y = (l_w[1] + r_w[1]) / 2
|
| 516 |
+
# Hands near head = wrist Y close to nose Y
|
| 517 |
+
alignments["hands_at_forehead"] = abs(avg_wrist_y - nose[1]) < 50
|
| 518 |
+
|
| 519 |
+
n_got = len(angles) + len([v for v in alignments.values() if v is not None])
|
| 520 |
+
confidence = min(1.0, n_got / 3) * pose2d.confidence
|
| 521 |
+
|
| 522 |
+
return BiomechFeatures(
|
| 523 |
+
test_name="trunk_stability_pushup", view=view, side="na",
|
| 524 |
+
angles=angles, alignments=alignments,
|
| 525 |
+
symmetry_delta=None,
|
| 526 |
+
timing={"n_frames_analyzed": len(trunk_sags), "max_sag_frame": max_sag_frame},
|
| 527 |
+
confidence=confidence,
|
| 528 |
+
notes="; ".join(notes_parts) if notes_parts else "",
|
| 529 |
+
)
|
| 530 |
+
|
| 531 |
+
# βββ Rotary Stability ββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 532 |
+
|
| 533 |
+
def _rotary_stability(self, pose2d: Pose2DResult, view: str, side: str) -> BiomechFeatures:
|
| 534 |
+
"""Rotary Stability: coordination of ipsilateral arm/leg extension."""
|
| 535 |
+
angles = {}
|
| 536 |
+
alignments = {}
|
| 537 |
+
notes_parts = []
|
| 538 |
+
|
| 539 |
+
# Look for the frame with max arm+leg extension
|
| 540 |
+
# Quadruped: hands + knees on ground, extending one arm + one leg
|
| 541 |
+
best_ext_frame = 0
|
| 542 |
+
best_ext_val = 0
|
| 543 |
+
|
| 544 |
+
for i, kps in enumerate(pose2d.keypoints):
|
| 545 |
+
l_w = _get_joint(kps, L_WRIST)
|
| 546 |
+
r_w = _get_joint(kps, R_WRIST)
|
| 547 |
+
l_a = _get_joint(kps, L_ANKLE)
|
| 548 |
+
r_a = _get_joint(kps, R_ANKLE)
|
| 549 |
+
l_sh = _get_joint(kps, L_SHOULDER)
|
| 550 |
+
r_sh = _get_joint(kps, R_SHOULDER)
|
| 551 |
+
|
| 552 |
+
# Extension = distance of wrist from shoulder + ankle from hip
|
| 553 |
+
ext_val = 0
|
| 554 |
+
if l_w and l_sh:
|
| 555 |
+
ext_val += abs(l_w[0] - l_sh[0])
|
| 556 |
+
if r_w and r_sh:
|
| 557 |
+
ext_val += abs(r_w[0] - r_sh[0])
|
| 558 |
+
if ext_val > best_ext_val:
|
| 559 |
+
best_ext_val = ext_val
|
| 560 |
+
best_ext_frame = i
|
| 561 |
+
|
| 562 |
+
kps = pose2d.keypoints[best_ext_frame] if pose2d.keypoints else {}
|
| 563 |
+
|
| 564 |
+
# Trunk stability: shoulders level, hips level
|
| 565 |
+
l_sh = _get_joint(kps, L_SHOULDER)
|
| 566 |
+
r_sh = _get_joint(kps, R_SHOULDER)
|
| 567 |
+
l_hip = _get_joint(kps, L_HIP)
|
| 568 |
+
r_hip = _get_joint(kps, R_HIP)
|
| 569 |
+
|
| 570 |
+
if l_sh and r_sh:
|
| 571 |
+
sh_tilt = abs(l_sh[1] - r_sh[1])
|
| 572 |
+
angles["shoulder_level_diff_px"] = sh_tilt
|
| 573 |
+
alignments["shoulders_level"] = sh_tilt < 20
|
| 574 |
+
|
| 575 |
+
if l_hip and r_hip:
|
| 576 |
+
hip_tilt = abs(l_hip[1] - r_hip[1])
|
| 577 |
+
angles["hip_level_diff_px"] = hip_tilt
|
| 578 |
+
alignments["hips_level"] = hip_tilt < 20
|
| 579 |
+
|
| 580 |
+
# Check for trunk sag across frames (similar to pushup)
|
| 581 |
+
trunk_variance = []
|
| 582 |
+
for kps_frame in pose2d.keypoints:
|
| 583 |
+
ls = _get_joint(kps_frame, L_SHOULDER)
|
| 584 |
+
rs = _get_joint(kps_frame, R_SHOULDER)
|
| 585 |
+
lh = _get_joint(kps_frame, L_HIP)
|
| 586 |
+
rh = _get_joint(kps_frame, R_HIP)
|
| 587 |
+
if ls and rs and lh and rh:
|
| 588 |
+
mid_sh_y = (ls[1] + rs[1]) / 2
|
| 589 |
+
mid_hip_y = (lh[1] + rh[1]) / 2
|
| 590 |
+
trunk_variance.append(mid_hip_y - mid_sh_y)
|
| 591 |
+
|
| 592 |
+
if trunk_variance:
|
| 593 |
+
std = (sum((x - sum(trunk_variance) / len(trunk_variance))**2
|
| 594 |
+
for x in trunk_variance) / len(trunk_variance)) ** 0.5
|
| 595 |
+
angles["trunk_stability_std_px"] = std
|
| 596 |
+
alignments["trunk_stable"] = std < 15
|
| 597 |
+
|
| 598 |
+
n_got = len(angles) + len([v for v in alignments.values() if v is not None])
|
| 599 |
+
confidence = min(1.0, n_got / 3) * pose2d.confidence
|
| 600 |
+
|
| 601 |
+
return BiomechFeatures(
|
| 602 |
+
test_name="rotary_stability", view=view, side="na",
|
| 603 |
+
angles=angles, alignments=alignments,
|
| 604 |
+
symmetry_delta=None,
|
| 605 |
+
timing={"peak_extension_frame": best_ext_frame},
|
| 606 |
+
confidence=confidence,
|
| 607 |
+
notes="; ".join(notes_parts) if notes_parts else "",
|
| 608 |
+
)
|
formscout/agents/body3d.py
ADDED
|
@@ -0,0 +1,221 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Body3DAgent β optional 3D mesh/joint angle recovery via SAM 3D Body.
|
| 3 |
+
|
| 4 |
+
Input: Pose2DResult, list of athlete masks, list of frames (np.ndarray BGR)
|
| 5 |
+
Output: Body3DResult(used, joints_3d, confidence)
|
| 6 |
+
Failure: ALWAYS returns Body3DResult(used=False) when enable_3d=False or
|
| 7 |
+
checkpoint unavailable β this is a normal success path, not an error.
|
| 8 |
+
Model: facebook/sam-3d-body-dinov3 (840M params, SAM License, GATED).
|
| 9 |
+
Gated: YES β access GRANTED June 4, 2026.
|
| 10 |
+
Params: ~0.84B (DINOv3-H+ variant).
|
| 11 |
+
|
| 12 |
+
API (verified from github.com/facebookresearch/sam-3d-body README, Jun 2026):
|
| 13 |
+
from notebook.utils import setup_sam_3d_body
|
| 14 |
+
estimator = setup_sam_3d_body(hf_repo_id="facebook/sam-3d-body-dinov3")
|
| 15 |
+
outputs = estimator.process_one_image(rgb_image) # single RGB np.ndarray
|
| 16 |
+
# outputs contains MHR joints, body mesh, etc.
|
| 17 |
+
"""
|
| 18 |
+
from __future__ import annotations
|
| 19 |
+
|
| 20 |
+
import numpy as np
|
| 21 |
+
|
| 22 |
+
from formscout.types import Pose2DResult, Body3DResult, IngestResult
|
| 23 |
+
from formscout import config
|
| 24 |
+
|
| 25 |
+
_NOT_USED = Body3DResult(
|
| 26 |
+
used=False, joints_3d=[], confidence=0.0,
|
| 27 |
+
notes="3D disabled or checkpoint unavailable",
|
| 28 |
+
)
|
| 29 |
+
|
| 30 |
+
# Subsample frames for 3D inference (expensive per-frame)
|
| 31 |
+
_MAX_3D_FRAMES = 30
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
class Body3DAgent:
|
| 35 |
+
"""
|
| 36 |
+
Optional 3D body joint estimation via SAM 3D Body (MHR rig).
|
| 37 |
+
Falls back gracefully when unavailable β returning Body3DResult(used=False)
|
| 38 |
+
is the expected success path for the 2D-only pipeline.
|
| 39 |
+
"""
|
| 40 |
+
|
| 41 |
+
def __init__(self, enable_3d: bool | None = None):
|
| 42 |
+
self._enabled = config.ENABLE_3D if enable_3d is None else enable_3d
|
| 43 |
+
self._estimator = None
|
| 44 |
+
if self._enabled:
|
| 45 |
+
self._estimator = self._try_load()
|
| 46 |
+
|
| 47 |
+
def _try_load(self):
|
| 48 |
+
"""
|
| 49 |
+
Attempt to load SAM 3D Body from HuggingFace.
|
| 50 |
+
Returns the estimator object or None on any failure.
|
| 51 |
+
"""
|
| 52 |
+
try:
|
| 53 |
+
from notebook.utils import setup_sam_3d_body # noqa: F401
|
| 54 |
+
estimator = setup_sam_3d_body(
|
| 55 |
+
hf_repo_id=config.SAM_3D_HF_REPO,
|
| 56 |
+
)
|
| 57 |
+
return estimator
|
| 58 |
+
except ImportError:
|
| 59 |
+
return None
|
| 60 |
+
except Exception:
|
| 61 |
+
return None
|
| 62 |
+
|
| 63 |
+
def run(
|
| 64 |
+
self,
|
| 65 |
+
pose2d: Pose2DResult,
|
| 66 |
+
masks: list,
|
| 67 |
+
frames: list | None = None,
|
| 68 |
+
) -> Body3DResult:
|
| 69 |
+
"""
|
| 70 |
+
Run 3D body estimation on selected keyframes.
|
| 71 |
+
|
| 72 |
+
Args:
|
| 73 |
+
pose2d: 2D pose results (used for confidence weighting)
|
| 74 |
+
masks: Per-frame athlete masks from SegmentationAgent
|
| 75 |
+
frames: Raw BGR frames from IngestResult.frames
|
| 76 |
+
|
| 77 |
+
Returns:
|
| 78 |
+
Body3DResult with used=True and 3D joints if successful,
|
| 79 |
+
or Body3DResult(used=False) if disabled/unavailable (normal path).
|
| 80 |
+
"""
|
| 81 |
+
if not self._enabled or self._estimator is None:
|
| 82 |
+
return _NOT_USED
|
| 83 |
+
|
| 84 |
+
if not frames:
|
| 85 |
+
return Body3DResult(
|
| 86 |
+
used=False, joints_3d=[], confidence=0.0,
|
| 87 |
+
notes="3D enabled but no frames provided",
|
| 88 |
+
)
|
| 89 |
+
|
| 90 |
+
try:
|
| 91 |
+
import cv2
|
| 92 |
+
|
| 93 |
+
# Subsample frames evenly for 3D (it's expensive per-image)
|
| 94 |
+
n_frames = len(frames)
|
| 95 |
+
step = max(1, n_frames // _MAX_3D_FRAMES)
|
| 96 |
+
selected_indices = list(range(0, n_frames, step))[:_MAX_3D_FRAMES]
|
| 97 |
+
|
| 98 |
+
joints_3d_per_frame: list[dict] = []
|
| 99 |
+
confidences: list[float] = []
|
| 100 |
+
|
| 101 |
+
for idx in selected_indices:
|
| 102 |
+
frame_bgr = frames[idx]
|
| 103 |
+
# SAM 3D Body expects RGB
|
| 104 |
+
frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB)
|
| 105 |
+
|
| 106 |
+
outputs = self._estimator.process_one_image(frame_rgb)
|
| 107 |
+
|
| 108 |
+
# Extract MHR joint positions from outputs
|
| 109 |
+
# The model returns joints in the MHR (Momentum Human Rig) format
|
| 110 |
+
frame_joints = self._extract_joints(outputs, idx)
|
| 111 |
+
joints_3d_per_frame.append(frame_joints)
|
| 112 |
+
|
| 113 |
+
# Confidence from detection quality
|
| 114 |
+
conf = self._estimate_confidence(outputs)
|
| 115 |
+
confidences.append(conf)
|
| 116 |
+
|
| 117 |
+
# Apply light temporal smoothing to reduce jitter
|
| 118 |
+
joints_3d_smoothed = self._temporal_smooth(joints_3d_per_frame)
|
| 119 |
+
|
| 120 |
+
overall_conf = float(np.mean(confidences)) if confidences else 0.0
|
| 121 |
+
|
| 122 |
+
return Body3DResult(
|
| 123 |
+
used=True,
|
| 124 |
+
joints_3d=joints_3d_smoothed,
|
| 125 |
+
confidence=overall_conf,
|
| 126 |
+
notes=f"3D mesh recovery on {len(selected_indices)}/{n_frames} frames",
|
| 127 |
+
)
|
| 128 |
+
|
| 129 |
+
except Exception as e:
|
| 130 |
+
return Body3DResult(
|
| 131 |
+
used=False, joints_3d=[], confidence=0.0,
|
| 132 |
+
notes=f"3D inference failed: {e}",
|
| 133 |
+
)
|
| 134 |
+
|
| 135 |
+
def _extract_joints(self, outputs: dict, frame_idx: int) -> dict:
|
| 136 |
+
"""
|
| 137 |
+
Extract 3D joint positions from SAM 3D Body outputs.
|
| 138 |
+
Maps MHR rig joints to a standardized dict format.
|
| 139 |
+
"""
|
| 140 |
+
joints: dict = {"frame_index": frame_idx}
|
| 141 |
+
|
| 142 |
+
# SAM 3D Body outputs MHR model params including joint positions
|
| 143 |
+
# The exact key depends on the model output format
|
| 144 |
+
if hasattr(outputs, "joints_3d"):
|
| 145 |
+
joint_data = outputs.joints_3d
|
| 146 |
+
elif isinstance(outputs, dict) and "joints_3d" in outputs:
|
| 147 |
+
joint_data = outputs["joints_3d"]
|
| 148 |
+
elif isinstance(outputs, dict) and "pred_joints" in outputs:
|
| 149 |
+
joint_data = outputs["pred_joints"]
|
| 150 |
+
else:
|
| 151 |
+
# Fallback: extract from vertices/body model params
|
| 152 |
+
joint_data = None
|
| 153 |
+
|
| 154 |
+
if joint_data is not None:
|
| 155 |
+
if hasattr(joint_data, "cpu"):
|
| 156 |
+
joint_data = joint_data.cpu().numpy()
|
| 157 |
+
if isinstance(joint_data, np.ndarray):
|
| 158 |
+
# Map to named joints (MHR has standard SMPL-like ordering)
|
| 159 |
+
joint_names = [
|
| 160 |
+
"pelvis", "left_hip", "right_hip", "spine1",
|
| 161 |
+
"left_knee", "right_knee", "spine2",
|
| 162 |
+
"left_ankle", "right_ankle", "spine3",
|
| 163 |
+
"left_foot", "right_foot", "neck",
|
| 164 |
+
"left_collar", "right_collar", "head",
|
| 165 |
+
"left_shoulder", "right_shoulder",
|
| 166 |
+
"left_elbow", "right_elbow",
|
| 167 |
+
"left_wrist", "right_wrist",
|
| 168 |
+
]
|
| 169 |
+
for i, name in enumerate(joint_names):
|
| 170 |
+
if i < len(joint_data):
|
| 171 |
+
pos = joint_data[i]
|
| 172 |
+
joints[name] = {
|
| 173 |
+
"x": float(pos[0]),
|
| 174 |
+
"y": float(pos[1]),
|
| 175 |
+
"z": float(pos[2]),
|
| 176 |
+
}
|
| 177 |
+
|
| 178 |
+
return joints
|
| 179 |
+
|
| 180 |
+
def _estimate_confidence(self, outputs) -> float:
|
| 181 |
+
"""Estimate confidence from the SAM 3D Body output quality."""
|
| 182 |
+
# If outputs have a confidence/score field, use it
|
| 183 |
+
if isinstance(outputs, dict):
|
| 184 |
+
if "confidence" in outputs:
|
| 185 |
+
return float(outputs["confidence"])
|
| 186 |
+
if "score" in outputs:
|
| 187 |
+
return float(outputs["score"])
|
| 188 |
+
# Default: assume reasonable confidence if we got outputs at all
|
| 189 |
+
return 0.75
|
| 190 |
+
|
| 191 |
+
def _temporal_smooth(
|
| 192 |
+
self, joints_3d: list[dict], alpha: float = 0.3
|
| 193 |
+
) -> list[dict]:
|
| 194 |
+
"""
|
| 195 |
+
Apply exponential moving average smoothing to 3D joint positions
|
| 196 |
+
to reduce per-frame jitter from single-image prediction.
|
| 197 |
+
"""
|
| 198 |
+
if len(joints_3d) <= 1:
|
| 199 |
+
return joints_3d
|
| 200 |
+
|
| 201 |
+
smoothed = [joints_3d[0]]
|
| 202 |
+
for i in range(1, len(joints_3d)):
|
| 203 |
+
prev = smoothed[-1]
|
| 204 |
+
curr = joints_3d[i]
|
| 205 |
+
smooth_frame = {"frame_index": curr.get("frame_index", i)}
|
| 206 |
+
|
| 207 |
+
for key in curr:
|
| 208 |
+
if key == "frame_index":
|
| 209 |
+
continue
|
| 210 |
+
if key in prev and isinstance(curr[key], dict) and isinstance(prev[key], dict):
|
| 211 |
+
smooth_frame[key] = {
|
| 212 |
+
"x": alpha * curr[key]["x"] + (1 - alpha) * prev[key]["x"],
|
| 213 |
+
"y": alpha * curr[key]["y"] + (1 - alpha) * prev[key]["y"],
|
| 214 |
+
"z": alpha * curr[key]["z"] + (1 - alpha) * prev[key]["z"],
|
| 215 |
+
}
|
| 216 |
+
else:
|
| 217 |
+
smooth_frame[key] = curr[key]
|
| 218 |
+
|
| 219 |
+
smoothed.append(smooth_frame)
|
| 220 |
+
|
| 221 |
+
return smoothed
|
formscout/agents/classifier.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
MovementClassifierAgent β identifies which FMS test is in the clip.
|
| 3 |
+
|
| 4 |
+
Input: IngestResult (keyframes), Pose2DResult (skeleton context)
|
| 5 |
+
Output: MovementResult(test_name, side, confidence)
|
| 6 |
+
Failure: returns MovementResult(test_name="unknown") β pipeline stops and asks for manual override.
|
| 7 |
+
Model: Qwen3-VL-8B-Instruct via llama.cpp (8B params, Apache-2.0).
|
| 8 |
+
Gated: No.
|
| 9 |
+
"""
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
import logging
|
| 13 |
+
from pathlib import Path
|
| 14 |
+
|
| 15 |
+
from formscout import config
|
| 16 |
+
from formscout.types import IngestResult, Pose2DResult, MovementResult
|
| 17 |
+
from formscout.serving.llama_cpp import LlamaCppClient
|
| 18 |
+
|
| 19 |
+
logger = logging.getLogger(__name__)
|
| 20 |
+
|
| 21 |
+
_PROMPT_PATH = Path(__file__).parent / "prompts" / "c1_classifier.md"
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
class MovementClassifierAgent:
|
| 25 |
+
"""Classifies which FMS test is being performed via VLM or manual override."""
|
| 26 |
+
|
| 27 |
+
def __init__(self):
|
| 28 |
+
self._client = LlamaCppClient(port=config.LLAMA_CPP_PORT_VLM)
|
| 29 |
+
self._system_prompt = _PROMPT_PATH.read_text(encoding="utf-8")
|
| 30 |
+
|
| 31 |
+
def run(
|
| 32 |
+
self,
|
| 33 |
+
ingest: IngestResult,
|
| 34 |
+
pose2d: Pose2DResult | None = None,
|
| 35 |
+
manual_override: str | None = None,
|
| 36 |
+
) -> MovementResult:
|
| 37 |
+
"""
|
| 38 |
+
Classify the movement. If manual_override is provided, use it directly.
|
| 39 |
+
Otherwise, use VLM inference on keyframes.
|
| 40 |
+
"""
|
| 41 |
+
if manual_override and manual_override != "unknown":
|
| 42 |
+
return MovementResult(
|
| 43 |
+
test_name=manual_override, side="na",
|
| 44 |
+
confidence=1.0, notes="manual override",
|
| 45 |
+
)
|
| 46 |
+
|
| 47 |
+
if not self._client.available:
|
| 48 |
+
return MovementResult(
|
| 49 |
+
test_name="unknown", side="na", confidence=0.0,
|
| 50 |
+
notes="VLM server unavailable β use manual override",
|
| 51 |
+
)
|
| 52 |
+
|
| 53 |
+
# Select keyframes for classification (3 evenly spaced)
|
| 54 |
+
n = len(ingest.frames)
|
| 55 |
+
indices = [0, n // 2, n - 1] if n >= 3 else list(range(n))
|
| 56 |
+
images = self._encode_frames(ingest.frames, indices)
|
| 57 |
+
|
| 58 |
+
prompt = f"{self._system_prompt}\n\nClassify this movement from the keyframes shown."
|
| 59 |
+
result = self._client.complete(prompt, images=images, max_tokens=256, temperature=0.1)
|
| 60 |
+
|
| 61 |
+
return self._parse_response(result)
|
| 62 |
+
|
| 63 |
+
def _encode_frames(self, frames: list, indices: list[int]) -> list[str]:
|
| 64 |
+
"""Encode selected frames as base64 JPEG for the VLM."""
|
| 65 |
+
import cv2
|
| 66 |
+
import base64
|
| 67 |
+
|
| 68 |
+
encoded = []
|
| 69 |
+
for idx in indices:
|
| 70 |
+
if idx < len(frames):
|
| 71 |
+
_, buf = cv2.imencode(".jpg", frames[idx], [cv2.IMWRITE_JPEG_QUALITY, 80])
|
| 72 |
+
encoded.append(base64.b64encode(buf.tobytes()).decode())
|
| 73 |
+
return encoded
|
| 74 |
+
|
| 75 |
+
def _parse_response(self, result: dict) -> MovementResult:
|
| 76 |
+
"""Parse VLM JSON response into MovementResult."""
|
| 77 |
+
if "error" in result:
|
| 78 |
+
return MovementResult(
|
| 79 |
+
test_name="unknown", side="na", confidence=0.0,
|
| 80 |
+
notes=f"VLM error: {result['error']}",
|
| 81 |
+
)
|
| 82 |
+
|
| 83 |
+
test = result.get("test", "unknown")
|
| 84 |
+
side = result.get("side", "na")
|
| 85 |
+
confidence = float(result.get("confidence", 0.0))
|
| 86 |
+
reason = result.get("reason", "")
|
| 87 |
+
|
| 88 |
+
valid_tests = {
|
| 89 |
+
"deep_squat", "hurdle_step", "inline_lunge",
|
| 90 |
+
"shoulder_mobility", "active_slr",
|
| 91 |
+
"trunk_stability_pushup", "rotary_stability", "unknown",
|
| 92 |
+
}
|
| 93 |
+
if test not in valid_tests:
|
| 94 |
+
test = "unknown"
|
| 95 |
+
|
| 96 |
+
if side not in ("left", "right", "na"):
|
| 97 |
+
side = "na"
|
| 98 |
+
|
| 99 |
+
return MovementResult(
|
| 100 |
+
test_name=test, side=side,
|
| 101 |
+
confidence=confidence, notes=reason,
|
| 102 |
+
)
|
formscout/agents/ingest.py
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
IngestAgent β decodes video, normalizes FPS, samples frames.
|
| 3 |
+
|
| 4 |
+
Input: video file path (str)
|
| 5 |
+
Output: IngestResult(frames, fps, duration, n_people, width, height)
|
| 6 |
+
Failure: returns IngestResult with confidence=0.0 and notes explaining the error.
|
| 7 |
+
Params: 0 (no model β pure OpenCV).
|
| 8 |
+
License: n/a.
|
| 9 |
+
Gated: no.
|
| 10 |
+
"""
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
import cv2
|
| 14 |
+
from pathlib import Path
|
| 15 |
+
|
| 16 |
+
from formscout.types import IngestResult
|
| 17 |
+
from formscout import config
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
class IngestAgent:
|
| 21 |
+
"""Deterministic video ingestion β no model, just OpenCV decode + frame sampling."""
|
| 22 |
+
|
| 23 |
+
def run(self, video_path: str) -> IngestResult:
|
| 24 |
+
p = Path(video_path)
|
| 25 |
+
if not p.exists():
|
| 26 |
+
return IngestResult(
|
| 27 |
+
frames=[], fps=0.0, duration=0.0, n_people=0,
|
| 28 |
+
width=0, height=0, confidence=0.0,
|
| 29 |
+
notes=f"video not found: {video_path}",
|
| 30 |
+
)
|
| 31 |
+
|
| 32 |
+
try:
|
| 33 |
+
cap = cv2.VideoCapture(str(p))
|
| 34 |
+
except Exception as e:
|
| 35 |
+
return IngestResult(
|
| 36 |
+
frames=[], fps=0.0, duration=0.0, n_people=0,
|
| 37 |
+
width=0, height=0, confidence=0.0,
|
| 38 |
+
notes=f"failed to open video: {e}",
|
| 39 |
+
)
|
| 40 |
+
|
| 41 |
+
if not cap.isOpened():
|
| 42 |
+
return IngestResult(
|
| 43 |
+
frames=[], fps=0.0, duration=0.0, n_people=0,
|
| 44 |
+
width=0, height=0, confidence=0.0,
|
| 45 |
+
notes=f"could not open video: {video_path}",
|
| 46 |
+
)
|
| 47 |
+
|
| 48 |
+
fps = cap.get(cv2.CAP_PROP_FPS) or config.TARGET_FPS
|
| 49 |
+
total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
|
| 50 |
+
w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
|
| 51 |
+
h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
|
| 52 |
+
duration = total / fps if fps > 0 else 0.0
|
| 53 |
+
|
| 54 |
+
notes_parts: list[str] = []
|
| 55 |
+
if duration > config.MAX_DURATION_SEC:
|
| 56 |
+
notes_parts.append(
|
| 57 |
+
f"video is {duration:.1f}s (>{config.MAX_DURATION_SEC}s) β capping frames"
|
| 58 |
+
)
|
| 59 |
+
|
| 60 |
+
# Sample frames evenly, capped at MAX_FRAMES
|
| 61 |
+
step = max(1, total // config.MAX_FRAMES)
|
| 62 |
+
frames: list = []
|
| 63 |
+
idx = 0
|
| 64 |
+
while True:
|
| 65 |
+
ret, frame = cap.read()
|
| 66 |
+
if not ret:
|
| 67 |
+
break
|
| 68 |
+
if idx % step == 0:
|
| 69 |
+
frames.append(frame)
|
| 70 |
+
idx += 1
|
| 71 |
+
if len(frames) >= config.MAX_FRAMES:
|
| 72 |
+
break
|
| 73 |
+
cap.release()
|
| 74 |
+
|
| 75 |
+
if not frames:
|
| 76 |
+
return IngestResult(
|
| 77 |
+
frames=[], fps=fps, duration=duration, n_people=0,
|
| 78 |
+
width=w, height=h, confidence=0.0,
|
| 79 |
+
notes="no frames decoded",
|
| 80 |
+
)
|
| 81 |
+
|
| 82 |
+
return IngestResult(
|
| 83 |
+
frames=frames,
|
| 84 |
+
fps=fps,
|
| 85 |
+
duration=duration,
|
| 86 |
+
n_people=-1, # unknown until segmentation/pose
|
| 87 |
+
width=w,
|
| 88 |
+
height=h,
|
| 89 |
+
confidence=1.0,
|
| 90 |
+
notes="; ".join(notes_parts) if notes_parts else "",
|
| 91 |
+
)
|
formscout/agents/judge.py
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
JudgeAgent β VLM-based final scorer with rationale, compensation tags, pain detection.
|
| 3 |
+
|
| 4 |
+
Input: BiomechFeatures, ScoreResult (rubric candidate), MovementResult, keyframes
|
| 5 |
+
Output: JudgeResult(score, rationale, compensation_tags, corrective_hint, needs_human)
|
| 6 |
+
Failure: returns JudgeResult(needs_human=True, score=None) when uncertain.
|
| 7 |
+
Model: Qwen3-VL-8B-Instruct via llama.cpp (8B params, Apache-2.0).
|
| 8 |
+
Gated: No.
|
| 9 |
+
|
| 10 |
+
Safety: NEVER auto-scores pain. If any indication of pain/clearing test,
|
| 11 |
+
sets needs_human=True and score=None.
|
| 12 |
+
"""
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
import json
|
| 16 |
+
import logging
|
| 17 |
+
from pathlib import Path
|
| 18 |
+
|
| 19 |
+
from formscout import config
|
| 20 |
+
from formscout.types import (
|
| 21 |
+
BiomechFeatures, ScoreResult, MovementResult,
|
| 22 |
+
IngestResult, JudgeResult,
|
| 23 |
+
)
|
| 24 |
+
from formscout.serving import get_vlm_client
|
| 25 |
+
|
| 26 |
+
logger = logging.getLogger(__name__)
|
| 27 |
+
|
| 28 |
+
_PROMPT_PATH = Path(__file__).parent / "prompts" / "c2_judge.md"
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
class JudgeAgent:
|
| 32 |
+
"""VLM judge that produces the final FMS score with rationale."""
|
| 33 |
+
|
| 34 |
+
def __init__(self):
|
| 35 |
+
self._client = get_vlm_client()
|
| 36 |
+
self._system_prompt = _PROMPT_PATH.read_text(encoding="utf-8")
|
| 37 |
+
|
| 38 |
+
def run(
|
| 39 |
+
self,
|
| 40 |
+
features: BiomechFeatures,
|
| 41 |
+
rubric_score: ScoreResult,
|
| 42 |
+
movement: MovementResult,
|
| 43 |
+
ingest: IngestResult | None = None,
|
| 44 |
+
) -> JudgeResult:
|
| 45 |
+
"""
|
| 46 |
+
Produce final score. Falls back to rubric score if VLM unavailable.
|
| 47 |
+
"""
|
| 48 |
+
if not config.ENABLE_JUDGE:
|
| 49 |
+
return self._fallback_from_rubric(rubric_score, features)
|
| 50 |
+
|
| 51 |
+
if not self._client.available:
|
| 52 |
+
logger.warning("JudgeAgent: VLM unavailable, using rubric score as final")
|
| 53 |
+
return self._fallback_from_rubric(rubric_score, features)
|
| 54 |
+
|
| 55 |
+
# Build context for the judge
|
| 56 |
+
context = {
|
| 57 |
+
"test": features.test_name,
|
| 58 |
+
"side": features.side,
|
| 59 |
+
"view": features.view,
|
| 60 |
+
"features": {"angles": features.angles, "alignments": features.alignments},
|
| 61 |
+
"candidate_score": rubric_score.score,
|
| 62 |
+
"candidate_confidence": rubric_score.confidence,
|
| 63 |
+
"exemplars": [], # Phase 3: populated by RetrievalAgent
|
| 64 |
+
}
|
| 65 |
+
|
| 66 |
+
prompt = f"{self._system_prompt}\n\n{json.dumps(context, indent=2)}"
|
| 67 |
+
|
| 68 |
+
# Optionally include keyframes
|
| 69 |
+
images = None
|
| 70 |
+
if ingest and ingest.frames:
|
| 71 |
+
images = self._encode_keyframes(ingest.frames)
|
| 72 |
+
|
| 73 |
+
result = self._client.complete(prompt, images=images, max_tokens=512, temperature=0.1)
|
| 74 |
+
if result.get("fallback"):
|
| 75 |
+
# transformers backend couldn't load/run β use the deterministic rubric
|
| 76 |
+
return self._fallback_from_rubric(rubric_score, features)
|
| 77 |
+
return self._parse_response(result)
|
| 78 |
+
|
| 79 |
+
def _encode_keyframes(self, frames: list) -> list[str]:
|
| 80 |
+
"""Encode 3 keyframes for VLM context."""
|
| 81 |
+
import cv2
|
| 82 |
+
import base64
|
| 83 |
+
|
| 84 |
+
n = len(frames)
|
| 85 |
+
indices = [0, n // 2, n - 1] if n >= 3 else list(range(n))
|
| 86 |
+
encoded = []
|
| 87 |
+
for idx in indices:
|
| 88 |
+
_, buf = cv2.imencode(".jpg", frames[idx], [cv2.IMWRITE_JPEG_QUALITY, 70])
|
| 89 |
+
encoded.append(base64.b64encode(buf.tobytes()).decode())
|
| 90 |
+
return encoded
|
| 91 |
+
|
| 92 |
+
def _parse_response(self, result: dict) -> JudgeResult:
|
| 93 |
+
"""Parse VLM JSON response into JudgeResult."""
|
| 94 |
+
if "error" in result:
|
| 95 |
+
return JudgeResult(
|
| 96 |
+
score=None, rationale=f"VLM error: {result['error']}",
|
| 97 |
+
compensation_tags=[], corrective_hint="",
|
| 98 |
+
confidence=0.0, needs_human=True,
|
| 99 |
+
)
|
| 100 |
+
|
| 101 |
+
needs_human = result.get("needs_human", False)
|
| 102 |
+
score = result.get("score") if not needs_human else None
|
| 103 |
+
if score is not None:
|
| 104 |
+
score = max(0, min(3, int(score)))
|
| 105 |
+
|
| 106 |
+
return JudgeResult(
|
| 107 |
+
score=score,
|
| 108 |
+
rationale=result.get("rationale", ""),
|
| 109 |
+
compensation_tags=result.get("compensation_tags", []),
|
| 110 |
+
corrective_hint=result.get("corrective_hint", ""),
|
| 111 |
+
confidence=float(result.get("confidence", 0.5)),
|
| 112 |
+
needs_human=needs_human,
|
| 113 |
+
)
|
| 114 |
+
|
| 115 |
+
def _fallback_from_rubric(self, rubric: ScoreResult, features: BiomechFeatures) -> JudgeResult:
|
| 116 |
+
"""When VLM is unavailable, promote the rubric score as the final score."""
|
| 117 |
+
return JudgeResult(
|
| 118 |
+
score=rubric.score,
|
| 119 |
+
rationale=f"[rubric-only] {rubric.rationale}",
|
| 120 |
+
compensation_tags=[],
|
| 121 |
+
corrective_hint="",
|
| 122 |
+
confidence=rubric.confidence * 0.8,
|
| 123 |
+
needs_human=rubric.needs_human,
|
| 124 |
+
notes="VLM unavailable β rubric score used as final",
|
| 125 |
+
)
|
formscout/agents/pdf_report.py
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
PdfReportAgent β renders a ReportResult + session entries to a branded PDF.
|
| 3 |
+
|
| 4 |
+
Input: ReportResult, list[SessionEntry], session_dir (str)
|
| 5 |
+
Output: path to the written PDF (str), or None on failure.
|
| 6 |
+
Failure: returns None, never raises.
|
| 7 |
+
Params: 0 (pure rendering β no model).
|
| 8 |
+
License: n/a.
|
| 9 |
+
Gated: no.
|
| 10 |
+
"""
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
import logging
|
| 14 |
+
import os
|
| 15 |
+
|
| 16 |
+
from formscout.types import ReportResult
|
| 17 |
+
|
| 18 |
+
logger = logging.getLogger(__name__)
|
| 19 |
+
|
| 20 |
+
DISCLAIMER = "Screening aid β not a diagnosis. Pain or clearing tests require a clinician."
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
class PdfReportAgent:
|
| 24 |
+
"""Assembles the screening-session PDF via ReportLab."""
|
| 25 |
+
|
| 26 |
+
def run(self, report: ReportResult, entries: list, session_dir: str) -> str | None:
|
| 27 |
+
try:
|
| 28 |
+
from reportlab.lib import colors
|
| 29 |
+
from reportlab.lib.pagesizes import LETTER
|
| 30 |
+
from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
|
| 31 |
+
from reportlab.lib.units import inch
|
| 32 |
+
from reportlab.platypus import (
|
| 33 |
+
Image, PageBreak, Paragraph, SimpleDocTemplate, Spacer, Table, TableStyle,
|
| 34 |
+
)
|
| 35 |
+
except Exception as e:
|
| 36 |
+
logger.warning("reportlab unavailable: %s", e)
|
| 37 |
+
return None
|
| 38 |
+
|
| 39 |
+
out_path = os.path.join(session_dir, "formscout_report.pdf")
|
| 40 |
+
try:
|
| 41 |
+
styles = getSampleStyleSheet()
|
| 42 |
+
banner = ParagraphStyle(
|
| 43 |
+
"banner", parent=styles["Normal"], fontSize=9, textColor=colors.white,
|
| 44 |
+
backColor=colors.HexColor("#cf922a"), alignment=1, borderPadding=6, spaceAfter=12,
|
| 45 |
+
)
|
| 46 |
+
ink = colors.HexColor("#243a34")
|
| 47 |
+
|
| 48 |
+
def _meas_table(pairs, col0=3.0, col1=1.6):
|
| 49 |
+
rows = [[str(k).replace("_", " "),
|
| 50 |
+
(f"{v:.2f}" if isinstance(v, float) else str(v))] for k, v in pairs]
|
| 51 |
+
tbl = Table(rows, colWidths=[col0 * inch, col1 * inch])
|
| 52 |
+
tbl.setStyle(TableStyle([
|
| 53 |
+
("FONTSIZE", (0, 0), (-1, -1), 8),
|
| 54 |
+
("TEXTCOLOR", (0, 0), (-1, -1), ink),
|
| 55 |
+
("ROWBACKGROUNDS", (0, 0), (-1, -1),
|
| 56 |
+
[colors.HexColor("#f7eedd"), colors.white]),
|
| 57 |
+
]))
|
| 58 |
+
return tbl
|
| 59 |
+
|
| 60 |
+
def _img(path, w=3.0, h=2.25):
|
| 61 |
+
if path and os.path.exists(path):
|
| 62 |
+
try:
|
| 63 |
+
return Image(path, width=w * inch, height=h * inch)
|
| 64 |
+
except Exception:
|
| 65 |
+
return None
|
| 66 |
+
return None
|
| 67 |
+
story = []
|
| 68 |
+
story.append(Paragraph(f"<b>⚠ {DISCLAIMER}</b>", banner))
|
| 69 |
+
story.append(Paragraph("FormScout β FMS Screening Report", styles["Title"]))
|
| 70 |
+
|
| 71 |
+
if report.composite is not None:
|
| 72 |
+
comp = f"Composite: <b>{report.composite} / 21</b>"
|
| 73 |
+
else:
|
| 74 |
+
comp = f"Composite: <b>Incomplete</b> β {len(entries)}/7 tests scored"
|
| 75 |
+
story.append(Paragraph(comp, styles["Heading2"]))
|
| 76 |
+
story.append(Spacer(1, 0.2 * inch))
|
| 77 |
+
|
| 78 |
+
for ei, e in enumerate(entries):
|
| 79 |
+
if ei > 0:
|
| 80 |
+
story.append(PageBreak())
|
| 81 |
+
title = e.test_name.replace("_", " ").title()
|
| 82 |
+
if e.side in ("left", "right"):
|
| 83 |
+
title += f" ({e.side})"
|
| 84 |
+
score_txt = "Clinician review required" if e.needs_human else f"Score: {e.score}/3"
|
| 85 |
+
story.append(Paragraph(f"<b>{title}</b> β {score_txt}", styles["Heading3"]))
|
| 86 |
+
story.append(Paragraph(f"<font size=8>view: {e.view} Β· confidence: "
|
| 87 |
+
f"{e.confidence:.0%}</font>", styles["Normal"]))
|
| 88 |
+
if e.rationale:
|
| 89 |
+
story.append(Paragraph(e.rationale, styles["Normal"]))
|
| 90 |
+
if e.compensation_tags:
|
| 91 |
+
story.append(Paragraph("<b>Compensations:</b> " + ", ".join(e.compensation_tags),
|
| 92 |
+
styles["Normal"]))
|
| 93 |
+
if e.corrective_hint:
|
| 94 |
+
story.append(Paragraph("<b>Corrective:</b> " + e.corrective_hint, styles["Normal"]))
|
| 95 |
+
|
| 96 |
+
# Key frame + flexion chart side by side
|
| 97 |
+
kf, fb = _img(e.keyframe_path), _img((e.chart_paths or {}).get("flexion"), w=3.2, h=2.0)
|
| 98 |
+
if kf or fb:
|
| 99 |
+
cells = [c for c in (kf, fb) if c] or [Paragraph("<i>(images unavailable)</i>",
|
| 100 |
+
styles["Normal"])]
|
| 101 |
+
story.append(Table([cells], hAlign="LEFT"))
|
| 102 |
+
|
| 103 |
+
# Relevant-joint flexion table
|
| 104 |
+
if e.flexion:
|
| 105 |
+
story.append(Paragraph("<b>Relevant joint flexion (key frame)</b>", styles["Normal"]))
|
| 106 |
+
story.append(_meas_table(
|
| 107 |
+
[(n, f"{v['deg']:.1f}Β° β {v['openness']}") for n, v in e.flexion.items()],
|
| 108 |
+
col0=2.6, col1=2.6))
|
| 109 |
+
|
| 110 |
+
# Laban Effort + radar
|
| 111 |
+
if e.laban:
|
| 112 |
+
eff, lab = e.laban.get("effort", {}), e.laban.get("labels", {})
|
| 113 |
+
story.append(Spacer(1, 0.08 * inch))
|
| 114 |
+
story.append(Paragraph("<b>Laban Effort (kinematic estimate)</b>", styles["Normal"]))
|
| 115 |
+
laban_tbl = _meas_table(
|
| 116 |
+
[(k.title(), f"{eff.get(k, 0):.2f} β {lab.get(k, '')}")
|
| 117 |
+
for k in ("space", "weight", "time", "flow")], col0=2.6, col1=2.6)
|
| 118 |
+
radar = _img((e.chart_paths or {}).get("radar"), w=2.6, h=2.6)
|
| 119 |
+
if radar:
|
| 120 |
+
story.append(Table([[laban_tbl, radar]], hAlign="LEFT"))
|
| 121 |
+
else:
|
| 122 |
+
story.append(laban_tbl)
|
| 123 |
+
if e.laban.get("body_emphasis"):
|
| 124 |
+
emph = ", ".join(f"{n}" for n, _ in e.laban["body_emphasis"])
|
| 125 |
+
story.append(Paragraph(f"<font size=8>Body emphasis: {emph} Β· "
|
| 126 |
+
f"{e.laban.get('notes', '')}</font>", styles["Normal"]))
|
| 127 |
+
|
| 128 |
+
# Angle + velocity charts
|
| 129 |
+
for kind in ("angle", "velocity"):
|
| 130 |
+
chart = _img((e.chart_paths or {}).get(kind), w=5.0, h=2.5)
|
| 131 |
+
if chart:
|
| 132 |
+
story.append(chart)
|
| 133 |
+
|
| 134 |
+
# Full measurement dump
|
| 135 |
+
if e.measurements:
|
| 136 |
+
story.append(Paragraph("<b>All measurements</b>", styles["Normal"]))
|
| 137 |
+
story.append(_meas_table(list(e.measurements.items())))
|
| 138 |
+
|
| 139 |
+
story.append(Spacer(1, 0.15 * inch))
|
| 140 |
+
|
| 141 |
+
if report.asymmetries:
|
| 142 |
+
story.append(PageBreak())
|
| 143 |
+
story.append(Paragraph("Asymmetries", styles["Heading2"]))
|
| 144 |
+
for a in report.asymmetries:
|
| 145 |
+
story.append(Paragraph(
|
| 146 |
+
f"{a['test'].replace('_', ' ').title()}: "
|
| 147 |
+
f"L={a['left_score']} R={a['right_score']} (Δ {a['delta']})",
|
| 148 |
+
styles["Normal"]))
|
| 149 |
+
try:
|
| 150 |
+
from formscout.analysis.charts import symmetry_bars
|
| 151 |
+
os.makedirs(os.path.join(session_dir, "charts"), exist_ok=True)
|
| 152 |
+
sym_png = symmetry_bars(report.asymmetries,
|
| 153 |
+
os.path.join(session_dir, "charts", "symmetry.png"))
|
| 154 |
+
sym_img = _img(sym_png, w=5.5, h=2.75)
|
| 155 |
+
if sym_img:
|
| 156 |
+
story.append(sym_img)
|
| 157 |
+
except Exception:
|
| 158 |
+
pass
|
| 159 |
+
|
| 160 |
+
flags = list(report.low_confidence_flags) + list(report.disagreement_flags)
|
| 161 |
+
if flags:
|
| 162 |
+
story.append(Paragraph("Flags", styles["Heading2"]))
|
| 163 |
+
for fl in flags:
|
| 164 |
+
story.append(Paragraph(fl, styles["Normal"]))
|
| 165 |
+
|
| 166 |
+
story.append(Spacer(1, 0.3 * inch))
|
| 167 |
+
story.append(Paragraph(f"<b>⚠ {DISCLAIMER}</b>", banner))
|
| 168 |
+
|
| 169 |
+
doc = SimpleDocTemplate(out_path, pagesize=LETTER,
|
| 170 |
+
topMargin=0.6 * inch, bottomMargin=0.6 * inch)
|
| 171 |
+
doc.build(story)
|
| 172 |
+
return out_path
|
| 173 |
+
except Exception as e:
|
| 174 |
+
logger.warning("pdf build failed: %s", e)
|
| 175 |
+
return None
|
formscout/agents/pose2d.py
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Pose2DAgent β 2D per-frame keypoint extraction.
|
| 3 |
+
|
| 4 |
+
Backends: yolo (local checkpoints, ultralytics), mediapipe (official Tasks API,
|
| 5 |
+
local .task checkpoint), sapiens2 (Meta HF/transformers).
|
| 6 |
+
All backends output COCO-17 keypoints: dict[int, {x, y, conf}] per frame.
|
| 7 |
+
|
| 8 |
+
Input: IngestResult
|
| 9 |
+
Output: Pose2DResult(keypoints per frame, fps, confidence)
|
| 10 |
+
Failure: Pose2DResult(confidence=0.0, notes=<reason>) β never raises.
|
| 11 |
+
Gated: yolo=no; mediapipe=no (local checkpoint); sapiens2=yes (access accepted).
|
| 12 |
+
"""
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
import logging
|
| 16 |
+
import numpy as np
|
| 17 |
+
|
| 18 |
+
from formscout import config
|
| 19 |
+
from formscout.types import IngestResult, Pose2DResult
|
| 20 |
+
|
| 21 |
+
logger = logging.getLogger(__name__)
|
| 22 |
+
|
| 23 |
+
COCO_KEYPOINTS = [
|
| 24 |
+
"nose", "left_eye", "right_eye", "left_ear", "right_ear",
|
| 25 |
+
"left_shoulder", "right_shoulder", "left_elbow", "right_elbow",
|
| 26 |
+
"left_wrist", "right_wrist", "left_hip", "right_hip",
|
| 27 |
+
"left_knee", "right_knee", "left_ankle", "right_ankle",
|
| 28 |
+
]
|
| 29 |
+
|
| 30 |
+
# BlazePose-33 source indices β COCO-17 target indices
|
| 31 |
+
# BlazePose: 0=nose, 2=left_eye, 5=right_eye, 7=left_ear, 8=right_ear,
|
| 32 |
+
# 11=left_shoulder, 12=right_shoulder, 13=left_elbow, 14=right_elbow,
|
| 33 |
+
# 15=left_wrist, 16=right_wrist, 23=left_hip, 24=right_hip,
|
| 34 |
+
# 25=left_knee, 26=right_knee, 27=left_ankle, 28=right_ankle
|
| 35 |
+
_BP_SRC = [0, 2, 5, 7, 8, 11, 12, 13, 14, 15, 16, 23, 24, 25, 26, 27, 28]
|
| 36 |
+
_BP_DST = list(range(17)) # COCO indices 0..16
|
| 37 |
+
|
| 38 |
+
_model_cache: dict[str, object] = {}
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
# ββ YOLO backend ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 42 |
+
|
| 43 |
+
def _get_yolo(path: str) -> object:
|
| 44 |
+
if path not in _model_cache:
|
| 45 |
+
from ultralytics import YOLO
|
| 46 |
+
_model_cache[path] = YOLO(path)
|
| 47 |
+
return _model_cache[path]
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _run_yolo(frames: list, path: str) -> list[dict]:
|
| 51 |
+
model = _get_yolo(path)
|
| 52 |
+
out = []
|
| 53 |
+
for frame in frames:
|
| 54 |
+
try:
|
| 55 |
+
results = model(frame, verbose=False)
|
| 56 |
+
kps: dict[int, dict] = {}
|
| 57 |
+
if results and results[0].keypoints is not None:
|
| 58 |
+
kp = results[0].keypoints
|
| 59 |
+
if kp.xy is not None and len(kp.xy) > 0:
|
| 60 |
+
xy = kp.xy[0].cpu().numpy()
|
| 61 |
+
conf = kp.conf[0].cpu().numpy()
|
| 62 |
+
for j in range(min(len(xy), 17)):
|
| 63 |
+
kps[j] = {"x": float(xy[j, 0]), "y": float(xy[j, 1]), "conf": float(conf[j])}
|
| 64 |
+
out.append(kps)
|
| 65 |
+
except Exception:
|
| 66 |
+
out.append({})
|
| 67 |
+
return out
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
# ββ MediaPipe backend (official Tasks API, local .task checkpoint) ββββββββββββ
|
| 71 |
+
|
| 72 |
+
def _get_mediapipe_landmarker(path: str) -> object:
|
| 73 |
+
"""Return PoseLandmarker cached by model path."""
|
| 74 |
+
cache_key = f"mp:{path}"
|
| 75 |
+
if cache_key not in _model_cache:
|
| 76 |
+
from mediapipe.tasks import python as mp_tasks
|
| 77 |
+
from mediapipe.tasks.python import vision
|
| 78 |
+
|
| 79 |
+
options = vision.PoseLandmarkerOptions(
|
| 80 |
+
base_options=mp_tasks.BaseOptions(model_asset_path=path),
|
| 81 |
+
running_mode=vision.RunningMode.IMAGE,
|
| 82 |
+
num_poses=1,
|
| 83 |
+
min_pose_detection_confidence=0.4,
|
| 84 |
+
min_pose_presence_confidence=0.4,
|
| 85 |
+
min_tracking_confidence=0.4,
|
| 86 |
+
)
|
| 87 |
+
_model_cache[cache_key] = vision.PoseLandmarker.create_from_options(options)
|
| 88 |
+
return _model_cache[cache_key]
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def _run_mediapipe(frames: list, path: str) -> list[dict]:
|
| 92 |
+
import cv2
|
| 93 |
+
import mediapipe as mp
|
| 94 |
+
|
| 95 |
+
try:
|
| 96 |
+
landmarker = _get_mediapipe_landmarker(path)
|
| 97 |
+
except Exception as e:
|
| 98 |
+
logger.warning("mediapipe load failed: %s", e)
|
| 99 |
+
return [{} for _ in frames]
|
| 100 |
+
|
| 101 |
+
out = []
|
| 102 |
+
for frame in frames:
|
| 103 |
+
try:
|
| 104 |
+
h, w = frame.shape[:2]
|
| 105 |
+
rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
| 106 |
+
mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=rgb)
|
| 107 |
+
detection = landmarker.detect(mp_image)
|
| 108 |
+
|
| 109 |
+
kps: dict[int, dict] = {}
|
| 110 |
+
if detection.pose_landmarks:
|
| 111 |
+
lms = detection.pose_landmarks[0]
|
| 112 |
+
for coco_idx, bp_idx in zip(_BP_DST, _BP_SRC):
|
| 113 |
+
if bp_idx < len(lms):
|
| 114 |
+
lm = lms[bp_idx]
|
| 115 |
+
kps[coco_idx] = {
|
| 116 |
+
"x": float(lm.x * w),
|
| 117 |
+
"y": float(lm.y * h),
|
| 118 |
+
"conf": float(lm.visibility),
|
| 119 |
+
}
|
| 120 |
+
out.append(kps)
|
| 121 |
+
except Exception:
|
| 122 |
+
out.append({})
|
| 123 |
+
return out
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
# ββ Sapiens2 backend (Meta HF, transformers) ββββββββββββββββββββββββββββββββββ
|
| 127 |
+
|
| 128 |
+
def _get_sapiens2(hf_id: str) -> object:
|
| 129 |
+
if hf_id not in _model_cache:
|
| 130 |
+
from transformers import pipeline as hf_pipeline
|
| 131 |
+
_model_cache[hf_id] = hf_pipeline("pose-estimation", model=hf_id)
|
| 132 |
+
return _model_cache[hf_id]
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def _run_sapiens2(frames: list, hf_id: str) -> list[dict]:
|
| 136 |
+
try:
|
| 137 |
+
pipe = _get_sapiens2(hf_id)
|
| 138 |
+
except Exception as e:
|
| 139 |
+
logger.warning("sapiens2 load failed: %s", e)
|
| 140 |
+
return [{} for _ in frames]
|
| 141 |
+
|
| 142 |
+
from PIL import Image
|
| 143 |
+
|
| 144 |
+
out = []
|
| 145 |
+
for frame in frames:
|
| 146 |
+
try:
|
| 147 |
+
pil_img = Image.fromarray(frame)
|
| 148 |
+
result = pipe(pil_img)
|
| 149 |
+
|
| 150 |
+
if not result:
|
| 151 |
+
out.append({})
|
| 152 |
+
continue
|
| 153 |
+
|
| 154 |
+
# Take highest-confidence person (first result)
|
| 155 |
+
person = result[0]
|
| 156 |
+
keypoints = person.get("keypoints", [])
|
| 157 |
+
scores = person.get("keypoint_scores", [])
|
| 158 |
+
|
| 159 |
+
# Build nameβ(x, y, score) lookup from pipeline output
|
| 160 |
+
kp_lookup: dict[str, tuple] = {}
|
| 161 |
+
for i, kp in enumerate(keypoints):
|
| 162 |
+
if isinstance(kp, dict):
|
| 163 |
+
name = kp.get("label", "")
|
| 164 |
+
x, y = kp.get("x", 0.0), kp.get("y", 0.0)
|
| 165 |
+
else:
|
| 166 |
+
name = ""
|
| 167 |
+
x, y = float(kp[0]), float(kp[1])
|
| 168 |
+
score = float(scores[i]) if i < len(scores) else 0.0
|
| 169 |
+
if name:
|
| 170 |
+
kp_lookup[name] = (x, y, score)
|
| 171 |
+
|
| 172 |
+
kps: dict[int, dict] = {}
|
| 173 |
+
for coco_idx, name in enumerate(COCO_KEYPOINTS):
|
| 174 |
+
if name in kp_lookup:
|
| 175 |
+
x, y, s = kp_lookup[name]
|
| 176 |
+
kps[coco_idx] = {"x": x, "y": y, "conf": s}
|
| 177 |
+
out.append(kps)
|
| 178 |
+
except Exception:
|
| 179 |
+
out.append({})
|
| 180 |
+
return out
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
# ββ Agent βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 184 |
+
|
| 185 |
+
class Pose2DAgent:
|
| 186 |
+
"""Extracts COCO-17 keypoints per frame; dispatches to YOLO, MediaPipe, or Sapiens2."""
|
| 187 |
+
|
| 188 |
+
def run(self, ingest: IngestResult, model_key: str | None = None) -> Pose2DResult:
|
| 189 |
+
if not ingest.frames:
|
| 190 |
+
return Pose2DResult(keypoints=[], fps=ingest.fps, confidence=0.0, notes="no frames in ingest")
|
| 191 |
+
|
| 192 |
+
key = model_key or config.DEFAULT_POSE_MODEL
|
| 193 |
+
spec = config.POSE_MODELS.get(key)
|
| 194 |
+
if spec is None:
|
| 195 |
+
logger.warning("Unknown model_key %r β falling back to %s", key, config.DEFAULT_POSE_MODEL)
|
| 196 |
+
spec = config.POSE_MODELS[config.DEFAULT_POSE_MODEL]
|
| 197 |
+
|
| 198 |
+
backend = spec["backend"]
|
| 199 |
+
try:
|
| 200 |
+
if backend == "yolo":
|
| 201 |
+
kps_per_frame = _run_yolo(ingest.frames, spec["path"])
|
| 202 |
+
elif backend == "mediapipe":
|
| 203 |
+
kps_per_frame = _run_mediapipe(ingest.frames, spec["path"])
|
| 204 |
+
elif backend == "sapiens2":
|
| 205 |
+
kps_per_frame = _run_sapiens2(ingest.frames, spec["hf_id"])
|
| 206 |
+
else:
|
| 207 |
+
return Pose2DResult(
|
| 208 |
+
keypoints=[{} for _ in ingest.frames],
|
| 209 |
+
fps=ingest.fps, confidence=0.0,
|
| 210 |
+
notes=f"unknown backend: {backend}",
|
| 211 |
+
)
|
| 212 |
+
except Exception as e:
|
| 213 |
+
return Pose2DResult(
|
| 214 |
+
keypoints=[{} for _ in ingest.frames],
|
| 215 |
+
fps=ingest.fps, confidence=0.0,
|
| 216 |
+
notes=str(e),
|
| 217 |
+
)
|
| 218 |
+
|
| 219 |
+
n_detected = sum(1 for f in kps_per_frame if f)
|
| 220 |
+
total_conf = sum(
|
| 221 |
+
sum(kp["conf"] for kp in f.values()) / len(f)
|
| 222 |
+
for f in kps_per_frame if f
|
| 223 |
+
)
|
| 224 |
+
overall_conf = (total_conf / n_detected) if n_detected > 0 else 0.0
|
| 225 |
+
notes = "" if n_detected > 0 else "no person detected in any frame"
|
| 226 |
+
|
| 227 |
+
return Pose2DResult(
|
| 228 |
+
keypoints=kps_per_frame,
|
| 229 |
+
fps=ingest.fps,
|
| 230 |
+
confidence=overall_conf,
|
| 231 |
+
notes=notes,
|
| 232 |
+
)
|
formscout/agents/prompts/c1_classifier.md
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
You are an FMS movement classifier. You are shown a few keyframes and a skeleton montage from a single short clip of one person performing ONE Functional Movement Screen test. Identify which test it is and, for one-sided tests, which side is being assessed.
|
| 2 |
+
|
| 3 |
+
The seven tests and their tells:
|
| 4 |
+
- deep_squat: feet shoulder-width, a dowel/bar held overhead with both arms, a deep two-legged squat.
|
| 5 |
+
- hurdle_step: stepping one leg over a low hurdle/cord while balancing on the other, dowel across shoulders.
|
| 6 |
+
- inline_lunge: feet in a narrow heel-to-toe line, a lunge down the line, dowel held vertically behind the back.
|
| 7 |
+
- shoulder_mobility: one hand reaching over the shoulder down the back, the other reaching up from below; fists measured.
|
| 8 |
+
- active_slr: lying supine, one leg raised straight up while the other stays flat on the ground.
|
| 9 |
+
- trunk_stability_pushup: prone push-up with hands high (near the head), body pressed up as one rigid unit.
|
| 10 |
+
- rotary_stability: quadruped (hands+knees), same-side or opposite arm and leg extended then drawn together.
|
| 11 |
+
- unknown: it does not clearly match any of the above, or the view is too poor to tell.
|
| 12 |
+
|
| 13 |
+
Rules:
|
| 14 |
+
- Prefer "unknown" over a low-confidence guess. A wrong test makes the whole score meaningless.
|
| 15 |
+
- "side" is "left" or "right" for one-sided tests (hurdle_step, inline_lunge, shoulder_mobility, active_slr); use "na" for two-sided tests (deep_squat, trunk_stability_pushup, rotary_stability) and unknown.
|
| 16 |
+
- Output ONLY this JSON object, nothing else:
|
| 17 |
+
{"test": "<one of the labels>", "side": "left|right|na", "confidence": <0.0-1.0>, "reason": "<one short sentence>"}
|
formscout/agents/prompts/c2_judge.md
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
You are an assistant scoring ONE Functional Movement Screen test from objective measurements. You are a SCREENING AID, not a clinician. You never diagnose and you never predict injury.
|
| 2 |
+
|
| 3 |
+
You are given, as JSON:
|
| 4 |
+
- test, side
|
| 5 |
+
- view: "3d" (reliable angles) or "2d" (angles are camera-angle dependent β caveat them)
|
| 6 |
+
- features: measured biomechanics for this test (angles in degrees, distances normalized)
|
| 7 |
+
- candidate_score: a model's provisional 0-3 (corroboration, may be absent)
|
| 8 |
+
- exemplars: physio-scored reference clips of the SAME test with their scores (anchors, may be empty)
|
| 9 |
+
- a few keyframes / skeleton overlay for context
|
| 10 |
+
|
| 11 |
+
FMS scoring scale (apply per side; the test score is the LOWER side):
|
| 12 |
+
- 3: the movement is performed to criterion with no compensation.
|
| 13 |
+
- 2: the movement is completed but with compensation / poor mechanics (or only with the allowed regression, e.g. deep_squat heels elevated).
|
| 14 |
+
- 1: the person cannot perform the movement pattern even with the allowed regression.
|
| 15 |
+
- 0: PAIN. You CANNOT see pain. Never assign 0 yourself.
|
| 16 |
+
|
| 17 |
+
Per-test criteria to weigh (use the features as primary evidence):
|
| 18 |
+
- deep_squat (3): femur below horizontal, torso roughly parallel to the tibia, knees tracking over the feet, dowel staying aligned over the feet, heels flat. (2): the same achieved only with heels elevated. (1): criteria unmet even with heels elevated.
|
| 19 |
+
- hurdle_step / inline_lunge: minimal sway/loss of balance, knee/hip/ankle alignment maintained, no contact with the hurdle, dowel/posture stable. Compensation -> 2; failure to complete -> 1. Report L/R asymmetry.
|
| 20 |
+
- shoulder_mobility: judge by the normalized inter-fist distance bands (per side). Report asymmetry.
|
| 21 |
+
- active_slr: judge the raised-leg hip-flexion angle relative to the standard band; the down leg stays flat.
|
| 22 |
+
- trunk_stability_pushup: the body must move as one rigid unit (low segment-angle variance through the press); sag/lag or needing the easier hand position -> 2.
|
| 23 |
+
- rotary_stability: smooth contralateral (or the allowed unilateral) coordination with a stable trunk; loss of coordination/balance -> lower.
|
| 24 |
+
|
| 25 |
+
Hard safety rules:
|
| 26 |
+
- If there is any clearing-test context, visible pain, grimacing, or an aborted rep, set needs_human=true and score=null. Do not score it.
|
| 27 |
+
- If view=="2d" on a depth/angle-critical test (deep_squat, inline_lunge, active_slr), include an explicit one-clause caveat that the angle is a 2D estimate dependent on camera position.
|
| 28 |
+
- If the measurements and the candidate_score disagree by a point or more, lower your confidence and say so.
|
| 29 |
+
- When the features are insufficient to decide, prefer needs_human=true over a confident guess.
|
| 30 |
+
|
| 31 |
+
Reason from the features first; use exemplars to calibrate borderline cases; treat candidate_score as a second opinion, not the answer.
|
| 32 |
+
|
| 33 |
+
Output ONLY this JSON object, nothing else:
|
| 34 |
+
{
|
| 35 |
+
"test": "<label>",
|
| 36 |
+
"side": "left|right|na",
|
| 37 |
+
"score": <0-3 or null>,
|
| 38 |
+
"needs_human": <true|false>,
|
| 39 |
+
"rationale": "<2-4 sentences citing the specific deciding measurement(s)>",
|
| 40 |
+
"compensation_tags": ["<short tag>", "..."],
|
| 41 |
+
"corrective_hint": "<one generic FMS-style suggestion, or '' if needs_human>",
|
| 42 |
+
"confidence": <0.0-1.0>
|
| 43 |
+
}
|
formscout/agents/report.py
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
ReportAgent β assembles per-test scorecard, composite, asymmetries.
|
| 3 |
+
|
| 4 |
+
Input: List of (MovementResult, BiomechFeatures, ScoreResult, JudgeResult) per test
|
| 5 |
+
Output: ReportResult(per_test, composite, asymmetries, overlay_video_path, pdf_path)
|
| 6 |
+
Failure: returns ReportResult with composite=None if any test unscored.
|
| 7 |
+
Params: 0 (pure assembly β no model).
|
| 8 |
+
License: n/a.
|
| 9 |
+
Gated: no.
|
| 10 |
+
"""
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
from formscout.types import (
|
| 14 |
+
MovementResult, BiomechFeatures, ScoreResult, JudgeResult, ReportResult,
|
| 15 |
+
)
|
| 16 |
+
from formscout import config
|
| 17 |
+
|
| 18 |
+
# Bilateral tests that need L/R scoring
|
| 19 |
+
BILATERAL_TESTS = {"hurdle_step", "inline_lunge", "shoulder_mobility", "active_slr"}
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class ReportAgent:
|
| 23 |
+
"""Assembles the final screening report from all test results."""
|
| 24 |
+
|
| 25 |
+
def run(self, test_results: list[dict]) -> ReportResult:
|
| 26 |
+
"""
|
| 27 |
+
Assemble the report.
|
| 28 |
+
|
| 29 |
+
Args:
|
| 30 |
+
test_results: list of dicts with keys:
|
| 31 |
+
- movement: MovementResult
|
| 32 |
+
- features: BiomechFeatures
|
| 33 |
+
- rubric_score: ScoreResult
|
| 34 |
+
- judge: JudgeResult
|
| 35 |
+
- side: str (for bilateral: "left" or "right")
|
| 36 |
+
"""
|
| 37 |
+
per_test = []
|
| 38 |
+
asymmetries = []
|
| 39 |
+
low_confidence_flags = []
|
| 40 |
+
disagreement_flags = []
|
| 41 |
+
|
| 42 |
+
# Group bilateral tests by test_name
|
| 43 |
+
bilateral_groups: dict[str, list[dict]] = {}
|
| 44 |
+
unilateral: list[dict] = []
|
| 45 |
+
|
| 46 |
+
for entry in test_results:
|
| 47 |
+
test_name = entry["movement"].test_name
|
| 48 |
+
if test_name in BILATERAL_TESTS:
|
| 49 |
+
bilateral_groups.setdefault(test_name, []).append(entry)
|
| 50 |
+
else:
|
| 51 |
+
unilateral.append(entry)
|
| 52 |
+
|
| 53 |
+
# Process bilateral tests β take the lower score, emit asymmetry
|
| 54 |
+
for test_name, entries in bilateral_groups.items():
|
| 55 |
+
scores = []
|
| 56 |
+
for entry in entries:
|
| 57 |
+
judge = entry["judge"]
|
| 58 |
+
side = entry.get("side", entry["movement"].side)
|
| 59 |
+
score = judge.score if judge.score is not None else None
|
| 60 |
+
scores.append({"side": side, "score": score, "entry": entry})
|
| 61 |
+
|
| 62 |
+
# Find best entry per side
|
| 63 |
+
left = next((s for s in scores if s["side"] == "left"), None)
|
| 64 |
+
right = next((s for s in scores if s["side"] == "right"), None)
|
| 65 |
+
|
| 66 |
+
left_score = left["score"] if left else None
|
| 67 |
+
right_score = right["score"] if right else None
|
| 68 |
+
|
| 69 |
+
# Report lower
|
| 70 |
+
if left_score is not None and right_score is not None:
|
| 71 |
+
final_score = min(left_score, right_score)
|
| 72 |
+
delta = abs(left_score - right_score)
|
| 73 |
+
asymmetries.append({
|
| 74 |
+
"test": test_name,
|
| 75 |
+
"left_score": left_score,
|
| 76 |
+
"right_score": right_score,
|
| 77 |
+
"delta": delta,
|
| 78 |
+
})
|
| 79 |
+
elif left_score is not None:
|
| 80 |
+
final_score = left_score
|
| 81 |
+
elif right_score is not None:
|
| 82 |
+
final_score = right_score
|
| 83 |
+
else:
|
| 84 |
+
final_score = None
|
| 85 |
+
|
| 86 |
+
# Use the entry with the lower score for details
|
| 87 |
+
primary = (left["entry"] if left and (right is None or (left_score or 4) <= (right_score or 4))
|
| 88 |
+
else right["entry"] if right else entries[0])
|
| 89 |
+
|
| 90 |
+
per_test.append({
|
| 91 |
+
"test_name": test_name,
|
| 92 |
+
"score": final_score,
|
| 93 |
+
"judge": primary["judge"],
|
| 94 |
+
"features": primary["features"],
|
| 95 |
+
"needs_human": primary["judge"].needs_human,
|
| 96 |
+
})
|
| 97 |
+
|
| 98 |
+
self._check_flags(primary, low_confidence_flags, disagreement_flags)
|
| 99 |
+
|
| 100 |
+
# Process unilateral tests
|
| 101 |
+
for entry in unilateral:
|
| 102 |
+
judge = entry["judge"]
|
| 103 |
+
per_test.append({
|
| 104 |
+
"test_name": entry["movement"].test_name,
|
| 105 |
+
"score": judge.score,
|
| 106 |
+
"judge": judge,
|
| 107 |
+
"features": entry["features"],
|
| 108 |
+
"needs_human": judge.needs_human,
|
| 109 |
+
})
|
| 110 |
+
self._check_flags(entry, low_confidence_flags, disagreement_flags)
|
| 111 |
+
|
| 112 |
+
# Composite β null if any test unscored
|
| 113 |
+
all_scores = [t["score"] for t in per_test]
|
| 114 |
+
composite = sum(all_scores) if all(s is not None for s in all_scores) else None
|
| 115 |
+
|
| 116 |
+
return ReportResult(
|
| 117 |
+
per_test=per_test,
|
| 118 |
+
composite=composite,
|
| 119 |
+
asymmetries=asymmetries,
|
| 120 |
+
overlay_video_path=None, # Phase 4
|
| 121 |
+
pdf_path=None, # Phase 4
|
| 122 |
+
low_confidence_flags=low_confidence_flags,
|
| 123 |
+
disagreement_flags=disagreement_flags,
|
| 124 |
+
)
|
| 125 |
+
|
| 126 |
+
def _check_flags(self, entry: dict, low_conf: list, disagree: list):
|
| 127 |
+
"""Check quality gates and populate flag lists."""
|
| 128 |
+
judge = entry["judge"]
|
| 129 |
+
rubric = entry["rubric_score"]
|
| 130 |
+
test_name = entry["movement"].test_name
|
| 131 |
+
|
| 132 |
+
if judge.confidence < config.MIN_CONFIDENCE:
|
| 133 |
+
low_conf.append(f"{test_name}: judge confidence {judge.confidence:.2f}")
|
| 134 |
+
|
| 135 |
+
if (judge.score is not None and rubric.score is not None
|
| 136 |
+
and abs(judge.score - rubric.score) >= config.SCORE_DISAGREE_THRESH):
|
| 137 |
+
disagree.append(
|
| 138 |
+
f"{test_name}: rubric={rubric.score} vs judge={judge.score}"
|
| 139 |
+
)
|
formscout/agents/visualizer.py
ADDED
|
@@ -0,0 +1,418 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
PoseVisualizer β annotated overlay video with skeleton, trails, velocity arrows.
|
| 3 |
+
|
| 4 |
+
Input: IngestResult + Pose2DResult
|
| 5 |
+
Output: .mp4 path (or None on failure/empty layers)
|
| 6 |
+
Failure: returns None, never raises.
|
| 7 |
+
"""
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import colorsys
|
| 11 |
+
import logging
|
| 12 |
+
import math
|
| 13 |
+
import tempfile
|
| 14 |
+
from collections import deque
|
| 15 |
+
|
| 16 |
+
import cv2
|
| 17 |
+
import numpy as np
|
| 18 |
+
|
| 19 |
+
logger = logging.getLogger(__name__)
|
| 20 |
+
|
| 21 |
+
# ββ COCO constants ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 22 |
+
|
| 23 |
+
COCO_KEYPOINTS = [
|
| 24 |
+
"nose", "left_eye", "right_eye", "left_ear", "right_ear",
|
| 25 |
+
"left_shoulder", "right_shoulder", "left_elbow", "right_elbow",
|
| 26 |
+
"left_wrist", "right_wrist", "left_hip", "right_hip",
|
| 27 |
+
"left_knee", "right_knee", "left_ankle", "right_ankle",
|
| 28 |
+
]
|
| 29 |
+
|
| 30 |
+
COCO_SKELETON = [
|
| 31 |
+
(0, 1), (0, 2), (1, 3), (2, 4), # face
|
| 32 |
+
(5, 6), (5, 7), (7, 9), (6, 8), (8, 10), # arms
|
| 33 |
+
(5, 11), (6, 12), (11, 12), # torso
|
| 34 |
+
(11, 13), (13, 15), (12, 14), (14, 16), # legs
|
| 35 |
+
]
|
| 36 |
+
|
| 37 |
+
TRAIL_LENGTH = 10
|
| 38 |
+
MAX_ARROW_PX = 40
|
| 39 |
+
CONF_THRESHOLD = 0.3
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
# ββ Kalman filter βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 43 |
+
|
| 44 |
+
class SimpleKalmanFilter:
|
| 45 |
+
"""4-state Kalman filter (x, y, vx, vy) for joint tracking."""
|
| 46 |
+
|
| 47 |
+
def __init__(self, process_noise: float = 0.01, measurement_noise: float = 0.1):
|
| 48 |
+
self.is_initialized = False
|
| 49 |
+
self.state = np.zeros(4)
|
| 50 |
+
self.cov = np.eye(4) * 0.1
|
| 51 |
+
self.Q = np.eye(4) * process_noise
|
| 52 |
+
self.R = np.eye(2) * measurement_noise
|
| 53 |
+
self.H = np.array([[1, 0, 0, 0], [0, 1, 0, 0]], dtype=float)
|
| 54 |
+
|
| 55 |
+
def predict(self, dt: float = 1.0):
|
| 56 |
+
F = np.array([[1, 0, dt, 0], [0, 1, 0, dt], [0, 0, 1, 0], [0, 0, 0, 1]], dtype=float)
|
| 57 |
+
self.state = F @ self.state
|
| 58 |
+
self.cov = F @ self.cov @ F.T + self.Q
|
| 59 |
+
|
| 60 |
+
def update(self, x: float, y: float):
|
| 61 |
+
z = np.array([x, y])
|
| 62 |
+
if not self.is_initialized:
|
| 63 |
+
self.state[:2] = z
|
| 64 |
+
self.is_initialized = True
|
| 65 |
+
return
|
| 66 |
+
S = self.H @ self.cov @ self.H.T + self.R
|
| 67 |
+
K = self.cov @ self.H.T @ np.linalg.inv(S)
|
| 68 |
+
self.state = self.state + K @ (z - self.H @ self.state)
|
| 69 |
+
self.cov = (np.eye(4) - K @ self.H) @ self.cov
|
| 70 |
+
|
| 71 |
+
def velocity_magnitude(self) -> float:
|
| 72 |
+
vx, vy = self.state[2], self.state[3]
|
| 73 |
+
return math.sqrt(vx * vx + vy * vy)
|
| 74 |
+
|
| 75 |
+
def velocity_vector(self) -> tuple[float, float]:
|
| 76 |
+
return float(self.state[2]), float(self.state[3])
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
# ββ Velocity computation ββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 80 |
+
|
| 81 |
+
def compute_joint_velocity(
|
| 82 |
+
keypoints_per_frame: list[dict],
|
| 83 |
+
fps: float,
|
| 84 |
+
) -> dict[int, list[float]]:
|
| 85 |
+
"""
|
| 86 |
+
Compute Kalman-filtered per-joint speed (px/s) for each frame.
|
| 87 |
+
|
| 88 |
+
Returns dict[joint_idx, [speed_frame0, ...]] for all 17 COCO joints.
|
| 89 |
+
Missing/low-confidence keypoints yield speed=0.0 for that frame.
|
| 90 |
+
"""
|
| 91 |
+
dt = 1.0 / fps if fps > 0 else 1.0
|
| 92 |
+
filters: dict[int, SimpleKalmanFilter] = {j: SimpleKalmanFilter() for j in range(17)}
|
| 93 |
+
result: dict[int, list[float]] = {j: [] for j in range(17)}
|
| 94 |
+
|
| 95 |
+
for frame_kps in keypoints_per_frame:
|
| 96 |
+
for j in range(17):
|
| 97 |
+
kf = filters[j]
|
| 98 |
+
kp = frame_kps.get(j)
|
| 99 |
+
kf.predict(dt)
|
| 100 |
+
if kp and kp.get("conf", 0.0) >= CONF_THRESHOLD:
|
| 101 |
+
kf.update(kp["x"], kp["y"])
|
| 102 |
+
speed = kf.velocity_magnitude()
|
| 103 |
+
else:
|
| 104 |
+
speed = 0.0
|
| 105 |
+
result[j].append(speed)
|
| 106 |
+
|
| 107 |
+
return result
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
# ββ Helpers βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 111 |
+
|
| 112 |
+
def _conf_to_bgr(conf: float) -> tuple[int, int, int]:
|
| 113 |
+
"""Map confidence 0β1 to BGR color redβgreen via HSV."""
|
| 114 |
+
hue = conf * 120.0 / 360.0
|
| 115 |
+
r, g, b = colorsys.hsv_to_rgb(hue, 1.0, 1.0)
|
| 116 |
+
return (int(b * 255), int(g * 255), int(r * 255))
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
# ββ PoseVisualizer ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 120 |
+
|
| 121 |
+
class PoseVisualizer:
|
| 122 |
+
"""Renders skeleton, trails, and velocity arrows onto video frames."""
|
| 123 |
+
|
| 124 |
+
def __init__(self):
|
| 125 |
+
self.last_velocities: dict[int, list[float]] = {}
|
| 126 |
+
|
| 127 |
+
# ββ Skeleton ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 128 |
+
|
| 129 |
+
def _draw_skeleton(self, frame: np.ndarray, kps: dict) -> np.ndarray:
|
| 130 |
+
"""Draw COCO-17 bones (white) and joints (confidence-colored) onto frame."""
|
| 131 |
+
visible = {j: kp for j, kp in kps.items() if kp.get("conf", 0.0) >= CONF_THRESHOLD}
|
| 132 |
+
|
| 133 |
+
# Bones
|
| 134 |
+
for j1, j2 in COCO_SKELETON:
|
| 135 |
+
if j1 in visible and j2 in visible:
|
| 136 |
+
p1 = (int(visible[j1]["x"]), int(visible[j1]["y"]))
|
| 137 |
+
p2 = (int(visible[j2]["x"]), int(visible[j2]["y"]))
|
| 138 |
+
cv2.line(frame, p1, p2, (255, 255, 255), 2)
|
| 139 |
+
|
| 140 |
+
# Joints
|
| 141 |
+
for j, kp in visible.items():
|
| 142 |
+
pt = (int(kp["x"]), int(kp["y"]))
|
| 143 |
+
color = _conf_to_bgr(kp["conf"])
|
| 144 |
+
cv2.circle(frame, pt, 4, color, -1)
|
| 145 |
+
cv2.circle(frame, pt, 5, (255, 255, 255), 1)
|
| 146 |
+
|
| 147 |
+
return frame
|
| 148 |
+
|
| 149 |
+
# ββ Trails βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 150 |
+
|
| 151 |
+
def _draw_trails(self, frame: np.ndarray, trail_history: dict) -> np.ndarray:
|
| 152 |
+
"""Draw fading motion trails for each joint."""
|
| 153 |
+
for joint_idx, trail in trail_history.items():
|
| 154 |
+
pts = list(trail)
|
| 155 |
+
if len(pts) < 2:
|
| 156 |
+
continue
|
| 157 |
+
for i in range(1, len(pts)):
|
| 158 |
+
alpha = i / len(pts)
|
| 159 |
+
brightness = int(255 * alpha)
|
| 160 |
+
color = (brightness, brightness, brightness)
|
| 161 |
+
thickness = max(1, int(3 * alpha))
|
| 162 |
+
p1 = (int(pts[i - 1][0]), int(pts[i - 1][1]))
|
| 163 |
+
p2 = (int(pts[i][0]), int(pts[i][1]))
|
| 164 |
+
cv2.line(frame, p1, p2, color, thickness)
|
| 165 |
+
return frame
|
| 166 |
+
|
| 167 |
+
# ββ Velocity arrows βββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 168 |
+
|
| 169 |
+
def _draw_velocity_arrows(
|
| 170 |
+
self,
|
| 171 |
+
frame: np.ndarray,
|
| 172 |
+
kps: dict,
|
| 173 |
+
prev_kps: dict | None,
|
| 174 |
+
velocities: dict[int, list[float]],
|
| 175 |
+
frame_idx: int,
|
| 176 |
+
) -> np.ndarray:
|
| 177 |
+
"""Draw per-joint velocity arrows scaled by speed."""
|
| 178 |
+
if prev_kps is None:
|
| 179 |
+
return frame
|
| 180 |
+
|
| 181 |
+
all_speeds = [velocities[j][frame_idx] for j in range(17) if frame_idx < len(velocities.get(j, []))]
|
| 182 |
+
peak = max(all_speeds) if all_speeds else 1.0
|
| 183 |
+
if peak == 0.0:
|
| 184 |
+
return frame
|
| 185 |
+
|
| 186 |
+
for j in range(17):
|
| 187 |
+
kp = kps.get(j)
|
| 188 |
+
pk = prev_kps.get(j)
|
| 189 |
+
if not kp or not pk:
|
| 190 |
+
continue
|
| 191 |
+
if kp.get("conf", 0.0) < CONF_THRESHOLD:
|
| 192 |
+
continue
|
| 193 |
+
speeds = velocities.get(j, [])
|
| 194 |
+
if frame_idx >= len(speeds):
|
| 195 |
+
continue
|
| 196 |
+
speed = speeds[frame_idx]
|
| 197 |
+
if speed == 0.0:
|
| 198 |
+
continue
|
| 199 |
+
|
| 200 |
+
dx = kp["x"] - pk["x"]
|
| 201 |
+
dy = kp["y"] - pk["y"]
|
| 202 |
+
mag = math.sqrt(dx * dx + dy * dy)
|
| 203 |
+
if mag < 1e-6:
|
| 204 |
+
continue
|
| 205 |
+
|
| 206 |
+
length = min(speed / peak * MAX_ARROW_PX, MAX_ARROW_PX)
|
| 207 |
+
nx, ny = dx / mag, dy / mag
|
| 208 |
+
start = (int(kp["x"]), int(kp["y"]))
|
| 209 |
+
end = (int(kp["x"] + nx * length), int(kp["y"] + ny * length))
|
| 210 |
+
|
| 211 |
+
ratio = speed / peak
|
| 212 |
+
if ratio < 0.33:
|
| 213 |
+
color = (0, 200, 0) # green
|
| 214 |
+
elif ratio < 0.66:
|
| 215 |
+
color = (0, 140, 255) # orange
|
| 216 |
+
else:
|
| 217 |
+
color = (0, 0, 255) # red
|
| 218 |
+
|
| 219 |
+
cv2.arrowedLine(frame, start, end, color, 2, tipLength=0.35)
|
| 220 |
+
|
| 221 |
+
return frame
|
| 222 |
+
|
| 223 |
+
# ββ Public ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 224 |
+
|
| 225 |
+
def render_video(
|
| 226 |
+
self,
|
| 227 |
+
ingest,
|
| 228 |
+
pose2d,
|
| 229 |
+
layers: set[str],
|
| 230 |
+
output_path: str,
|
| 231 |
+
) -> str | None:
|
| 232 |
+
"""
|
| 233 |
+
Render annotated video. Returns output_path on success, None otherwise.
|
| 234 |
+
layers: subset of {"skeleton", "trails", "velocity_arrows"}
|
| 235 |
+
"""
|
| 236 |
+
if not layers:
|
| 237 |
+
return None
|
| 238 |
+
|
| 239 |
+
if not any(pose2d.keypoints):
|
| 240 |
+
return None
|
| 241 |
+
|
| 242 |
+
try:
|
| 243 |
+
velocities = compute_joint_velocity(pose2d.keypoints, ingest.fps)
|
| 244 |
+
self.last_velocities = velocities
|
| 245 |
+
|
| 246 |
+
frames = ingest.frames
|
| 247 |
+
orig_h, orig_w = frames[0].shape[:2]
|
| 248 |
+
fps = ingest.fps or 30.0
|
| 249 |
+
|
| 250 |
+
# Cap at 1280px wide β big frames are slow and don't need to be HQ
|
| 251 |
+
max_w = 1280
|
| 252 |
+
if orig_w > max_w:
|
| 253 |
+
scale = max_w / orig_w
|
| 254 |
+
out_w = max_w
|
| 255 |
+
out_h = int(orig_h * scale)
|
| 256 |
+
else:
|
| 257 |
+
scale = 1.0
|
| 258 |
+
out_w, out_h = orig_w, orig_h
|
| 259 |
+
|
| 260 |
+
# Scale keypoint coordinates to match resized frames
|
| 261 |
+
def _scale_kps(kps: dict) -> dict:
|
| 262 |
+
if scale == 1.0:
|
| 263 |
+
return kps
|
| 264 |
+
return {
|
| 265 |
+
j: {**kp, "x": kp["x"] * scale, "y": kp["y"] * scale}
|
| 266 |
+
for j, kp in kps.items()
|
| 267 |
+
}
|
| 268 |
+
|
| 269 |
+
scaled_keypoints = [_scale_kps(k) for k in pose2d.keypoints]
|
| 270 |
+
|
| 271 |
+
# Write raw mp4v to a temp file, then remux with ffmpeg faststart
|
| 272 |
+
import subprocess
|
| 273 |
+
import tempfile as _tf
|
| 274 |
+
tmp = _tf.NamedTemporaryFile(suffix="_raw.mp4", delete=False)
|
| 275 |
+
tmp_path = tmp.name
|
| 276 |
+
tmp.close()
|
| 277 |
+
|
| 278 |
+
fourcc = cv2.VideoWriter_fourcc(*"mp4v")
|
| 279 |
+
writer = cv2.VideoWriter(tmp_path, fourcc, fps, (out_w, out_h))
|
| 280 |
+
if not writer.isOpened():
|
| 281 |
+
logger.warning("VideoWriter failed to open: %s", tmp_path)
|
| 282 |
+
return None
|
| 283 |
+
|
| 284 |
+
trail_history: dict[int, deque] = {j: deque(maxlen=TRAIL_LENGTH) for j in range(17)}
|
| 285 |
+
prev_kps: dict | None = None
|
| 286 |
+
|
| 287 |
+
for frame_idx, (frame, kps) in enumerate(zip(frames, scaled_keypoints)):
|
| 288 |
+
if scale != 1.0:
|
| 289 |
+
out_frame = cv2.resize(frame, (out_w, out_h), interpolation=cv2.INTER_AREA)
|
| 290 |
+
else:
|
| 291 |
+
out_frame = frame.copy()
|
| 292 |
+
|
| 293 |
+
if "trails" in layers:
|
| 294 |
+
for j, kp in kps.items():
|
| 295 |
+
if kp.get("conf", 0.0) >= CONF_THRESHOLD:
|
| 296 |
+
trail_history[j].append((kp["x"], kp["y"]))
|
| 297 |
+
out_frame = self._draw_trails(out_frame, trail_history)
|
| 298 |
+
|
| 299 |
+
if "skeleton" in layers:
|
| 300 |
+
out_frame = self._draw_skeleton(out_frame, kps)
|
| 301 |
+
|
| 302 |
+
if "velocity_arrows" in layers:
|
| 303 |
+
out_frame = self._draw_velocity_arrows(
|
| 304 |
+
out_frame, kps, prev_kps, velocities, frame_idx
|
| 305 |
+
)
|
| 306 |
+
|
| 307 |
+
writer.write(out_frame)
|
| 308 |
+
prev_kps = kps
|
| 309 |
+
|
| 310 |
+
writer.release()
|
| 311 |
+
|
| 312 |
+
# Remux with faststart so browsers can seek without downloading the whole file
|
| 313 |
+
try:
|
| 314 |
+
subprocess.run(
|
| 315 |
+
["ffmpeg", "-y", "-i", tmp_path, "-c", "copy",
|
| 316 |
+
"-movflags", "+faststart", output_path],
|
| 317 |
+
check=True, capture_output=True,
|
| 318 |
+
)
|
| 319 |
+
import os
|
| 320 |
+
os.unlink(tmp_path)
|
| 321 |
+
except Exception as ffmpeg_err:
|
| 322 |
+
logger.warning("ffmpeg remux failed (%s) β using raw mp4v", ffmpeg_err)
|
| 323 |
+
import shutil
|
| 324 |
+
shutil.move(tmp_path, output_path)
|
| 325 |
+
|
| 326 |
+
return output_path
|
| 327 |
+
|
| 328 |
+
except Exception as e:
|
| 329 |
+
logger.warning("render_video failed: %s", e)
|
| 330 |
+
return None
|
| 331 |
+
|
| 332 |
+
def render_frame(
|
| 333 |
+
self,
|
| 334 |
+
ingest,
|
| 335 |
+
pose2d,
|
| 336 |
+
frame_idx: int,
|
| 337 |
+
layers: set[str],
|
| 338 |
+
caption: str = "",
|
| 339 |
+
out_png: str | None = None,
|
| 340 |
+
) -> str | None:
|
| 341 |
+
"""Render a single annotated still (skeleton + optional trails + caption).
|
| 342 |
+
|
| 343 |
+
frame_idx is typically the governing frame from BiomechFeatures.timing.
|
| 344 |
+
Returns the PNG path on success, None on any failure. Never raises.
|
| 345 |
+
"""
|
| 346 |
+
try:
|
| 347 |
+
if not (0 <= frame_idx < len(ingest.frames)) or frame_idx >= len(pose2d.keypoints):
|
| 348 |
+
return None
|
| 349 |
+
|
| 350 |
+
frame = ingest.frames[frame_idx].copy()
|
| 351 |
+
kps = pose2d.keypoints[frame_idx]
|
| 352 |
+
|
| 353 |
+
if "trails" in layers:
|
| 354 |
+
trail: dict[int, deque] = {j: deque(maxlen=TRAIL_LENGTH) for j in range(17)}
|
| 355 |
+
start = max(0, frame_idx - TRAIL_LENGTH)
|
| 356 |
+
for fi in range(start, frame_idx + 1):
|
| 357 |
+
for j, kp in pose2d.keypoints[fi].items():
|
| 358 |
+
if kp.get("conf", 0.0) >= CONF_THRESHOLD:
|
| 359 |
+
trail[j].append((kp["x"], kp["y"]))
|
| 360 |
+
frame = self._draw_trails(frame, trail)
|
| 361 |
+
|
| 362 |
+
if "skeleton" in layers:
|
| 363 |
+
frame = self._draw_skeleton(frame, kps)
|
| 364 |
+
|
| 365 |
+
if caption:
|
| 366 |
+
cv2.rectangle(frame, (0, 0), (frame.shape[1], 28), (0, 0, 0), -1)
|
| 367 |
+
cv2.putText(frame, caption[:80], (8, 20), cv2.FONT_HERSHEY_SIMPLEX,
|
| 368 |
+
0.55, (255, 255, 255), 1, cv2.LINE_AA)
|
| 369 |
+
|
| 370 |
+
if out_png is None:
|
| 371 |
+
out_png = tempfile.NamedTemporaryFile(suffix=".png", delete=False).name
|
| 372 |
+
|
| 373 |
+
ok = cv2.imwrite(out_png, frame)
|
| 374 |
+
return out_png if ok else None
|
| 375 |
+
except Exception as e:
|
| 376 |
+
logger.warning("render_frame failed: %s", e)
|
| 377 |
+
return None
|
| 378 |
+
|
| 379 |
+
|
| 380 |
+
# ββ Velocity summary ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 381 |
+
|
| 382 |
+
def build_velocity_summary(
|
| 383 |
+
keypoints_per_frame: list[dict],
|
| 384 |
+
velocities: dict[int, list[float]],
|
| 385 |
+
) -> str:
|
| 386 |
+
"""Return markdown table of per-joint avg/peak velocity. Empty string if no valid joints."""
|
| 387 |
+
n_frames = len(keypoints_per_frame)
|
| 388 |
+
if n_frames == 0:
|
| 389 |
+
return ""
|
| 390 |
+
|
| 391 |
+
rows = []
|
| 392 |
+
for j in range(17):
|
| 393 |
+
detected = sum(
|
| 394 |
+
1 for kps in keypoints_per_frame
|
| 395 |
+
if kps.get(j, {}).get("conf", 0.0) >= CONF_THRESHOLD
|
| 396 |
+
)
|
| 397 |
+
if detected < n_frames * 0.5:
|
| 398 |
+
continue
|
| 399 |
+
|
| 400 |
+
speeds = velocities.get(j, [])
|
| 401 |
+
if not speeds:
|
| 402 |
+
continue
|
| 403 |
+
|
| 404 |
+
avg_speed = sum(speeds) / len(speeds)
|
| 405 |
+
peak_speed = max(speeds)
|
| 406 |
+
rows.append((COCO_KEYPOINTS[j], avg_speed, peak_speed))
|
| 407 |
+
|
| 408 |
+
if not rows:
|
| 409 |
+
return ""
|
| 410 |
+
|
| 411 |
+
rows.sort(key=lambda r: r[2], reverse=True)
|
| 412 |
+
lines = [
|
| 413 |
+
"| Joint | Avg (px/s) | Peak (px/s) |",
|
| 414 |
+
"|---|---|---|",
|
| 415 |
+
]
|
| 416 |
+
for name, avg, peak in rows:
|
| 417 |
+
lines.append(f"| {name} | {avg:.1f} | {peak:.1f} |")
|
| 418 |
+
return "\n".join(lines)
|
formscout/analysis/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""FormScout movement-analysis engine: relevant joints, time series, Laban, charts."""
|
formscout/analysis/charts.py
ADDED
|
@@ -0,0 +1,171 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Matplotlib chart generators for the screening report.
|
| 3 |
+
|
| 4 |
+
Every function returns a PNG path on success or None on failure (never raises),
|
| 5 |
+
so a chart problem degrades the report but never blocks scoring. Charts use the
|
| 6 |
+
Silas palette and an Agg backend so they render headless on the Space.
|
| 7 |
+
"""
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import logging
|
| 11 |
+
|
| 12 |
+
import matplotlib
|
| 13 |
+
|
| 14 |
+
matplotlib.use("Agg")
|
| 15 |
+
import matplotlib.pyplot as plt # noqa: E402
|
| 16 |
+
import numpy as np # noqa: E402
|
| 17 |
+
|
| 18 |
+
from formscout.analysis.relevant_joints import COCO_NAMES # noqa: E402
|
| 19 |
+
|
| 20 |
+
logger = logging.getLogger(__name__)
|
| 21 |
+
|
| 22 |
+
TEAL = "#2b8a8a"
|
| 23 |
+
GOLD = "#e0a43b"
|
| 24 |
+
SAGE = "#9cbcad"
|
| 25 |
+
INK = "#243a34"
|
| 26 |
+
RED = "#d9534f"
|
| 27 |
+
_PALETTE = [TEAL, GOLD, SAGE, "#7a5ca0", "#c2683c", "#3c8dbc"]
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _save(fig, out_png: str) -> str | None:
|
| 31 |
+
try:
|
| 32 |
+
fig.savefig(out_png, dpi=110, bbox_inches="tight", facecolor="white")
|
| 33 |
+
return out_png
|
| 34 |
+
except Exception as e:
|
| 35 |
+
logger.warning("chart save failed: %s", e)
|
| 36 |
+
return None
|
| 37 |
+
finally:
|
| 38 |
+
plt.close(fig)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def angle_over_time(series: dict, primary: str | None, governing_idx: int | None,
|
| 42 |
+
out_png: str, title: str = "Joint angle over time") -> str | None:
|
| 43 |
+
"""Angle-vs-frame for the relevant angles; primary emphasised, key-frame marked."""
|
| 44 |
+
try:
|
| 45 |
+
if not series:
|
| 46 |
+
return None
|
| 47 |
+
fig, ax = plt.subplots(figsize=(6.4, 3.2))
|
| 48 |
+
for i, (name, vals) in enumerate(series.items()):
|
| 49 |
+
arr = np.array(vals, dtype=float)
|
| 50 |
+
is_primary = name == primary
|
| 51 |
+
ax.plot(np.arange(len(arr)), arr,
|
| 52 |
+
color=(TEAL if is_primary else _PALETTE[i % len(_PALETTE)]),
|
| 53 |
+
lw=2.4 if is_primary else 1.3,
|
| 54 |
+
alpha=1.0 if is_primary else 0.6,
|
| 55 |
+
label=name.replace("_", " ") + (" β
" if is_primary else ""))
|
| 56 |
+
if governing_idx is not None:
|
| 57 |
+
ax.axvline(governing_idx, color=GOLD, ls="--", lw=1.5, label="key frame")
|
| 58 |
+
ax.set_xlabel("frame")
|
| 59 |
+
ax.set_ylabel("degrees")
|
| 60 |
+
ax.set_title(title, color=INK)
|
| 61 |
+
ax.legend(fontsize=7, loc="best")
|
| 62 |
+
ax.grid(True, alpha=0.2)
|
| 63 |
+
return _save(fig, out_png)
|
| 64 |
+
except Exception as e:
|
| 65 |
+
logger.warning("angle_over_time failed: %s", e)
|
| 66 |
+
return None
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def velocity_profile(keypoints: list, fps: float, joints: list[int],
|
| 70 |
+
out_png: str, title: str = "Joint speed over time") -> str | None:
|
| 71 |
+
"""Per-frame speed (px/s) of the relevant joints across the clip."""
|
| 72 |
+
try:
|
| 73 |
+
from formscout.agents.visualizer import compute_joint_velocity
|
| 74 |
+
vel = compute_joint_velocity(keypoints, fps or 30.0)
|
| 75 |
+
plot_joints = [j for j in joints if j in vel] or list(vel.keys())[:4]
|
| 76 |
+
if not plot_joints:
|
| 77 |
+
return None
|
| 78 |
+
fig, ax = plt.subplots(figsize=(6.4, 3.2))
|
| 79 |
+
for i, j in enumerate(plot_joints):
|
| 80 |
+
ax.plot(vel[j], color=_PALETTE[i % len(_PALETTE)], lw=1.6,
|
| 81 |
+
label=COCO_NAMES.get(j, str(j)).replace("_", " "))
|
| 82 |
+
ax.set_xlabel("frame")
|
| 83 |
+
ax.set_ylabel("speed (px/s)")
|
| 84 |
+
ax.set_title(title, color=INK)
|
| 85 |
+
ax.legend(fontsize=7, loc="best")
|
| 86 |
+
ax.grid(True, alpha=0.2)
|
| 87 |
+
return _save(fig, out_png)
|
| 88 |
+
except Exception as e:
|
| 89 |
+
logger.warning("velocity_profile failed: %s", e)
|
| 90 |
+
return None
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def laban_radar(effort: dict, out_png: str, title: str = "Laban Effort") -> str | None:
|
| 94 |
+
"""4-axis radar of the Effort factors (Space, Weight, Time, Flow)."""
|
| 95 |
+
try:
|
| 96 |
+
axes_order = ["space", "weight", "time", "flow"]
|
| 97 |
+
labels = ["Space\n(direct)", "Weight\n(strong)", "Time\n(sudden)", "Flow\n(free)"]
|
| 98 |
+
vals = [float(effort.get(k, 0.0)) for k in axes_order]
|
| 99 |
+
angles = np.linspace(0, 2 * np.pi, len(axes_order), endpoint=False).tolist()
|
| 100 |
+
vals_loop = vals + vals[:1]
|
| 101 |
+
angles_loop = angles + angles[:1]
|
| 102 |
+
|
| 103 |
+
fig, ax = plt.subplots(figsize=(4.2, 4.2), subplot_kw={"polar": True})
|
| 104 |
+
ax.plot(angles_loop, vals_loop, color=TEAL, lw=2)
|
| 105 |
+
ax.fill(angles_loop, vals_loop, color=TEAL, alpha=0.25)
|
| 106 |
+
ax.set_xticks(angles)
|
| 107 |
+
ax.set_xticklabels(labels, fontsize=8, color=INK)
|
| 108 |
+
ax.set_ylim(0, 1)
|
| 109 |
+
ax.set_yticks([0.25, 0.5, 0.75, 1.0])
|
| 110 |
+
ax.set_yticklabels(["", "0.5", "", "1.0"], fontsize=7)
|
| 111 |
+
ax.set_title(title, color=INK, pad=18)
|
| 112 |
+
return _save(fig, out_png)
|
| 113 |
+
except Exception as e:
|
| 114 |
+
logger.warning("laban_radar failed: %s", e)
|
| 115 |
+
return None
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def flexion_bars(flexion: dict, out_png: str,
|
| 119 |
+
title: str = "Relevant joint flexion") -> str | None:
|
| 120 |
+
"""Horizontal bars of relevant joint angles (deg) at the key frame."""
|
| 121 |
+
try:
|
| 122 |
+
if not flexion:
|
| 123 |
+
return None
|
| 124 |
+
names = [n.replace("_", " ") for n in flexion]
|
| 125 |
+
degs = [flexion[n]["deg"] for n in flexion]
|
| 126 |
+
colors = [TEAL if d >= 160 else GOLD if d >= 110 else RED for d in degs]
|
| 127 |
+
fig, ax = plt.subplots(figsize=(6.0, max(1.6, 0.5 * len(names) + 0.8)))
|
| 128 |
+
y = np.arange(len(names))
|
| 129 |
+
ax.barh(y, degs, color=colors)
|
| 130 |
+
ax.set_yticks(y)
|
| 131 |
+
ax.set_yticklabels(names, fontsize=8)
|
| 132 |
+
ax.set_xlim(0, 200)
|
| 133 |
+
ax.axvline(160, color=SAGE, ls=":", lw=1)
|
| 134 |
+
for yi, d in zip(y, degs):
|
| 135 |
+
ax.text(d + 3, yi, f"{d:.0f}Β°", va="center", fontsize=8, color=INK)
|
| 136 |
+
ax.set_xlabel("interior angle (Β°) Β· higher = more open")
|
| 137 |
+
ax.set_title(title, color=INK)
|
| 138 |
+
ax.invert_yaxis()
|
| 139 |
+
return _save(fig, out_png)
|
| 140 |
+
except Exception as e:
|
| 141 |
+
logger.warning("flexion_bars failed: %s", e)
|
| 142 |
+
return None
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def symmetry_bars(asymmetries: list, out_png: str,
|
| 146 |
+
title: str = "Left / right symmetry") -> str | None:
|
| 147 |
+
"""Grouped L vs R score bars for bilateral tests."""
|
| 148 |
+
try:
|
| 149 |
+
rows = [a for a in asymmetries
|
| 150 |
+
if a.get("left_score") is not None and a.get("right_score") is not None]
|
| 151 |
+
if not rows:
|
| 152 |
+
return None
|
| 153 |
+
names = [a["test"].replace("_", " ") for a in rows]
|
| 154 |
+
left = [a["left_score"] for a in rows]
|
| 155 |
+
right = [a["right_score"] for a in rows]
|
| 156 |
+
x = np.arange(len(names))
|
| 157 |
+
w = 0.36
|
| 158 |
+
fig, ax = plt.subplots(figsize=(6.4, 3.2))
|
| 159 |
+
ax.bar(x - w / 2, left, w, color=TEAL, label="left")
|
| 160 |
+
ax.bar(x + w / 2, right, w, color=GOLD, label="right")
|
| 161 |
+
ax.set_xticks(x)
|
| 162 |
+
ax.set_xticklabels(names, fontsize=8, rotation=15, ha="right")
|
| 163 |
+
ax.set_ylim(0, 3.4)
|
| 164 |
+
ax.set_ylabel("score (0β3)")
|
| 165 |
+
ax.set_title(title, color=INK)
|
| 166 |
+
ax.legend(fontsize=8)
|
| 167 |
+
ax.grid(True, axis="y", alpha=0.2)
|
| 168 |
+
return _save(fig, out_png)
|
| 169 |
+
except Exception as e:
|
| 170 |
+
logger.warning("symmetry_bars failed: %s", e)
|
| 171 |
+
return None
|
formscout/analysis/laban.py
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Laban Movement Analysis β Effort factors from pose kinematics.
|
| 3 |
+
|
| 4 |
+
Computes the four Effort factors over the joints relevant to a screening test:
|
| 5 |
+
|
| 6 |
+
- Space (indirect 0 β¦ direct 1) β path directness of the leading joint
|
| 7 |
+
- Weight (light 0 β¦ strong 1) β motion-energy of the relevant joints
|
| 8 |
+
- Time (sustained 0 β¦ sudden 1) β impulsivity (peak vs mean speed)
|
| 9 |
+
- Flow (bound 0 β¦ free 1) β smoothness (inverse normalised jerk)
|
| 10 |
+
|
| 11 |
+
These are reproducible kinematic heuristics, not clinical LMA notation β the
|
| 12 |
+
report labels them as such. Distances are normalised by torso length so the
|
| 13 |
+
factors are scale-invariant across cameras. Pure function β no model, no I/O.
|
| 14 |
+
"""
|
| 15 |
+
from __future__ import annotations
|
| 16 |
+
|
| 17 |
+
import math
|
| 18 |
+
|
| 19 |
+
from formscout.agents.biomechanics import _get_joint
|
| 20 |
+
from formscout.analysis.relevant_joints import (
|
| 21 |
+
COCO_NAMES, L_HIP, L_SHOULDER, R_HIP, R_SHOULDER, relevant_joints,
|
| 22 |
+
)
|
| 23 |
+
|
| 24 |
+
# Heuristic calibration references (movement is normalised to torso-lengths/sec).
|
| 25 |
+
_WEIGHT_REF = 1.0 # energy (bl/s)^2 giving ~0.63 weight
|
| 26 |
+
_TIME_LO, _TIME_HI = 1.5, 4.0 # peak/mean speed ratio mapped to [0, 1]
|
| 27 |
+
_FLOW_JERK_REF = 6.0 # normalised jerk (bl/s^3) giving ~0.37 flow
|
| 28 |
+
|
| 29 |
+
_LABELS = {
|
| 30 |
+
"space": ("indirect", "direct"),
|
| 31 |
+
"weight": ("light", "strong"),
|
| 32 |
+
"time": ("sustained", "sudden"),
|
| 33 |
+
"flow": ("bound", "free"),
|
| 34 |
+
}
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _torso_scale(frames) -> float:
|
| 38 |
+
"""Median shoulder-hip distance across frames; 1.0 if unmeasurable."""
|
| 39 |
+
lengths = []
|
| 40 |
+
for kps in frames:
|
| 41 |
+
for sh, hip in ((L_SHOULDER, L_HIP), (R_SHOULDER, R_HIP)):
|
| 42 |
+
a, b = _get_joint(kps, sh), _get_joint(kps, hip)
|
| 43 |
+
if a and b:
|
| 44 |
+
lengths.append(math.hypot(a[0] - b[0], a[1] - b[1]))
|
| 45 |
+
if not lengths:
|
| 46 |
+
return 1.0
|
| 47 |
+
lengths.sort()
|
| 48 |
+
med = lengths[len(lengths) // 2]
|
| 49 |
+
return med if med > 1e-6 else 1.0
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def _joint_kinematics(frames, joint_id: int, dt: float, scale: float) -> dict | None:
|
| 53 |
+
"""Speed/accel/jerk/directness for one joint trajectory (torso-length units)."""
|
| 54 |
+
pts = [_get_joint(kps, joint_id) for kps in frames]
|
| 55 |
+
valid = [(i, p) for i, p in enumerate(pts) if p is not None]
|
| 56 |
+
if len(valid) < 3:
|
| 57 |
+
return None
|
| 58 |
+
|
| 59 |
+
speeds, path_len = [], 0.0
|
| 60 |
+
for (i0, p0), (i1, p1) in zip(valid, valid[1:]):
|
| 61 |
+
d = math.hypot(p1[0] - p0[0], p1[1] - p0[1]) / scale
|
| 62 |
+
path_len += d
|
| 63 |
+
gap = max(1, i1 - i0)
|
| 64 |
+
speeds.append(d / (gap * dt))
|
| 65 |
+
if not speeds:
|
| 66 |
+
return None
|
| 67 |
+
|
| 68 |
+
net = math.hypot(valid[-1][1][0] - valid[0][1][0],
|
| 69 |
+
valid[-1][1][1] - valid[0][1][1]) / scale
|
| 70 |
+
directness = net / path_len if path_len > 1e-6 else 0.0
|
| 71 |
+
|
| 72 |
+
accels = [abs(speeds[i + 1] - speeds[i]) / dt for i in range(len(speeds) - 1)]
|
| 73 |
+
jerks = [abs(accels[i + 1] - accels[i]) / dt for i in range(len(accels) - 1)]
|
| 74 |
+
|
| 75 |
+
mean_speed = sum(speeds) / len(speeds)
|
| 76 |
+
peak_speed = max(speeds)
|
| 77 |
+
return {
|
| 78 |
+
"mean_speed": mean_speed,
|
| 79 |
+
"peak_speed": peak_speed,
|
| 80 |
+
"energy": sum(s * s for s in speeds) / len(speeds),
|
| 81 |
+
"ratio": peak_speed / (mean_speed + 1e-6),
|
| 82 |
+
"mean_jerk": (sum(jerks) / len(jerks)) if jerks else 0.0,
|
| 83 |
+
"directness": min(1.0, max(0.0, directness)),
|
| 84 |
+
}
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def _clip01(x: float) -> float:
|
| 88 |
+
return min(1.0, max(0.0, x))
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def compute_laban(pose2d, test_name: str, fps: float) -> dict:
|
| 92 |
+
"""Return the four Effort factors, their labels, and body emphasis."""
|
| 93 |
+
frames = pose2d.keypoints
|
| 94 |
+
dt = 1.0 / fps if fps and fps > 0 else 1.0 / 30.0
|
| 95 |
+
scale = _torso_scale(frames)
|
| 96 |
+
joints = relevant_joints(test_name) or list(range(17))
|
| 97 |
+
|
| 98 |
+
kin = {j: k for j in joints if (k := _joint_kinematics(frames, j, dt, scale))}
|
| 99 |
+
if not kin:
|
| 100 |
+
return {
|
| 101 |
+
"effort": {"space": 0.0, "weight": 0.0, "time": 0.0, "flow": 0.0},
|
| 102 |
+
"labels": {k: v[0] for k, v in _LABELS.items()},
|
| 103 |
+
"body_emphasis": [],
|
| 104 |
+
"notes": "insufficient motion to estimate Effort",
|
| 105 |
+
}
|
| 106 |
+
|
| 107 |
+
leader = max(kin, key=lambda j: kin[j]["mean_speed"])
|
| 108 |
+
lead = kin[leader]
|
| 109 |
+
|
| 110 |
+
weight = _clip01(1.0 - math.exp(-(sum(k["energy"] for k in kin.values()) / len(kin)) / _WEIGHT_REF))
|
| 111 |
+
time = _clip01((lead["ratio"] - _TIME_LO) / (_TIME_HI - _TIME_LO))
|
| 112 |
+
flow = _clip01(math.exp(-lead["mean_jerk"] / _FLOW_JERK_REF))
|
| 113 |
+
space = _clip01(lead["directness"])
|
| 114 |
+
|
| 115 |
+
effort = {"space": space, "weight": weight, "time": time, "flow": flow}
|
| 116 |
+
labels = {k: _LABELS[k][1] if v >= 0.5 else _LABELS[k][0] for k, v in effort.items()}
|
| 117 |
+
|
| 118 |
+
emphasis = sorted(kin.items(), key=lambda kv: kv[1]["mean_speed"], reverse=True)[:3]
|
| 119 |
+
body_emphasis = [(COCO_NAMES.get(j, str(j)), round(k["mean_speed"], 3)) for j, k in emphasis]
|
| 120 |
+
|
| 121 |
+
return {
|
| 122 |
+
"effort": {k: round(v, 3) for k, v in effort.items()},
|
| 123 |
+
"labels": labels,
|
| 124 |
+
"body_emphasis": body_emphasis,
|
| 125 |
+
"leading_joint": COCO_NAMES.get(leader, str(leader)),
|
| 126 |
+
"notes": "kinematic Effort estimate (heuristic, not clinical LMA notation)",
|
| 127 |
+
}
|
formscout/analysis/relevant_joints.py
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Per-test relevant-joint and relevant-angle definitions.
|
| 3 |
+
|
| 4 |
+
Drives the "always describe just the joint relevant to the screening action"
|
| 5 |
+
requirement: each FMS test names the joints and the joint-angles that matter for
|
| 6 |
+
its rubric, plus the single primary angle used for the headline angle-over-time
|
| 7 |
+
graph. Angles are COCO (a, b, c) triplets β the angle is measured at joint b.
|
| 8 |
+
"""
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
# COCO-17 joint indices
|
| 12 |
+
NOSE = 0
|
| 13 |
+
L_SHOULDER, R_SHOULDER = 5, 6
|
| 14 |
+
L_ELBOW, R_ELBOW = 7, 8
|
| 15 |
+
L_WRIST, R_WRIST = 9, 10
|
| 16 |
+
L_HIP, R_HIP = 11, 12
|
| 17 |
+
L_KNEE, R_KNEE = 13, 14
|
| 18 |
+
L_ANKLE, R_ANKLE = 15, 16
|
| 19 |
+
|
| 20 |
+
COCO_NAMES = {
|
| 21 |
+
0: "nose", 1: "left_eye", 2: "right_eye", 3: "left_ear", 4: "right_ear",
|
| 22 |
+
5: "left_shoulder", 6: "right_shoulder", 7: "left_elbow", 8: "right_elbow",
|
| 23 |
+
9: "left_wrist", 10: "right_wrist", 11: "left_hip", 12: "right_hip",
|
| 24 |
+
13: "left_knee", 14: "right_knee", 15: "left_ankle", 16: "right_ankle",
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
# Each test: the joints that matter, the named angles (a, b, c) measured at b,
|
| 28 |
+
# and the primary angle for the headline graph.
|
| 29 |
+
RELEVANT: dict[str, dict] = {
|
| 30 |
+
"deep_squat": {
|
| 31 |
+
"joints": [L_SHOULDER, R_SHOULDER, L_HIP, R_HIP, L_KNEE, R_KNEE, L_ANKLE, R_ANKLE],
|
| 32 |
+
"angles": {
|
| 33 |
+
"left_knee_flexion": (L_HIP, L_KNEE, L_ANKLE),
|
| 34 |
+
"right_knee_flexion": (R_HIP, R_KNEE, R_ANKLE),
|
| 35 |
+
"left_hip_flexion": (L_SHOULDER, L_HIP, L_KNEE),
|
| 36 |
+
"right_hip_flexion": (R_SHOULDER, R_HIP, R_KNEE),
|
| 37 |
+
},
|
| 38 |
+
"primary_angle": "left_knee_flexion",
|
| 39 |
+
},
|
| 40 |
+
"hurdle_step": {
|
| 41 |
+
"joints": [L_HIP, R_HIP, L_KNEE, R_KNEE, L_ANKLE, R_ANKLE],
|
| 42 |
+
"angles": {
|
| 43 |
+
"left_knee_flexion": (L_HIP, L_KNEE, L_ANKLE),
|
| 44 |
+
"right_knee_flexion": (R_HIP, R_KNEE, R_ANKLE),
|
| 45 |
+
"left_hip_flexion": (L_SHOULDER, L_HIP, L_KNEE),
|
| 46 |
+
"right_hip_flexion": (R_SHOULDER, R_HIP, R_KNEE),
|
| 47 |
+
},
|
| 48 |
+
"primary_angle": "left_hip_flexion",
|
| 49 |
+
},
|
| 50 |
+
"inline_lunge": {
|
| 51 |
+
"joints": [L_HIP, R_HIP, L_KNEE, R_KNEE, L_ANKLE, R_ANKLE],
|
| 52 |
+
"angles": {
|
| 53 |
+
"left_knee_flexion": (L_HIP, L_KNEE, L_ANKLE),
|
| 54 |
+
"right_knee_flexion": (R_HIP, R_KNEE, R_ANKLE),
|
| 55 |
+
},
|
| 56 |
+
"primary_angle": "left_knee_flexion",
|
| 57 |
+
},
|
| 58 |
+
"shoulder_mobility": {
|
| 59 |
+
"joints": [L_SHOULDER, R_SHOULDER, L_ELBOW, R_ELBOW, L_WRIST, R_WRIST],
|
| 60 |
+
"angles": {
|
| 61 |
+
"left_shoulder_angle": (L_ELBOW, L_SHOULDER, L_HIP),
|
| 62 |
+
"right_shoulder_angle": (R_ELBOW, R_SHOULDER, R_HIP),
|
| 63 |
+
"left_elbow_flexion": (L_SHOULDER, L_ELBOW, L_WRIST),
|
| 64 |
+
"right_elbow_flexion": (R_SHOULDER, R_ELBOW, R_WRIST),
|
| 65 |
+
},
|
| 66 |
+
"primary_angle": "left_shoulder_angle",
|
| 67 |
+
},
|
| 68 |
+
"active_slr": {
|
| 69 |
+
"joints": [L_HIP, R_HIP, L_KNEE, R_KNEE, L_ANKLE, R_ANKLE],
|
| 70 |
+
"angles": {
|
| 71 |
+
"left_hip_flexion": (L_SHOULDER, L_HIP, L_KNEE),
|
| 72 |
+
"right_hip_flexion": (R_SHOULDER, R_HIP, R_KNEE),
|
| 73 |
+
"left_knee_flexion": (L_HIP, L_KNEE, L_ANKLE),
|
| 74 |
+
"right_knee_flexion": (R_HIP, R_KNEE, R_ANKLE),
|
| 75 |
+
},
|
| 76 |
+
"primary_angle": "left_hip_flexion",
|
| 77 |
+
},
|
| 78 |
+
"trunk_stability_pushup": {
|
| 79 |
+
"joints": [L_SHOULDER, R_SHOULDER, L_ELBOW, R_ELBOW, L_HIP, R_HIP, L_ANKLE, R_ANKLE],
|
| 80 |
+
"angles": {
|
| 81 |
+
"left_elbow_flexion": (L_SHOULDER, L_ELBOW, L_WRIST),
|
| 82 |
+
"right_elbow_flexion": (R_SHOULDER, R_ELBOW, R_WRIST),
|
| 83 |
+
"left_hip_line": (L_SHOULDER, L_HIP, L_ANKLE),
|
| 84 |
+
"right_hip_line": (R_SHOULDER, R_HIP, R_ANKLE),
|
| 85 |
+
},
|
| 86 |
+
"primary_angle": "left_hip_line",
|
| 87 |
+
},
|
| 88 |
+
"rotary_stability": {
|
| 89 |
+
"joints": [L_SHOULDER, R_SHOULDER, L_ELBOW, R_ELBOW, L_HIP, R_HIP, L_KNEE, R_KNEE],
|
| 90 |
+
"angles": {
|
| 91 |
+
"left_hip_line": (L_SHOULDER, L_HIP, L_KNEE),
|
| 92 |
+
"right_hip_line": (R_SHOULDER, R_HIP, R_KNEE),
|
| 93 |
+
},
|
| 94 |
+
"primary_angle": "left_hip_line",
|
| 95 |
+
},
|
| 96 |
+
}
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def relevant_joints(test_name: str) -> list[int]:
|
| 100 |
+
"""COCO joint indices relevant to this test (empty for unknown tests)."""
|
| 101 |
+
return list(RELEVANT.get(test_name, {}).get("joints", []))
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def relevant_angles(test_name: str) -> dict[str, tuple]:
|
| 105 |
+
"""Named (a, b, c) angle triplets relevant to this test."""
|
| 106 |
+
return dict(RELEVANT.get(test_name, {}).get("angles", {}))
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def primary_angle(test_name: str) -> str | None:
|
| 110 |
+
"""Name of the headline angle for the angle-over-time graph, or None."""
|
| 111 |
+
return RELEVANT.get(test_name, {}).get("primary_angle")
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def openness_label(angle_deg: float) -> str:
|
| 115 |
+
"""Describe how open/closed a joint is from its interior angle in degrees."""
|
| 116 |
+
if angle_deg >= 160:
|
| 117 |
+
return "open / extended"
|
| 118 |
+
if angle_deg >= 110:
|
| 119 |
+
return "mid-range"
|
| 120 |
+
if angle_deg >= 60:
|
| 121 |
+
return "flexed"
|
| 122 |
+
return "deeply flexed / closed"
|
formscout/analysis/timeseries.py
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Per-frame time series for the joints relevant to a screening test.
|
| 3 |
+
|
| 4 |
+
Builds angle-over-time series (used by the angle graph) and a per-joint flexion
|
| 5 |
+
summary at a chosen frame (degrees + open/closed label). Reuses the biomechanics
|
| 6 |
+
geometry helpers so angle definitions never diverge.
|
| 7 |
+
"""
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import math
|
| 11 |
+
|
| 12 |
+
from formscout.agents.biomechanics import _angle_between_points, _get_joint
|
| 13 |
+
from formscout.analysis.relevant_joints import openness_label, relevant_angles
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def angle_series(pose2d, test_name: str) -> dict[str, list[float]]:
|
| 17 |
+
"""Return {angle_name: [deg per frame]} for this test's relevant angles.
|
| 18 |
+
|
| 19 |
+
Frames where the angle cannot be measured hold NaN.
|
| 20 |
+
"""
|
| 21 |
+
angles = relevant_angles(test_name)
|
| 22 |
+
series: dict[str, list[float]] = {name: [] for name in angles}
|
| 23 |
+
for kps in pose2d.keypoints:
|
| 24 |
+
for name, (a, b, c) in angles.items():
|
| 25 |
+
pa, pb, pc = _get_joint(kps, a), _get_joint(kps, b), _get_joint(kps, c)
|
| 26 |
+
if pa and pb and pc:
|
| 27 |
+
series[name].append(_angle_between_points(pa, pb, pc))
|
| 28 |
+
else:
|
| 29 |
+
series[name].append(float("nan"))
|
| 30 |
+
return series
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def relevant_flexion_at(pose2d, test_name: str, frame_idx: int) -> dict[str, dict]:
|
| 34 |
+
"""At one frame, the relevant joint angles with degree value + openness label.
|
| 35 |
+
|
| 36 |
+
Returns {angle_name: {"deg": float, "openness": str}}; angles that cannot be
|
| 37 |
+
measured at that frame are omitted.
|
| 38 |
+
"""
|
| 39 |
+
out: dict[str, dict] = {}
|
| 40 |
+
if not (0 <= frame_idx < len(pose2d.keypoints)):
|
| 41 |
+
return out
|
| 42 |
+
kps = pose2d.keypoints[frame_idx]
|
| 43 |
+
for name, (a, b, c) in relevant_angles(test_name).items():
|
| 44 |
+
pa, pb, pc = _get_joint(kps, a), _get_joint(kps, b), _get_joint(kps, c)
|
| 45 |
+
if pa and pb and pc:
|
| 46 |
+
deg = _angle_between_points(pa, pb, pc)
|
| 47 |
+
if not math.isnan(deg):
|
| 48 |
+
out[name] = {"deg": deg, "openness": openness_label(deg)}
|
| 49 |
+
return out
|
formscout/config.py
ADDED
|
@@ -0,0 +1,181 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
FormScout pipeline configuration.
|
| 3 |
+
All model IDs, thresholds, k-values, and feature flags live here.
|
| 4 |
+
No scattered literals elsewhere in the codebase.
|
| 5 |
+
"""
|
| 6 |
+
import os
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
|
| 9 |
+
ROOT = Path(__file__).parent.parent
|
| 10 |
+
|
| 11 |
+
# βββ Model IDs βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 12 |
+
_YOLO_DIR = ROOT / "checkpoints" / "yolo26"
|
| 13 |
+
|
| 14 |
+
POSE_MODELS: dict[str, dict] = {
|
| 15 |
+
# ββ MediaPipe (official Tasks API, local checkpoint) βββββββββββββββββββ
|
| 16 |
+
"MediaPipe-Pose β full (~9 MB, CPU-friendly)": {
|
| 17 |
+
"backend": "mediapipe",
|
| 18 |
+
"path": str(ROOT / "checkpoints" / "mediapipe" / "pose_landmarker_full.task"),
|
| 19 |
+
"params_m": 4.2,
|
| 20 |
+
},
|
| 21 |
+
# ββ YOLO26 (local checkpoints) βββββββββββββββββββββββββββββββββββββββββ
|
| 22 |
+
"YOLO26n β nano (0.7M, fastest)": {
|
| 23 |
+
"backend": "yolo",
|
| 24 |
+
"path": str(_YOLO_DIR / "yolo26n-pose.pt"),
|
| 25 |
+
"params_m": 0.7,
|
| 26 |
+
},
|
| 27 |
+
"YOLO26s β small (3.5M)": {
|
| 28 |
+
"backend": "yolo",
|
| 29 |
+
"path": str(_YOLO_DIR / "yolo26s-pose.pt"),
|
| 30 |
+
"params_m": 3.5,
|
| 31 |
+
},
|
| 32 |
+
"YOLO26m β medium (9M)": {
|
| 33 |
+
"backend": "yolo",
|
| 34 |
+
"path": str(_YOLO_DIR / "yolo26m-pose.pt"),
|
| 35 |
+
"params_m": 9.0,
|
| 36 |
+
},
|
| 37 |
+
"YOLO26l β large (25.9M)": {
|
| 38 |
+
"backend": "yolo",
|
| 39 |
+
"path": str(_YOLO_DIR / "yolo26l-pose.pt"),
|
| 40 |
+
"params_m": 25.9,
|
| 41 |
+
},
|
| 42 |
+
"YOLO26x β extra-large (57.6M)": {
|
| 43 |
+
"backend": "yolo",
|
| 44 |
+
"path": str(_YOLO_DIR / "yolo26x-pose.pt"),
|
| 45 |
+
"params_m": 57.6,
|
| 46 |
+
},
|
| 47 |
+
# ββ Sapiens2 (Phase 3 β needs custom repo + detector, 308-kp Sociopticon) β
|
| 48 |
+
"Sapiens2-0.4B [Phase 3, ~1.6 GB]": {
|
| 49 |
+
"backend": "sapiens2",
|
| 50 |
+
"hf_id": "facebook/sapiens2-pose-0.4b",
|
| 51 |
+
"params_m": 400,
|
| 52 |
+
},
|
| 53 |
+
"Sapiens2-0.8B [Phase 3, ~3.2 GB]": {
|
| 54 |
+
"backend": "sapiens2",
|
| 55 |
+
"hf_id": "facebook/sapiens2-pose-0.8b",
|
| 56 |
+
"params_m": 800,
|
| 57 |
+
},
|
| 58 |
+
"Sapiens2-1B [Phase 3, ~6 GB]": {
|
| 59 |
+
"backend": "sapiens2",
|
| 60 |
+
"hf_id": "facebook/sapiens2-pose-1b",
|
| 61 |
+
"params_m": 1000,
|
| 62 |
+
},
|
| 63 |
+
"Sapiens2-5B [Phase 3, ~20 GB, large GPU]": {
|
| 64 |
+
"backend": "sapiens2",
|
| 65 |
+
"hf_id": "facebook/sapiens2-pose-5b",
|
| 66 |
+
"params_m": 5000,
|
| 67 |
+
},
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
DEFAULT_POSE_MODEL = "YOLO26n β nano (0.7M, fastest)"
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def _is_model_available(spec: dict) -> bool:
|
| 74 |
+
"""Return True if the model checkpoint is present and the backend is importable."""
|
| 75 |
+
backend = spec["backend"]
|
| 76 |
+
if backend in ("yolo", "mediapipe"):
|
| 77 |
+
return Path(spec["path"]).exists()
|
| 78 |
+
if backend == "sapiens2":
|
| 79 |
+
try:
|
| 80 |
+
import sapiens # noqa: F401 β custom repo must be installed
|
| 81 |
+
return True
|
| 82 |
+
except ImportError:
|
| 83 |
+
return False
|
| 84 |
+
return False
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def available_pose_models() -> dict[str, dict]:
|
| 88 |
+
"""Subset of POSE_MODELS whose checkpoints/backends are actually ready."""
|
| 89 |
+
return {name: spec for name, spec in POSE_MODELS.items() if _is_model_available(spec)}
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
# Backward-compat aliases
|
| 93 |
+
YOLO_POSE_MODEL = str(_YOLO_DIR / "yolo26l-pose.pt")
|
| 94 |
+
YOLO_POSE_MODEL_HQ = str(_YOLO_DIR / "yolo26x-pose.pt")
|
| 95 |
+
SAM_CHECKPOINT = "sam2.1_hiera_base_plus.pt"
|
| 96 |
+
SAM_3D_CHECKPOINT = ROOT / "checkpoints" / "sam-3d-body-dinov3" / "model.ckpt"
|
| 97 |
+
SAM_3D_HF_REPO = "facebook/sam-3d-body-dinov3"
|
| 98 |
+
SAM_3D_MHR_PATH = ROOT / "checkpoints" / "sam-3d-body-dinov3" / "assets" / "mhr_model.pt"
|
| 99 |
+
# βββ Judge / Classifier VLM (Qwen3-VL-8B-Instruct via llama.cpp) ββββββββββββ
|
| 100 |
+
# Default: stock Qwen3-VL-8B-Instruct Q4_K_M. To swap in a fine-tuned GGUF,
|
| 101 |
+
# set FORMSCOUT_JUDGE_GGUF (and FORMSCOUT_JUDGE_MMPROJ if it has its own
|
| 102 |
+
# projector) β no code change needed.
|
| 103 |
+
_QWEN_DIR = ROOT / "checkpoints" / "qwen3-vl"
|
| 104 |
+
JUDGE_GGUF = Path(os.environ.get(
|
| 105 |
+
"FORMSCOUT_JUDGE_GGUF", _QWEN_DIR / "Qwen3VL-8B-Instruct-Q4_K_M.gguf"
|
| 106 |
+
))
|
| 107 |
+
JUDGE_MMPROJ = Path(os.environ.get(
|
| 108 |
+
"FORMSCOUT_JUDGE_MMPROJ", _QWEN_DIR / "mmproj-Qwen3VL-8B-Instruct-F16.gguf"
|
| 109 |
+
))
|
| 110 |
+
JUDGE_HF_REPO = "Qwen/Qwen3-VL-8B-Instruct-GGUF"
|
| 111 |
+
|
| 112 |
+
QWEN_VLM_GGUF = str(JUDGE_GGUF) # backward-compat alias
|
| 113 |
+
QWEN_EMBED_GGUF = "Qwen3-VL-Embedding-8B-Q4_K_M.gguf"
|
| 114 |
+
STGCN_CHECKPOINT = ROOT / "checkpoints" / "stgcn_fms.pth"
|
| 115 |
+
|
| 116 |
+
# βββ Pipeline flags ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 117 |
+
ENABLE_3D = False # SAM 3D Body β access granted Jun 2026, off until integrated
|
| 118 |
+
ENABLE_STGCN = False # Phase 3
|
| 119 |
+
ENABLE_RAG = False # Phase 3
|
| 120 |
+
ENABLE_JUDGE = True # VLM judge/classifier β falls back to rubric when llama-server is down
|
| 121 |
+
|
| 122 |
+
# βββ Thresholds ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 123 |
+
MIN_CONFIDENCE = 0.6
|
| 124 |
+
SCORE_DISAGREE_THRESH = 1 # flag if |stgcn - judge| >= this
|
| 125 |
+
RETRIEVAL_K = 3
|
| 126 |
+
|
| 127 |
+
# βββ Video / Ingest βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 128 |
+
TARGET_FPS = 30.0
|
| 129 |
+
MAX_FRAMES = 300 # hard cap to avoid OOM
|
| 130 |
+
MAX_DURATION_SEC = 60.0 # warn on longer videos
|
| 131 |
+
|
| 132 |
+
# βββ Pose ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 133 |
+
POSE_BACKEND = "yolo" # "yolo" | "sapiens"
|
| 134 |
+
POSE_CONF_THRESHOLD = 0.5
|
| 135 |
+
NUM_KEYPOINTS = 17
|
| 136 |
+
|
| 137 |
+
# βββ Biomechanics thresholds ββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 138 |
+
DEEP_SQUAT_FEMUR_HORIZONTAL_DEG = 90.0
|
| 139 |
+
DEEP_SQUAT_TORSO_TIBIA_MAX_DEG = 15.0
|
| 140 |
+
DEEP_SQUAT_KNEE_TRACKING_MARGIN_PX = 20
|
| 141 |
+
|
| 142 |
+
# βββ Serving (llama.cpp) ββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 143 |
+
LLAMA_CPP_HOST = "127.0.0.1"
|
| 144 |
+
LLAMA_CPP_PORT_VLM = 8080
|
| 145 |
+
LLAMA_CPP_PORT_EMBED = 8081
|
| 146 |
+
|
| 147 |
+
# βββ Judge backend selection ββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 148 |
+
# "llama_cpp" β local llama-server (default for local dev; works perfectly)
|
| 149 |
+
# "transformers"β in-process Qwen3-VL via transformers, GPU on HF Spaces (ZeroGPU)
|
| 150 |
+
# "auto" β transformers ONLY on a GPU/ZeroGPU Space, else llama_cpp
|
| 151 |
+
JUDGE_BACKEND = os.environ.get("FORMSCOUT_JUDGE_BACKEND", "auto")
|
| 152 |
+
JUDGE_HF_MODEL = os.environ.get("FORMSCOUT_JUDGE_HF_MODEL", "Qwen/Qwen3-VL-8B-Instruct")
|
| 153 |
+
ON_HF_SPACE = bool(os.environ.get("SPACE_ID"))
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def has_gpu() -> bool:
|
| 157 |
+
"""True on a ZeroGPU Space (env flag) or when CUDA is actually present.
|
| 158 |
+
|
| 159 |
+
ZeroGPU exposes no CUDA outside @spaces.GPU, so it is detected via the
|
| 160 |
+
SPACES_ZERO_GPU env flag; ordinary GPU Spaces report via torch.cuda.
|
| 161 |
+
"""
|
| 162 |
+
if os.environ.get("SPACES_ZERO_GPU") or os.environ.get("ZERO_GPU"):
|
| 163 |
+
return True
|
| 164 |
+
try:
|
| 165 |
+
import torch
|
| 166 |
+
return bool(torch.cuda.is_available())
|
| 167 |
+
except Exception:
|
| 168 |
+
return False
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
def resolve_judge_backend() -> str:
|
| 172 |
+
"""Resolve the effective judge backend from JUDGE_BACKEND + environment.
|
| 173 |
+
|
| 174 |
+
`auto` only engages the heavy in-process transformers model when a GPU is
|
| 175 |
+
actually available β a CPU-only Space stays on llama_cpp (which is then
|
| 176 |
+
unreachable, so the Judge falls back to the fast rubric instead of trying to
|
| 177 |
+
run a 17 GB model on CPU).
|
| 178 |
+
"""
|
| 179 |
+
if JUDGE_BACKEND in ("llama_cpp", "transformers"):
|
| 180 |
+
return JUDGE_BACKEND
|
| 181 |
+
return "transformers" if (ON_HF_SPACE and has_gpu()) else "llama_cpp"
|
formscout/pipeline.py
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Director β deterministic state machine orchestrating the FormScout pipeline.
|
| 3 |
+
|
| 4 |
+
NOT an LLM. Runs each agent in sequence, applies quality gates, and assembles
|
| 5 |
+
the final PipelineState. Exposes run(video_path, config) -> PipelineState.
|
| 6 |
+
"""
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
+
from formscout import config
|
| 12 |
+
from formscout.types import (
|
| 13 |
+
PipelineState, Body3DResult, MovementResult,
|
| 14 |
+
)
|
| 15 |
+
from formscout.agents.ingest import IngestAgent
|
| 16 |
+
from formscout.agents.pose2d import Pose2DAgent
|
| 17 |
+
from formscout.agents.body3d import Body3DAgent
|
| 18 |
+
from formscout.agents.biomechanics import BiomechanicsAgent
|
| 19 |
+
from formscout.agents.classifier import MovementClassifierAgent
|
| 20 |
+
from formscout.agents.judge import JudgeAgent
|
| 21 |
+
from formscout.agents.report import ReportAgent
|
| 22 |
+
from formscout.rubric import score_test
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class Director:
|
| 26 |
+
"""
|
| 27 |
+
Orchestrates the FormScout agent pipeline as a deterministic state machine.
|
| 28 |
+
Quality gates are applied after each agent β never silently passes bad data.
|
| 29 |
+
"""
|
| 30 |
+
|
| 31 |
+
def __init__(self):
|
| 32 |
+
self._ingest = IngestAgent()
|
| 33 |
+
self._pose2d = Pose2DAgent()
|
| 34 |
+
self._body3d = Body3DAgent()
|
| 35 |
+
self._biomechanics = BiomechanicsAgent()
|
| 36 |
+
self._classifier = MovementClassifierAgent()
|
| 37 |
+
self._judge = JudgeAgent()
|
| 38 |
+
self._report = ReportAgent()
|
| 39 |
+
|
| 40 |
+
def run(self, video_path: str, test_name: str = "deep_squat", side: str = "na", model_key: str | None = None) -> PipelineState:
|
| 41 |
+
"""
|
| 42 |
+
Run the full pipeline on a single video.
|
| 43 |
+
test_name/side serve as manual override when provided (skips classifier).
|
| 44 |
+
model_key selects the pose backend (see config.POSE_MODELS).
|
| 45 |
+
"""
|
| 46 |
+
state = PipelineState(video_path=video_path)
|
| 47 |
+
|
| 48 |
+
# βββ Ingest βββ
|
| 49 |
+
state.ingest = self._ingest.run(video_path)
|
| 50 |
+
if state.ingest.confidence < config.MIN_CONFIDENCE:
|
| 51 |
+
state.errors.append("ingest: low confidence β video may be corrupt")
|
| 52 |
+
return state
|
| 53 |
+
|
| 54 |
+
# βββ Pose 2D βββ
|
| 55 |
+
state.pose2d = self._pose2d.run(state.ingest, model_key=model_key)
|
| 56 |
+
if state.pose2d.confidence < config.MIN_CONFIDENCE:
|
| 57 |
+
state.warnings.append("pose2d: low confidence β no clear person detected")
|
| 58 |
+
|
| 59 |
+
# βββ Body 3D (optional) βββ
|
| 60 |
+
masks = state.segment.masks if state.segment else []
|
| 61 |
+
frames = state.ingest.frames if state.ingest else []
|
| 62 |
+
state.body3d = self._body3d.run(state.pose2d, masks, frames=frames)
|
| 63 |
+
|
| 64 |
+
# βββ Movement classification βββ
|
| 65 |
+
if test_name and test_name != "unknown":
|
| 66 |
+
# Manual override
|
| 67 |
+
state.movement = MovementResult(
|
| 68 |
+
test_name=test_name, side=side,
|
| 69 |
+
confidence=1.0, notes="manually specified",
|
| 70 |
+
)
|
| 71 |
+
else:
|
| 72 |
+
state.movement = self._classifier.run(state.ingest, state.pose2d)
|
| 73 |
+
|
| 74 |
+
# Gate: unknown test β stop
|
| 75 |
+
if state.movement.test_name == "unknown":
|
| 76 |
+
state.errors.append("movement classifier returned 'unknown' β manual override required")
|
| 77 |
+
return state
|
| 78 |
+
|
| 79 |
+
# βββ Biomechanics βββ
|
| 80 |
+
state.features = self._biomechanics.run(
|
| 81 |
+
state.pose2d,
|
| 82 |
+
state.body3d or Body3DResult(used=False, joints_3d=[]),
|
| 83 |
+
state.movement,
|
| 84 |
+
)
|
| 85 |
+
if state.features.confidence < config.MIN_CONFIDENCE:
|
| 86 |
+
state.warnings.append(
|
| 87 |
+
f"biomechanics: low confidence ({state.features.confidence:.2f}) β physio review recommended"
|
| 88 |
+
)
|
| 89 |
+
|
| 90 |
+
# βββ Rubric Score βββ
|
| 91 |
+
rubric_result = score_test(state.features)
|
| 92 |
+
state.stgcn_score = rubric_result # Reusing field for rubric until ST-GCN is built
|
| 93 |
+
|
| 94 |
+
# βββ Judge βββ
|
| 95 |
+
state.judge = self._judge.run(
|
| 96 |
+
state.features, rubric_result, state.movement, state.ingest,
|
| 97 |
+
)
|
| 98 |
+
|
| 99 |
+
# βββ Quality gates βββ
|
| 100 |
+
# Gate: score disagreement
|
| 101 |
+
if (state.judge.score is not None and rubric_result.score is not None
|
| 102 |
+
and abs(state.judge.score - rubric_result.score) >= config.SCORE_DISAGREE_THRESH):
|
| 103 |
+
state.warnings.append(
|
| 104 |
+
f"score disagreement: rubric={rubric_result.score} vs judge={state.judge.score} β review recommended"
|
| 105 |
+
)
|
| 106 |
+
|
| 107 |
+
# Gate: needs_human
|
| 108 |
+
if state.judge.needs_human:
|
| 109 |
+
state.warnings.append("judge flagged needs_human β no auto-score emitted")
|
| 110 |
+
|
| 111 |
+
return state
|
formscout/rubric/__init__.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
FormScout rubric scorers β one pure-function scorer per FMS test.
|
| 3 |
+
"""
|
| 4 |
+
from formscout.rubric.deep_squat import score_deep_squat
|
| 5 |
+
from formscout.rubric.hurdle_step import score_hurdle_step
|
| 6 |
+
from formscout.rubric.inline_lunge import score_inline_lunge
|
| 7 |
+
from formscout.rubric.shoulder_mobility import score_shoulder_mobility
|
| 8 |
+
from formscout.rubric.active_slr import score_active_slr
|
| 9 |
+
from formscout.rubric.trunk_stability_pushup import score_trunk_stability_pushup
|
| 10 |
+
from formscout.rubric.rotary_stability import score_rotary_stability
|
| 11 |
+
from formscout.types import BiomechFeatures, ScoreResult
|
| 12 |
+
|
| 13 |
+
SCORERS = {
|
| 14 |
+
"deep_squat": score_deep_squat,
|
| 15 |
+
"hurdle_step": score_hurdle_step,
|
| 16 |
+
"inline_lunge": score_inline_lunge,
|
| 17 |
+
"shoulder_mobility": score_shoulder_mobility,
|
| 18 |
+
"active_slr": score_active_slr,
|
| 19 |
+
"trunk_stability_pushup": score_trunk_stability_pushup,
|
| 20 |
+
"rotary_stability": score_rotary_stability,
|
| 21 |
+
}
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def score_test(features: BiomechFeatures) -> ScoreResult:
|
| 25 |
+
"""Dispatch to the appropriate rubric scorer by test name."""
|
| 26 |
+
fn = SCORERS.get(features.test_name)
|
| 27 |
+
if fn is None:
|
| 28 |
+
return ScoreResult(
|
| 29 |
+
score=1, rationale=f"No rubric for test '{features.test_name}'",
|
| 30 |
+
confidence=0.0, notes="unknown test",
|
| 31 |
+
)
|
| 32 |
+
return fn(features)
|
formscout/rubric/active_slr.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Active Straight-Leg Raise rubric scorer β pure function, no model calls.
|
| 3 |
+
|
| 4 |
+
FMS ASLR Criteria (bilateral):
|
| 5 |
+
- Score 3: raised leg malleolus past contralateral knee (>70Β°), down leg flat.
|
| 6 |
+
- Score 2: malleolus between mid-thigh and knee (45-70Β°).
|
| 7 |
+
- Score 1: malleolus below mid-thigh (<45Β°).
|
| 8 |
+
- Score 0: PAIN β never auto-scored.
|
| 9 |
+
"""
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
from formscout.types import BiomechFeatures, ScoreResult
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def score_active_slr(features: BiomechFeatures) -> ScoreResult:
|
| 16 |
+
"""Pure rubric scorer for active straight-leg raise."""
|
| 17 |
+
angles = features.angles
|
| 18 |
+
alignments = features.alignments
|
| 19 |
+
|
| 20 |
+
has_angle = "raised_leg_angle_deg" in angles
|
| 21 |
+
if not has_angle:
|
| 22 |
+
return ScoreResult(
|
| 23 |
+
score=1, rationale="Insufficient data: leg raise angle not measurable",
|
| 24 |
+
confidence=0.3, notes="missing key measurements",
|
| 25 |
+
)
|
| 26 |
+
|
| 27 |
+
angle = angles["raised_leg_angle_deg"]
|
| 28 |
+
past_knee = alignments.get("past_contralateral_knee", False)
|
| 29 |
+
past_mid = alignments.get("past_mid_thigh", False)
|
| 30 |
+
down_flat = alignments.get("down_leg_flat", True)
|
| 31 |
+
|
| 32 |
+
rationale_parts = []
|
| 33 |
+
|
| 34 |
+
if past_knee and down_flat:
|
| 35 |
+
score = 3
|
| 36 |
+
rationale_parts.append(f"Raised leg at {angle:.0f}Β° (past contralateral knee)")
|
| 37 |
+
elif past_mid:
|
| 38 |
+
score = 2
|
| 39 |
+
rationale_parts.append(f"Raised leg at {angle:.0f}Β° (between mid-thigh and knee)")
|
| 40 |
+
if not down_flat:
|
| 41 |
+
rationale_parts.append("down leg lifted off surface")
|
| 42 |
+
else:
|
| 43 |
+
score = 1
|
| 44 |
+
rationale_parts.append(f"Raised leg only {angle:.0f}Β° (below mid-thigh)")
|
| 45 |
+
|
| 46 |
+
confidence = features.confidence * 0.9
|
| 47 |
+
|
| 48 |
+
return ScoreResult(
|
| 49 |
+
score=score, rationale="; ".join(rationale_parts),
|
| 50 |
+
confidence=confidence, notes="",
|
| 51 |
+
)
|
formscout/rubric/deep_squat.py
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Deep Squat rubric scorer β pure function, no model calls.
|
| 3 |
+
|
| 4 |
+
FMS Deep Squat Criteria:
|
| 5 |
+
- Score 3: femur below horizontal, torso parallel to tibia, knees tracking
|
| 6 |
+
over feet, dowel over feet, heels flat.
|
| 7 |
+
- Score 2: criteria met only with heels elevated.
|
| 8 |
+
- Score 1: criteria unmet even with heels elevated.
|
| 9 |
+
- Score 0: PAIN β never auto-scored by this function.
|
| 10 |
+
|
| 11 |
+
Input: BiomechFeatures for deep_squat
|
| 12 |
+
Output: ScoreResult(score, rationale, confidence, needs_human)
|
| 13 |
+
"""
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import math
|
| 17 |
+
|
| 18 |
+
from formscout.types import BiomechFeatures, ScoreResult
|
| 19 |
+
from formscout import config
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def score_deep_squat(features: BiomechFeatures) -> ScoreResult:
|
| 23 |
+
"""
|
| 24 |
+
Pure rubric scorer for deep squat.
|
| 25 |
+
Returns ScoreResult with score 1-3 based on biomechanical measurements.
|
| 26 |
+
Never assigns score 0 (pain) β that requires needs_human=True from JudgeAgent.
|
| 27 |
+
"""
|
| 28 |
+
angles = features.angles
|
| 29 |
+
alignments = features.alignments
|
| 30 |
+
|
| 31 |
+
# Check if we have enough data to score
|
| 32 |
+
has_femur = any(
|
| 33 |
+
k in angles for k in ("left_femur_from_horizontal_deg", "right_femur_from_horizontal_deg")
|
| 34 |
+
)
|
| 35 |
+
has_torso_tibia = "torso_tibia_angle_deg" in angles
|
| 36 |
+
|
| 37 |
+
if not has_femur:
|
| 38 |
+
return ScoreResult(
|
| 39 |
+
score=1,
|
| 40 |
+
rationale="Insufficient data: femur angle not measurable",
|
| 41 |
+
confidence=0.3,
|
| 42 |
+
needs_human=False,
|
| 43 |
+
notes="missing femur measurements β defaulting to lowest passing score",
|
| 44 |
+
)
|
| 45 |
+
|
| 46 |
+
# Evaluate criteria
|
| 47 |
+
# Femur below horizontal: femur angle from horizontal > 90Β° means above horizontal
|
| 48 |
+
# In our measurement: angle is from horizontal, so < 90 means below horizontal
|
| 49 |
+
femur_angles = []
|
| 50 |
+
if "left_femur_from_horizontal_deg" in angles:
|
| 51 |
+
femur_angles.append(angles["left_femur_from_horizontal_deg"])
|
| 52 |
+
if "right_femur_from_horizontal_deg" in angles:
|
| 53 |
+
femur_angles.append(angles["right_femur_from_horizontal_deg"])
|
| 54 |
+
|
| 55 |
+
# Femur below horizontal means the thigh slopes down steeply (angle > ~60Β° from horizontal in image coords)
|
| 56 |
+
femur_below_horizontal = any(a > 60.0 for a in femur_angles) if femur_angles else False
|
| 57 |
+
|
| 58 |
+
# Torso parallel to tibia
|
| 59 |
+
torso_parallel_tibia = (
|
| 60 |
+
angles.get("torso_tibia_angle_deg", 999) <= config.DEEP_SQUAT_TORSO_TIBIA_MAX_DEG
|
| 61 |
+
)
|
| 62 |
+
|
| 63 |
+
# Knee tracking
|
| 64 |
+
knees_tracking = alignments.get("knees_tracking_over_feet", False)
|
| 65 |
+
|
| 66 |
+
# Dowel alignment
|
| 67 |
+
dowel_over_feet = alignments.get("dowel_over_feet", False)
|
| 68 |
+
|
| 69 |
+
# Heels
|
| 70 |
+
heels_elevated = alignments.get("heels_elevated", False)
|
| 71 |
+
|
| 72 |
+
# Scoring logic
|
| 73 |
+
all_criteria = femur_below_horizontal and torso_parallel_tibia and knees_tracking and dowel_over_feet
|
| 74 |
+
|
| 75 |
+
rationale_parts: list[str] = []
|
| 76 |
+
|
| 77 |
+
if all_criteria and not heels_elevated:
|
| 78 |
+
score = 3
|
| 79 |
+
rationale_parts.append("All criteria met with heels flat")
|
| 80 |
+
elif all_criteria and heels_elevated:
|
| 81 |
+
score = 2
|
| 82 |
+
rationale_parts.append("Criteria met only with heels elevated")
|
| 83 |
+
else:
|
| 84 |
+
# Check what failed
|
| 85 |
+
if not femur_below_horizontal:
|
| 86 |
+
rationale_parts.append("femur not below horizontal")
|
| 87 |
+
if not torso_parallel_tibia:
|
| 88 |
+
rationale_parts.append(
|
| 89 |
+
f"torso-tibia angle {angles.get('torso_tibia_angle_deg', '?')}Β° "
|
| 90 |
+
f"exceeds {config.DEEP_SQUAT_TORSO_TIBIA_MAX_DEG}Β° threshold"
|
| 91 |
+
)
|
| 92 |
+
if not knees_tracking:
|
| 93 |
+
rationale_parts.append("knees not tracking over feet")
|
| 94 |
+
if not dowel_over_feet:
|
| 95 |
+
rationale_parts.append("dowel not aligned over feet")
|
| 96 |
+
|
| 97 |
+
if heels_elevated:
|
| 98 |
+
score = 1
|
| 99 |
+
rationale_parts.append("criteria unmet even with heels elevated")
|
| 100 |
+
else:
|
| 101 |
+
# They might score 2 with heel elevation β but without it, still 1
|
| 102 |
+
score = 1
|
| 103 |
+
rationale_parts.append("criteria unmet with heels flat")
|
| 104 |
+
|
| 105 |
+
confidence = features.confidence * (0.9 if has_torso_tibia else 0.6)
|
| 106 |
+
|
| 107 |
+
return ScoreResult(
|
| 108 |
+
score=score,
|
| 109 |
+
rationale="; ".join(rationale_parts),
|
| 110 |
+
confidence=confidence,
|
| 111 |
+
needs_human=False,
|
| 112 |
+
notes="",
|
| 113 |
+
)
|
formscout/rubric/hurdle_step.py
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Hurdle Step rubric scorer β pure function, no model calls.
|
| 3 |
+
|
| 4 |
+
FMS Hurdle Step Criteria (bilateral β score each side, report lower):
|
| 5 |
+
- Score 3: hips/knees/ankles aligned, minimal trunk movement, dowel/posture stable,
|
| 6 |
+
no contact with hurdle.
|
| 7 |
+
- Score 2: movement completed with compensation (trunk lean, loss of alignment).
|
| 8 |
+
- Score 1: contact with hurdle, loss of balance, or inability to maintain alignment.
|
| 9 |
+
- Score 0: PAIN β never auto-scored.
|
| 10 |
+
"""
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
from formscout.types import BiomechFeatures, ScoreResult
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def score_hurdle_step(features: BiomechFeatures) -> ScoreResult:
|
| 17 |
+
"""Pure rubric scorer for hurdle step."""
|
| 18 |
+
angles = features.angles
|
| 19 |
+
alignments = features.alignments
|
| 20 |
+
|
| 21 |
+
has_hip_flex = "step_hip_flexion_deg" in angles
|
| 22 |
+
if not has_hip_flex:
|
| 23 |
+
return ScoreResult(
|
| 24 |
+
score=1, rationale="Insufficient data: hip flexion not measurable",
|
| 25 |
+
confidence=0.3, notes="missing key measurements",
|
| 26 |
+
)
|
| 27 |
+
|
| 28 |
+
trunk_stable = alignments.get("trunk_stable", False)
|
| 29 |
+
stance_extended = alignments.get("stance_knee_extended", False)
|
| 30 |
+
hip_flex = angles.get("step_hip_flexion_deg", 0)
|
| 31 |
+
|
| 32 |
+
rationale_parts = []
|
| 33 |
+
|
| 34 |
+
# Score 3: good hip flexion, trunk stable, stance solid
|
| 35 |
+
if hip_flex > 90 and trunk_stable and stance_extended:
|
| 36 |
+
score = 3
|
| 37 |
+
rationale_parts.append("Hip flexion adequate, trunk stable, stance knee extended")
|
| 38 |
+
elif hip_flex > 70 or (trunk_stable and stance_extended):
|
| 39 |
+
score = 2
|
| 40 |
+
if not trunk_stable:
|
| 41 |
+
rationale_parts.append("trunk lean detected")
|
| 42 |
+
if not stance_extended:
|
| 43 |
+
rationale_parts.append("stance knee flexion")
|
| 44 |
+
if hip_flex <= 90:
|
| 45 |
+
rationale_parts.append(f"hip flexion {hip_flex:.0f}Β° (borderline)")
|
| 46 |
+
rationale_parts.insert(0, "Movement completed with compensation")
|
| 47 |
+
else:
|
| 48 |
+
score = 1
|
| 49 |
+
rationale_parts.append("Unable to maintain alignment")
|
| 50 |
+
if not trunk_stable:
|
| 51 |
+
rationale_parts.append("significant trunk lean")
|
| 52 |
+
if not stance_extended:
|
| 53 |
+
rationale_parts.append("stance knee collapse")
|
| 54 |
+
|
| 55 |
+
confidence = features.confidence * 0.85
|
| 56 |
+
|
| 57 |
+
return ScoreResult(
|
| 58 |
+
score=score, rationale="; ".join(rationale_parts),
|
| 59 |
+
confidence=confidence, notes="",
|
| 60 |
+
)
|
formscout/rubric/inline_lunge.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
In-Line Lunge rubric scorer β pure function, no model calls.
|
| 3 |
+
|
| 4 |
+
FMS In-Line Lunge Criteria (bilateral):
|
| 5 |
+
- Score 3: dowel contacts maintained, no torso movement, knee touches behind heel.
|
| 6 |
+
- Score 2: movement completed with compensation (trunk lean, loss of balance).
|
| 7 |
+
- Score 1: loss of balance, inability to maintain foot contact or posture.
|
| 8 |
+
- Score 0: PAIN β never auto-scored.
|
| 9 |
+
"""
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
from formscout.types import BiomechFeatures, ScoreResult
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def score_inline_lunge(features: BiomechFeatures) -> ScoreResult:
|
| 16 |
+
"""Pure rubric scorer for in-line lunge."""
|
| 17 |
+
angles = features.angles
|
| 18 |
+
alignments = features.alignments
|
| 19 |
+
|
| 20 |
+
has_knee = "front_knee_flexion_deg" in angles
|
| 21 |
+
if not has_knee:
|
| 22 |
+
return ScoreResult(
|
| 23 |
+
score=1, rationale="Insufficient data: knee flexion not measurable",
|
| 24 |
+
confidence=0.3, notes="missing key measurements",
|
| 25 |
+
)
|
| 26 |
+
|
| 27 |
+
knee_flex = angles.get("front_knee_flexion_deg", 180)
|
| 28 |
+
trunk_upright = alignments.get("trunk_upright", False)
|
| 29 |
+
knee_over_ankle = alignments.get("knee_over_ankle", False)
|
| 30 |
+
|
| 31 |
+
rationale_parts = []
|
| 32 |
+
|
| 33 |
+
# Good lunge: knee flexion < 90Β° (deep), trunk upright, knee aligned
|
| 34 |
+
deep_enough = knee_flex < 100
|
| 35 |
+
if deep_enough and trunk_upright and knee_over_ankle:
|
| 36 |
+
score = 3
|
| 37 |
+
rationale_parts.append("Deep lunge with trunk upright and knee aligned")
|
| 38 |
+
elif deep_enough or (trunk_upright and knee_over_ankle):
|
| 39 |
+
score = 2
|
| 40 |
+
if not trunk_upright:
|
| 41 |
+
rationale_parts.append(f"trunk lean {angles.get('trunk_lean_from_vertical_deg', '?')}Β°")
|
| 42 |
+
if not knee_over_ankle:
|
| 43 |
+
rationale_parts.append("knee drifts past ankle")
|
| 44 |
+
if not deep_enough:
|
| 45 |
+
rationale_parts.append(f"knee flexion {knee_flex:.0f}Β° (insufficient depth)")
|
| 46 |
+
rationale_parts.insert(0, "Completed with compensation")
|
| 47 |
+
else:
|
| 48 |
+
score = 1
|
| 49 |
+
rationale_parts.append("Unable to complete lunge pattern")
|
| 50 |
+
if not deep_enough:
|
| 51 |
+
rationale_parts.append(f"knee flexion only {knee_flex:.0f}Β°")
|
| 52 |
+
|
| 53 |
+
confidence = features.confidence * 0.85
|
| 54 |
+
|
| 55 |
+
return ScoreResult(
|
| 56 |
+
score=score, rationale="; ".join(rationale_parts),
|
| 57 |
+
confidence=confidence, notes="",
|
| 58 |
+
)
|
formscout/rubric/rotary_stability.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Rotary Stability rubric scorer β pure function, no model calls.
|
| 3 |
+
|
| 4 |
+
FMS Rotary Stability Criteria:
|
| 5 |
+
- Score 3: unilateral (same-side) arm/leg extension with trunk stable,
|
| 6 |
+
elbow/knee touch performed smoothly.
|
| 7 |
+
- Score 2: contralateral (opposite) arm/leg extension performed with trunk stable.
|
| 8 |
+
- Score 1: inability to maintain trunk stability during contralateral pattern.
|
| 9 |
+
- Score 0: PAIN (spinal flexion clearing test) β never auto-scored.
|
| 10 |
+
"""
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
from formscout.types import BiomechFeatures, ScoreResult
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def score_rotary_stability(features: BiomechFeatures) -> ScoreResult:
|
| 17 |
+
"""Pure rubric scorer for rotary stability."""
|
| 18 |
+
angles = features.angles
|
| 19 |
+
alignments = features.alignments
|
| 20 |
+
|
| 21 |
+
has_data = "trunk_stability_std_px" in angles or "shoulder_level_diff_px" in angles
|
| 22 |
+
if not has_data:
|
| 23 |
+
return ScoreResult(
|
| 24 |
+
score=1, rationale="Insufficient data: trunk stability not measurable",
|
| 25 |
+
confidence=0.3, notes="missing key measurements",
|
| 26 |
+
)
|
| 27 |
+
|
| 28 |
+
trunk_stable = alignments.get("trunk_stable", False)
|
| 29 |
+
shoulders_level = alignments.get("shoulders_level", False)
|
| 30 |
+
hips_level = alignments.get("hips_level", False)
|
| 31 |
+
|
| 32 |
+
rationale_parts = []
|
| 33 |
+
|
| 34 |
+
# Without video classification of ipsi vs contra, assume contralateral (safer)
|
| 35 |
+
if trunk_stable and shoulders_level and hips_level:
|
| 36 |
+
score = 2 # Assume contralateral unless classifier says ipsilateral
|
| 37 |
+
rationale_parts.append("Trunk stable during extension, shoulders and hips level")
|
| 38 |
+
rationale_parts.append("scored as contralateral pattern (default)")
|
| 39 |
+
elif trunk_stable or (shoulders_level and hips_level):
|
| 40 |
+
score = 2
|
| 41 |
+
if not trunk_stable:
|
| 42 |
+
rationale_parts.append("minor trunk instability")
|
| 43 |
+
rationale_parts.insert(0, "Contralateral pattern with minor compensation")
|
| 44 |
+
else:
|
| 45 |
+
score = 1
|
| 46 |
+
std = angles.get("trunk_stability_std_px", 0)
|
| 47 |
+
rationale_parts.append(f"Trunk instability detected (std {std:.1f}px)")
|
| 48 |
+
if not shoulders_level:
|
| 49 |
+
rationale_parts.append("shoulder asymmetry during extension")
|
| 50 |
+
|
| 51 |
+
confidence = features.confidence * 0.75 # Lower confidence β hard to assess from 2D
|
| 52 |
+
|
| 53 |
+
return ScoreResult(
|
| 54 |
+
score=score, rationale="; ".join(rationale_parts),
|
| 55 |
+
confidence=confidence, notes="ipsi/contra distinction requires VLM classifier",
|
| 56 |
+
)
|
formscout/rubric/shoulder_mobility.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Shoulder Mobility rubric scorer β pure function, no model calls.
|
| 3 |
+
|
| 4 |
+
FMS Shoulder Mobility Criteria (bilateral):
|
| 5 |
+
- Score 3: fists within one hand-length of each other.
|
| 6 |
+
- Score 2: fists within 1.5 hand-lengths.
|
| 7 |
+
- Score 1: fists more than 1.5 hand-lengths apart.
|
| 8 |
+
- Score 0: PAIN (clearing test) β never auto-scored.
|
| 9 |
+
"""
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
from formscout.types import BiomechFeatures, ScoreResult
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def score_shoulder_mobility(features: BiomechFeatures) -> ScoreResult:
|
| 16 |
+
"""Pure rubric scorer for shoulder mobility."""
|
| 17 |
+
alignments = features.alignments
|
| 18 |
+
angles = features.angles
|
| 19 |
+
|
| 20 |
+
has_measure = "inter_fist_normalized" in angles
|
| 21 |
+
if not has_measure:
|
| 22 |
+
return ScoreResult(
|
| 23 |
+
score=1, rationale="Insufficient data: inter-fist distance not measurable",
|
| 24 |
+
confidence=0.3, notes="missing key measurements",
|
| 25 |
+
)
|
| 26 |
+
|
| 27 |
+
norm_dist = angles["inter_fist_normalized"]
|
| 28 |
+
within_one = alignments.get("fists_within_one_hand", False)
|
| 29 |
+
within_1_5 = alignments.get("fists_within_1_5_hand", False)
|
| 30 |
+
|
| 31 |
+
if within_one:
|
| 32 |
+
score = 3
|
| 33 |
+
rationale = f"Fists within one hand-length (normalized distance {norm_dist:.2f})"
|
| 34 |
+
elif within_1_5:
|
| 35 |
+
score = 2
|
| 36 |
+
rationale = f"Fists within 1.5 hand-lengths (normalized distance {norm_dist:.2f})"
|
| 37 |
+
else:
|
| 38 |
+
score = 1
|
| 39 |
+
rationale = f"Fists beyond 1.5 hand-lengths apart (normalized distance {norm_dist:.2f})"
|
| 40 |
+
|
| 41 |
+
confidence = features.confidence * 0.9
|
| 42 |
+
|
| 43 |
+
return ScoreResult(
|
| 44 |
+
score=score, rationale=rationale,
|
| 45 |
+
confidence=confidence, notes="",
|
| 46 |
+
)
|
formscout/rubric/trunk_stability_pushup.py
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Trunk Stability Push-Up rubric scorer β pure function, no model calls.
|
| 3 |
+
|
| 4 |
+
FMS Trunk Stability Push-Up Criteria:
|
| 5 |
+
- Score 3: body moves as one unit (rigid) with hands at forehead level (men)
|
| 6 |
+
or chin level (women). No sag or segment lag.
|
| 7 |
+
- Score 2: body moves as one unit but with hands at chin (men) or clavicle (women).
|
| 8 |
+
- Score 1: unable to perform with hands lowered; body sags or segments.
|
| 9 |
+
- Score 0: PAIN (spinal extension clearing test) β never auto-scored.
|
| 10 |
+
"""
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
from formscout.types import BiomechFeatures, ScoreResult
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def score_trunk_stability_pushup(features: BiomechFeatures) -> ScoreResult:
|
| 17 |
+
"""Pure rubric scorer for trunk stability push-up."""
|
| 18 |
+
angles = features.angles
|
| 19 |
+
alignments = features.alignments
|
| 20 |
+
|
| 21 |
+
has_data = "max_sag_px" in angles
|
| 22 |
+
if not has_data:
|
| 23 |
+
return ScoreResult(
|
| 24 |
+
score=1, rationale="Insufficient data: trunk rigidity not measurable",
|
| 25 |
+
confidence=0.3, notes="missing key measurements",
|
| 26 |
+
)
|
| 27 |
+
|
| 28 |
+
body_rigid = alignments.get("body_rigid", False)
|
| 29 |
+
no_sag = alignments.get("no_sag", False)
|
| 30 |
+
hands_high = alignments.get("hands_at_forehead", False)
|
| 31 |
+
|
| 32 |
+
rationale_parts = []
|
| 33 |
+
|
| 34 |
+
if body_rigid and hands_high:
|
| 35 |
+
score = 3
|
| 36 |
+
rationale_parts.append("Body rigid as one unit, hands at forehead position")
|
| 37 |
+
elif body_rigid or no_sag:
|
| 38 |
+
score = 2
|
| 39 |
+
if not hands_high:
|
| 40 |
+
rationale_parts.append("rigid body but hands in lower position")
|
| 41 |
+
else:
|
| 42 |
+
rationale_parts.append("minor trunk variance detected")
|
| 43 |
+
rationale_parts.insert(0, "Completed with regression")
|
| 44 |
+
else:
|
| 45 |
+
score = 1
|
| 46 |
+
sag = angles.get("max_sag_px", 0)
|
| 47 |
+
variance = angles.get("trunk_variance_px", 0)
|
| 48 |
+
rationale_parts.append(f"Body sag detected ({sag:.0f}px), variance {variance:.1f}px")
|
| 49 |
+
|
| 50 |
+
confidence = features.confidence * 0.8
|
| 51 |
+
|
| 52 |
+
return ScoreResult(
|
| 53 |
+
score=score, rationale="; ".join(rationale_parts),
|
| 54 |
+
confidence=confidence, notes="",
|
| 55 |
+
)
|
formscout/run.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
FormScout headless CLI entrypoint.
|
| 3 |
+
Usage: python -m formscout.run sample.mp4
|
| 4 |
+
"""
|
| 5 |
+
from __future__ import annotations
|
| 6 |
+
|
| 7 |
+
import sys
|
| 8 |
+
import json
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
+
from formscout.pipeline import Director
|
| 12 |
+
from formscout.rubric import score_test
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def main():
|
| 16 |
+
if len(sys.argv) < 2:
|
| 17 |
+
print("Usage: python -m formscout.run <video_path> [test_name] [side]")
|
| 18 |
+
sys.exit(1)
|
| 19 |
+
|
| 20 |
+
video_path = sys.argv[1]
|
| 21 |
+
test_name = sys.argv[2] if len(sys.argv) > 2 else "deep_squat"
|
| 22 |
+
side = sys.argv[3] if len(sys.argv) > 3 else "na"
|
| 23 |
+
|
| 24 |
+
print(f"FormScout β processing: {video_path}")
|
| 25 |
+
print(f" Test: {test_name}, Side: {side}")
|
| 26 |
+
print()
|
| 27 |
+
|
| 28 |
+
director = Director()
|
| 29 |
+
state = director.run(video_path, test_name=test_name, side=side)
|
| 30 |
+
|
| 31 |
+
# Print pipeline state
|
| 32 |
+
if state.errors:
|
| 33 |
+
print("ERRORS:")
|
| 34 |
+
for e in state.errors:
|
| 35 |
+
print(f" β {e}")
|
| 36 |
+
print()
|
| 37 |
+
|
| 38 |
+
if state.warnings:
|
| 39 |
+
print("WARNINGS:")
|
| 40 |
+
for w in state.warnings:
|
| 41 |
+
print(f" β {w}")
|
| 42 |
+
print()
|
| 43 |
+
|
| 44 |
+
if state.ingest:
|
| 45 |
+
print(f"Ingest: {len(state.ingest.frames)} frames, {state.ingest.fps:.1f}fps, "
|
| 46 |
+
f"{state.ingest.duration:.1f}s, {state.ingest.width}x{state.ingest.height}")
|
| 47 |
+
|
| 48 |
+
if state.pose2d:
|
| 49 |
+
n_detected = sum(1 for kps in state.pose2d.keypoints if kps)
|
| 50 |
+
print(f"Pose2D: {n_detected}/{len(state.pose2d.keypoints)} frames with detections, "
|
| 51 |
+
f"confidence={state.pose2d.confidence:.2f}")
|
| 52 |
+
|
| 53 |
+
if state.body3d:
|
| 54 |
+
print(f"Body3D: used={state.body3d.used}")
|
| 55 |
+
|
| 56 |
+
if state.features:
|
| 57 |
+
print(f"Biomechanics: view={state.features.view}, "
|
| 58 |
+
f"confidence={state.features.confidence:.2f}")
|
| 59 |
+
if state.features.angles:
|
| 60 |
+
print(f" Angles: {json.dumps({k: round(v, 1) for k, v in state.features.angles.items()}, indent=4)}")
|
| 61 |
+
if state.features.alignments:
|
| 62 |
+
print(f" Alignments: {json.dumps(state.features.alignments, indent=4)}")
|
| 63 |
+
|
| 64 |
+
# Score via rubric
|
| 65 |
+
if state.features and test_name == "deep_squat":
|
| 66 |
+
score_result = score_test(state.features)
|
| 67 |
+
print(f"\nSCORE: {score_result.score}/3")
|
| 68 |
+
print(f" Rationale: {score_result.rationale}")
|
| 69 |
+
print(f" Confidence: {score_result.confidence:.2f}")
|
| 70 |
+
if score_result.needs_human:
|
| 71 |
+
print(" β NEEDS HUMAN REVIEW")
|
| 72 |
+
|
| 73 |
+
# Judge result
|
| 74 |
+
if state.judge:
|
| 75 |
+
print(f"\nJUDGE: score={state.judge.score}, needs_human={state.judge.needs_human}")
|
| 76 |
+
print(f" Rationale: {state.judge.rationale}")
|
| 77 |
+
if state.judge.compensation_tags:
|
| 78 |
+
print(f" Compensations: {state.judge.compensation_tags}")
|
| 79 |
+
if state.judge.corrective_hint:
|
| 80 |
+
print(f" Corrective: {state.judge.corrective_hint}")
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
if __name__ == "__main__":
|
| 84 |
+
main()
|
formscout/serving/__init__.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Serving backends for the FormScout judge/classifier VLM."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
def get_vlm_client():
|
| 6 |
+
"""Return the VLM client for the resolved judge backend.
|
| 7 |
+
|
| 8 |
+
transformers (in-process, ZeroGPU) on a Space; llama-server locally. Falls
|
| 9 |
+
back to the llama.cpp client if the transformers backend can't be imported.
|
| 10 |
+
"""
|
| 11 |
+
from formscout import config
|
| 12 |
+
|
| 13 |
+
if config.resolve_judge_backend() == "transformers":
|
| 14 |
+
try:
|
| 15 |
+
from formscout.serving.transformers_vlm import TransformersVLMClient
|
| 16 |
+
return TransformersVLMClient()
|
| 17 |
+
except Exception:
|
| 18 |
+
pass
|
| 19 |
+
from formscout.serving.llama_cpp import LlamaCppClient
|
| 20 |
+
return LlamaCppClient(port=config.LLAMA_CPP_PORT_VLM)
|
formscout/serving/llama_cpp.py
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
llama.cpp HTTP client wrapper for FormScout.
|
| 3 |
+
|
| 4 |
+
Wraps the llama.cpp server's /completion and /embedding endpoints.
|
| 5 |
+
Falls back gracefully when the server is unavailable.
|
| 6 |
+
|
| 7 |
+
Model: Qwen3-VL-8B-Instruct (Q4_K_M GGUF) for VLM inference.
|
| 8 |
+
Model: Qwen3-VL-Embedding-8B (Q4_K_M GGUF) for embeddings.
|
| 9 |
+
Params: 8B each (shared backbone).
|
| 10 |
+
License: Apache-2.0.
|
| 11 |
+
"""
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
|
| 14 |
+
import base64
|
| 15 |
+
import json
|
| 16 |
+
import logging
|
| 17 |
+
from pathlib import Path
|
| 18 |
+
from typing import Any
|
| 19 |
+
|
| 20 |
+
import requests
|
| 21 |
+
|
| 22 |
+
from formscout import config
|
| 23 |
+
|
| 24 |
+
logger = logging.getLogger(__name__)
|
| 25 |
+
|
| 26 |
+
_TIMEOUT = 120 # seconds β VLM can be slow
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class LlamaCppClient:
|
| 30 |
+
"""HTTP client for a llama.cpp server instance."""
|
| 31 |
+
|
| 32 |
+
def __init__(self, host: str | None = None, port: int | None = None):
|
| 33 |
+
self.host = host or config.LLAMA_CPP_HOST
|
| 34 |
+
self.port = port or config.LLAMA_CPP_PORT_VLM
|
| 35 |
+
self.base_url = f"http://{self.host}:{self.port}"
|
| 36 |
+
|
| 37 |
+
@property
|
| 38 |
+
def available(self) -> bool:
|
| 39 |
+
"""Check if the server is reachable."""
|
| 40 |
+
try:
|
| 41 |
+
r = requests.get(f"{self.base_url}/health", timeout=5)
|
| 42 |
+
return r.status_code == 200
|
| 43 |
+
except (requests.ConnectionError, requests.Timeout):
|
| 44 |
+
return False
|
| 45 |
+
|
| 46 |
+
def complete(
|
| 47 |
+
self,
|
| 48 |
+
prompt: str,
|
| 49 |
+
images: list[str] | None = None,
|
| 50 |
+
max_tokens: int = 512,
|
| 51 |
+
temperature: float = 0.1,
|
| 52 |
+
stop: list[str] | None = None,
|
| 53 |
+
) -> dict[str, Any]:
|
| 54 |
+
"""
|
| 55 |
+
Send a chat-completion request (OpenAI-compatible /v1/chat/completions β
|
| 56 |
+
required for multimodal: llama-server routes images through the mmproj
|
| 57 |
+
only on this endpoint). Returns parsed JSON if the response is JSON,
|
| 58 |
+
otherwise returns {"text": raw_text}.
|
| 59 |
+
|
| 60 |
+
Args:
|
| 61 |
+
prompt: The text prompt (system + user combined).
|
| 62 |
+
images: Optional list of base64-encoded JPEGs or file paths.
|
| 63 |
+
max_tokens: Max generation tokens.
|
| 64 |
+
temperature: Sampling temperature.
|
| 65 |
+
stop: Stop sequences (default: none β JSON output must not be truncated).
|
| 66 |
+
"""
|
| 67 |
+
content: list[dict[str, Any]] = [{"type": "text", "text": prompt}]
|
| 68 |
+
for img in images or []:
|
| 69 |
+
if len(img) < 4096 and Path(img).exists():
|
| 70 |
+
with open(img, "rb") as f:
|
| 71 |
+
b64 = base64.b64encode(f.read()).decode()
|
| 72 |
+
else:
|
| 73 |
+
b64 = img # already base64
|
| 74 |
+
content.append({
|
| 75 |
+
"type": "image_url",
|
| 76 |
+
"image_url": {"url": f"data:image/jpeg;base64,{b64}"},
|
| 77 |
+
})
|
| 78 |
+
|
| 79 |
+
payload: dict[str, Any] = {
|
| 80 |
+
"messages": [{"role": "user", "content": content}],
|
| 81 |
+
"max_tokens": max_tokens,
|
| 82 |
+
"temperature": temperature,
|
| 83 |
+
}
|
| 84 |
+
if stop:
|
| 85 |
+
payload["stop"] = stop
|
| 86 |
+
|
| 87 |
+
try:
|
| 88 |
+
r = requests.post(
|
| 89 |
+
f"{self.base_url}/v1/chat/completions",
|
| 90 |
+
json=payload,
|
| 91 |
+
timeout=_TIMEOUT,
|
| 92 |
+
)
|
| 93 |
+
r.raise_for_status()
|
| 94 |
+
result = r.json()
|
| 95 |
+
text = result["choices"][0]["message"]["content"] or ""
|
| 96 |
+
return self._parse_json_reply(text)
|
| 97 |
+
except requests.ConnectionError:
|
| 98 |
+
return {"error": "llama.cpp server not available", "text": ""}
|
| 99 |
+
except requests.Timeout:
|
| 100 |
+
return {"error": "llama.cpp server timeout", "text": ""}
|
| 101 |
+
except Exception as e:
|
| 102 |
+
return {"error": str(e), "text": ""}
|
| 103 |
+
|
| 104 |
+
@staticmethod
|
| 105 |
+
def _parse_json_reply(text: str) -> dict[str, Any]:
|
| 106 |
+
"""Parse model output as JSON, tolerating markdown fences."""
|
| 107 |
+
stripped = text.strip()
|
| 108 |
+
if stripped.startswith("```"):
|
| 109 |
+
stripped = stripped.split("\n", 1)[-1]
|
| 110 |
+
stripped = stripped.rsplit("```", 1)[0].strip()
|
| 111 |
+
try:
|
| 112 |
+
parsed = json.loads(stripped)
|
| 113 |
+
if isinstance(parsed, dict):
|
| 114 |
+
return parsed
|
| 115 |
+
except (json.JSONDecodeError, TypeError):
|
| 116 |
+
pass
|
| 117 |
+
return {"text": text}
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
class EmbeddingClient:
|
| 121 |
+
"""HTTP client for the llama.cpp embedding server."""
|
| 122 |
+
|
| 123 |
+
def __init__(self, host: str | None = None, port: int | None = None):
|
| 124 |
+
self.host = host or config.LLAMA_CPP_HOST
|
| 125 |
+
self.port = port or config.LLAMA_CPP_PORT_EMBED
|
| 126 |
+
self.base_url = f"http://{self.host}:{self.port}"
|
| 127 |
+
|
| 128 |
+
@property
|
| 129 |
+
def available(self) -> bool:
|
| 130 |
+
try:
|
| 131 |
+
r = requests.get(f"{self.base_url}/health", timeout=5)
|
| 132 |
+
return r.status_code == 200
|
| 133 |
+
except (requests.ConnectionError, requests.Timeout):
|
| 134 |
+
return False
|
| 135 |
+
|
| 136 |
+
def embed(self, text: str) -> list[float] | None:
|
| 137 |
+
"""Get embedding vector for text. Returns None on failure."""
|
| 138 |
+
try:
|
| 139 |
+
r = requests.post(
|
| 140 |
+
f"{self.base_url}/embedding",
|
| 141 |
+
json={"content": text},
|
| 142 |
+
timeout=30,
|
| 143 |
+
)
|
| 144 |
+
r.raise_for_status()
|
| 145 |
+
data = r.json()
|
| 146 |
+
return data.get("embedding")
|
| 147 |
+
except Exception:
|
| 148 |
+
return None
|
formscout/serving/transformers_vlm.py
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
In-process Qwen3-VL backend via transformers β the HF Spaces (ZeroGPU) path.
|
| 3 |
+
|
| 4 |
+
Mirrors LlamaCppClient's interface (`available` + `complete(...) -> dict`) so it
|
| 5 |
+
is a drop-in alternative selected by config.resolve_judge_backend(). Inference is
|
| 6 |
+
wrapped in spaces.GPU when the `spaces` package is present (ZeroGPU); otherwise it
|
| 7 |
+
runs on whatever device transformers picks. The model is loaded lazily on first
|
| 8 |
+
use and cached for the process.
|
| 9 |
+
|
| 10 |
+
Any load/inference failure returns {"error": ..., "fallback": True} so JudgeAgent
|
| 11 |
+
falls back to the deterministic rubric score instead of flagging needs_human.
|
| 12 |
+
|
| 13 |
+
Model: Qwen3-VL-8B-Instruct (transformers weights, ~16 GB). License: Apache-2.0.
|
| 14 |
+
|
| 15 |
+
NOTE: requires validation on actual ZeroGPU hardware β it cannot be exercised in
|
| 16 |
+
the CPU test environment (would download 16 GB).
|
| 17 |
+
"""
|
| 18 |
+
from __future__ import annotations
|
| 19 |
+
|
| 20 |
+
import base64
|
| 21 |
+
import importlib.util
|
| 22 |
+
import logging
|
| 23 |
+
|
| 24 |
+
from formscout import config
|
| 25 |
+
from formscout.serving.llama_cpp import LlamaCppClient
|
| 26 |
+
|
| 27 |
+
logger = logging.getLogger(__name__)
|
| 28 |
+
|
| 29 |
+
# Module-level model cache (loaded once per process).
|
| 30 |
+
_CACHE: dict = {}
|
| 31 |
+
|
| 32 |
+
# spaces.GPU decorator when on HF infra; a no-op decorator otherwise.
|
| 33 |
+
try: # pragma: no cover - depends on runtime env
|
| 34 |
+
import spaces
|
| 35 |
+
|
| 36 |
+
_gpu = spaces.GPU(duration=120)
|
| 37 |
+
except Exception: # pragma: no cover
|
| 38 |
+
def _gpu(fn):
|
| 39 |
+
return fn
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def _ensure_loaded(model_id: str): # pragma: no cover - downloads ~16 GB
|
| 43 |
+
"""Load processor + model to CPU once (cached). Kept OUT of the GPU window so
|
| 44 |
+
the 17 GB download/load does not eat ZeroGPU time."""
|
| 45 |
+
if "model" not in _CACHE:
|
| 46 |
+
import torch
|
| 47 |
+
from transformers import AutoModelForImageTextToText, AutoProcessor
|
| 48 |
+
_CACHE["processor"] = AutoProcessor.from_pretrained(model_id)
|
| 49 |
+
_CACHE["model"] = AutoModelForImageTextToText.from_pretrained(
|
| 50 |
+
model_id, torch_dtype=torch.bfloat16,
|
| 51 |
+
)
|
| 52 |
+
return _CACHE["processor"], _CACHE["model"]
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
@_gpu
|
| 56 |
+
def _generate(prompt: str, pil_images: list, max_tokens: int,
|
| 57 |
+
temperature: float) -> str: # pragma: no cover - needs GPU + model
|
| 58 |
+
"""Move the cached model to CUDA and run Qwen3-VL (ZeroGPU window)."""
|
| 59 |
+
import torch
|
| 60 |
+
|
| 61 |
+
processor, model = _CACHE["processor"], _CACHE["model"]
|
| 62 |
+
model.to("cuda")
|
| 63 |
+
|
| 64 |
+
content = [{"type": "image", "image": im} for im in pil_images]
|
| 65 |
+
content.append({"type": "text", "text": prompt})
|
| 66 |
+
messages = [{"role": "user", "content": content}]
|
| 67 |
+
|
| 68 |
+
inputs = processor.apply_chat_template(
|
| 69 |
+
messages, tokenize=True, add_generation_prompt=True,
|
| 70 |
+
return_tensors="pt", return_dict=True,
|
| 71 |
+
).to("cuda")
|
| 72 |
+
|
| 73 |
+
with torch.no_grad():
|
| 74 |
+
out = model.generate(
|
| 75 |
+
**inputs, max_new_tokens=max_tokens,
|
| 76 |
+
do_sample=temperature > 0, temperature=max(temperature, 1e-2),
|
| 77 |
+
)
|
| 78 |
+
gen = out[:, inputs["input_ids"].shape[1]:]
|
| 79 |
+
return processor.batch_decode(gen, skip_special_tokens=True)[0]
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
class TransformersVLMClient:
|
| 83 |
+
"""In-process Qwen3-VL client (ZeroGPU on Spaces)."""
|
| 84 |
+
|
| 85 |
+
def __init__(self, model_id: str | None = None):
|
| 86 |
+
self.model_id = model_id or config.JUDGE_HF_MODEL
|
| 87 |
+
self._failed = False
|
| 88 |
+
|
| 89 |
+
@property
|
| 90 |
+
def available(self) -> bool:
|
| 91 |
+
"""Cheap check β does NOT load the model (so tests stay download-free)."""
|
| 92 |
+
if self._failed:
|
| 93 |
+
return False
|
| 94 |
+
return importlib.util.find_spec("transformers") is not None
|
| 95 |
+
|
| 96 |
+
def complete(self, prompt: str, images: list[str] | None = None,
|
| 97 |
+
max_tokens: int = 512, temperature: float = 0.1,
|
| 98 |
+
stop: list[str] | None = None) -> dict:
|
| 99 |
+
try:
|
| 100 |
+
pil_images = self._decode_images(images)
|
| 101 |
+
_ensure_loaded(self.model_id) # CPU load (no GPU time)
|
| 102 |
+
text = _generate(prompt, pil_images, max_tokens, temperature)
|
| 103 |
+
return LlamaCppClient._parse_json_reply(text)
|
| 104 |
+
except Exception as e: # pragma: no cover - needs GPU + model
|
| 105 |
+
logger.warning("transformers VLM failed (%s) β falling back to rubric", e)
|
| 106 |
+
self._failed = True
|
| 107 |
+
return {"error": str(e), "fallback": True, "text": ""}
|
| 108 |
+
|
| 109 |
+
@staticmethod
|
| 110 |
+
def _decode_images(images: list[str] | None) -> list:
|
| 111 |
+
"""Decode base64 JPEGs (as the JudgeAgent encodes them) into PIL images."""
|
| 112 |
+
if not images:
|
| 113 |
+
return []
|
| 114 |
+
import io
|
| 115 |
+
|
| 116 |
+
from PIL import Image
|
| 117 |
+
|
| 118 |
+
out = []
|
| 119 |
+
for img in images:
|
| 120 |
+
try:
|
| 121 |
+
raw = base64.b64decode(img)
|
| 122 |
+
out.append(Image.open(io.BytesIO(raw)).convert("RGB"))
|
| 123 |
+
except Exception:
|
| 124 |
+
continue
|
| 125 |
+
return out
|