wangjazz commited on
Commit
75be9ec
Β·
1 Parent(s): b908489

Add web demo UI and fix quantized model loading

Browse files

- Add Flask web app (web_app.py + templates/index.html) for interactive
video temporal grounding with timeline visualization
- Fix quantized model loading: apply nn.quantize structure before loading
8-bit weights to create QuantizedLinear layers (fixes shape mismatch)
- Replace OpenCV VideoCapture with ffmpeg-based frame extraction to avoid
SIGSEGV crash in libopencv_videoio FFMPEG backend

Files changed (4) hide show
  1. mlx_vidi/preprocessing.py +43 -24
  2. mlx_vidi/run.py +11 -0
  3. templates/index.html +690 -0
  4. web_app.py +192 -0
mlx_vidi/preprocessing.py CHANGED
@@ -30,7 +30,7 @@ def extract_video_frames(
30
  ) -> np.ndarray:
31
  """Extract frames from video at specified FPS.
32
 
33
- Uses decord if available, falls back to OpenCV.
34
 
35
  Returns:
36
  np.ndarray of shape (N, H, W, 3) in uint8 RGB.
@@ -42,11 +42,8 @@ def extract_video_frames(
42
  except ImportError:
43
  pass
44
 
45
- try:
46
- import cv2
47
- return _extract_frames_cv2(video_path, fps, max_frames)
48
- except ImportError:
49
- raise ImportError("Either decord or opencv-python is required for video processing")
50
 
51
 
52
  def _extract_frames_decord(video_path, fps, max_frames):
@@ -63,25 +60,47 @@ def _extract_frames_decord(video_path, fps, max_frames):
63
  return frames
64
 
65
 
66
- def _extract_frames_cv2(video_path, fps, max_frames):
67
- import cv2
68
- cap = cv2.VideoCapture(video_path)
69
- if not cap.isOpened():
70
- raise ValueError(f"Cannot open video: {video_path}")
71
- video_fps = cap.get(cv2.CAP_PROP_FPS) or 30.0
72
- total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
73
- duration = total_frames / video_fps
 
 
 
 
 
 
 
74
  num_frames = min(max(int(duration * fps), 1), max_frames)
75
- indices = np.linspace(0, total_frames - 1, num_frames, dtype=int)
76
- frames = []
77
- for idx in indices:
78
- cap.set(cv2.CAP_PROP_POS_FRAMES, int(idx))
79
- ret, frame = cap.read()
80
- if ret:
81
- frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
82
- cap.release()
83
- if not frames:
84
- raise ValueError(f"No frames extracted from {video_path}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
85
  return np.stack(frames)
86
 
87
 
 
30
  ) -> np.ndarray:
31
  """Extract frames from video at specified FPS.
32
 
33
+ Uses decord if available, falls back to ffmpeg (PIL-based).
34
 
35
  Returns:
36
  np.ndarray of shape (N, H, W, 3) in uint8 RGB.
 
42
  except ImportError:
43
  pass
44
 
45
+ # Use ffmpeg directly β€” avoids OpenCV FFMPEG backend SIGSEGV
46
+ return _extract_frames_ffmpeg(video_path, fps, max_frames)
 
 
 
47
 
48
 
49
  def _extract_frames_decord(video_path, fps, max_frames):
 
60
  return frames
61
 
62
 
63
+ def _extract_frames_ffmpeg(video_path, fps, max_frames):
64
+ """Extract frames using ffmpeg β†’ PNG β†’ PIL. Avoids OpenCV SIGSEGV."""
65
+ from PIL import Image
66
+
67
+ # Get video duration via ffprobe
68
+ try:
69
+ result = subprocess.run(
70
+ ["ffprobe", "-v", "quiet", "-show_entries", "format=duration",
71
+ "-of", "default=noprint_wrappers=1:nokey=1", video_path],
72
+ capture_output=True, text=True, check=True,
73
+ )
74
+ duration = float(result.stdout.strip())
75
+ except Exception:
76
+ duration = 60.0 # fallback
77
+
78
  num_frames = min(max(int(duration * fps), 1), max_frames)
79
+
80
+ with tempfile.TemporaryDirectory() as tmpdir:
81
+ # Extract frames at target fps using ffmpeg
82
+ cmd = [
83
+ "ffmpeg", "-y", "-i", video_path,
84
+ "-vf", f"fps={fps}",
85
+ "-frames:v", str(num_frames),
86
+ "-q:v", "2",
87
+ os.path.join(tmpdir, "frame_%05d.png"),
88
+ ]
89
+ subprocess.run(cmd, capture_output=True, check=True)
90
+
91
+ # Load frames
92
+ frame_files = sorted(
93
+ f for f in os.listdir(tmpdir) if f.endswith(".png")
94
+ )[:max_frames]
95
+
96
+ if not frame_files:
97
+ raise ValueError(f"No frames extracted from {video_path}")
98
+
99
+ frames = []
100
+ for fname in frame_files:
101
+ img = Image.open(os.path.join(tmpdir, fname)).convert("RGB")
102
+ frames.append(np.array(img))
103
+
104
  return np.stack(frames)
105
 
106
 
mlx_vidi/run.py CHANGED
@@ -86,6 +86,17 @@ def load_model(model_path: str, config: ModelConfig) -> VidiEngine:
86
  """Load Vidi model with MLX weights."""
87
  engine = VidiEngine(config)
88
 
 
 
 
 
 
 
 
 
 
 
 
89
  # Load weights
90
  weight_files = sorted(Path(model_path).glob("*.safetensors"))
91
  if not weight_files:
 
86
  """Load Vidi model with MLX weights."""
87
  engine = VidiEngine(config)
88
 
89
+ # Check if model is quantized and apply quantization structure first
90
+ config_path = os.path.join(model_path, "config.json")
91
+ with open(config_path) as f:
92
+ raw_config = json.load(f)
93
+
94
+ if "quantization" in raw_config:
95
+ quant = raw_config["quantization"]
96
+ print(f"Applying {quant['bits']}-bit quantization structure (group_size={quant['group_size']})...")
97
+ from .quantize import quantize_engine
98
+ quantize_engine(engine, bits=quant["bits"], group_size=quant["group_size"])
99
+
100
  # Load weights
101
  weight_files = sorted(Path(model_path).glob("*.safetensors"))
102
  if not weight_files:
templates/index.html ADDED
@@ -0,0 +1,690 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="UTF-8">
5
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
6
+ <title>Vidi1.5-9B MLX β€” Video Temporal Grounding</title>
7
+ <style>
8
+ :root {
9
+ --bg: #0f1117;
10
+ --surface: #1a1d27;
11
+ --surface2: #252833;
12
+ --border: #363a48;
13
+ --text: #e4e6eb;
14
+ --text2: #9aa0ad;
15
+ --accent: #6c8aff;
16
+ --accent2: #4ecdc4;
17
+ --danger: #ff6b6b;
18
+ --segment: rgba(108, 138, 255, 0.35);
19
+ --segment-border: rgba(108, 138, 255, 0.8);
20
+ }
21
+
22
+ * { box-sizing: border-box; margin: 0; padding: 0; }
23
+
24
+ body {
25
+ font-family: -apple-system, BlinkMacSystemFont, 'SF Pro', 'Segoe UI', system-ui, sans-serif;
26
+ background: var(--bg);
27
+ color: var(--text);
28
+ min-height: 100vh;
29
+ }
30
+
31
+ .container {
32
+ max-width: 1100px;
33
+ margin: 0 auto;
34
+ padding: 24px;
35
+ }
36
+
37
+ header {
38
+ text-align: center;
39
+ padding: 32px 0 24px;
40
+ }
41
+ header h1 {
42
+ font-size: 28px;
43
+ font-weight: 700;
44
+ background: linear-gradient(135deg, var(--accent), var(--accent2));
45
+ -webkit-background-clip: text;
46
+ -webkit-text-fill-color: transparent;
47
+ }
48
+ header p {
49
+ color: var(--text2);
50
+ margin-top: 8px;
51
+ font-size: 15px;
52
+ }
53
+
54
+ .card {
55
+ background: var(--surface);
56
+ border: 1px solid var(--border);
57
+ border-radius: 12px;
58
+ padding: 24px;
59
+ margin-bottom: 20px;
60
+ }
61
+
62
+ .card h2 {
63
+ font-size: 16px;
64
+ font-weight: 600;
65
+ margin-bottom: 16px;
66
+ color: var(--text2);
67
+ text-transform: uppercase;
68
+ letter-spacing: 0.5px;
69
+ }
70
+
71
+ /* Upload area */
72
+ .upload-zone {
73
+ border: 2px dashed var(--border);
74
+ border-radius: 10px;
75
+ padding: 40px;
76
+ text-align: center;
77
+ cursor: pointer;
78
+ transition: all 0.2s;
79
+ position: relative;
80
+ }
81
+ .upload-zone:hover, .upload-zone.drag-over {
82
+ border-color: var(--accent);
83
+ background: rgba(108, 138, 255, 0.05);
84
+ }
85
+ .upload-zone.has-file {
86
+ border-color: var(--accent2);
87
+ border-style: solid;
88
+ padding: 16px;
89
+ }
90
+ .upload-zone input[type="file"] {
91
+ position: absolute;
92
+ inset: 0;
93
+ opacity: 0;
94
+ cursor: pointer;
95
+ }
96
+ .upload-icon { font-size: 40px; margin-bottom: 12px; }
97
+ .upload-text { color: var(--text2); font-size: 14px; }
98
+ .file-info {
99
+ display: flex;
100
+ align-items: center;
101
+ gap: 12px;
102
+ font-size: 14px;
103
+ }
104
+ .file-info .name { font-weight: 600; }
105
+ .file-info .meta { color: var(--text2); }
106
+
107
+ /* Query input */
108
+ .query-row {
109
+ display: flex;
110
+ gap: 12px;
111
+ margin-top: 16px;
112
+ }
113
+ .query-input {
114
+ flex: 1;
115
+ padding: 12px 16px;
116
+ border: 1px solid var(--border);
117
+ border-radius: 8px;
118
+ background: var(--surface2);
119
+ color: var(--text);
120
+ font-size: 15px;
121
+ outline: none;
122
+ transition: border-color 0.2s;
123
+ }
124
+ .query-input:focus { border-color: var(--accent); }
125
+ .query-input::placeholder { color: var(--text2); }
126
+
127
+ .btn {
128
+ padding: 12px 24px;
129
+ border: none;
130
+ border-radius: 8px;
131
+ font-size: 15px;
132
+ font-weight: 600;
133
+ cursor: pointer;
134
+ transition: all 0.2s;
135
+ }
136
+ .btn-primary {
137
+ background: var(--accent);
138
+ color: #fff;
139
+ }
140
+ .btn-primary:hover { filter: brightness(1.15); }
141
+ .btn-primary:disabled {
142
+ opacity: 0.5;
143
+ cursor: not-allowed;
144
+ }
145
+
146
+ /* Quick queries */
147
+ .quick-queries {
148
+ display: flex;
149
+ flex-wrap: wrap;
150
+ gap: 8px;
151
+ margin-top: 12px;
152
+ }
153
+ .quick-btn {
154
+ padding: 6px 14px;
155
+ border: 1px solid var(--border);
156
+ border-radius: 20px;
157
+ background: transparent;
158
+ color: var(--text2);
159
+ font-size: 13px;
160
+ cursor: pointer;
161
+ transition: all 0.2s;
162
+ }
163
+ .quick-btn:hover {
164
+ border-color: var(--accent);
165
+ color: var(--accent);
166
+ }
167
+
168
+ /* Video player */
169
+ .video-container {
170
+ position: relative;
171
+ background: #000;
172
+ border-radius: 10px;
173
+ overflow: hidden;
174
+ }
175
+ video {
176
+ width: 100%;
177
+ display: block;
178
+ border-radius: 10px;
179
+ }
180
+
181
+ /* Timeline */
182
+ .timeline-container { margin-top: 16px; }
183
+ .timeline-label {
184
+ font-size: 13px;
185
+ color: var(--text2);
186
+ margin-bottom: 6px;
187
+ }
188
+ .timeline {
189
+ position: relative;
190
+ height: 40px;
191
+ background: var(--surface2);
192
+ border-radius: 6px;
193
+ overflow: hidden;
194
+ cursor: pointer;
195
+ }
196
+ .timeline-segment {
197
+ position: absolute;
198
+ top: 0;
199
+ height: 100%;
200
+ background: var(--segment);
201
+ border-left: 2px solid var(--segment-border);
202
+ border-right: 2px solid var(--segment-border);
203
+ cursor: pointer;
204
+ transition: background 0.15s;
205
+ display: flex;
206
+ align-items: center;
207
+ justify-content: center;
208
+ font-size: 11px;
209
+ color: rgba(255,255,255,0.9);
210
+ font-weight: 600;
211
+ min-width: 4px;
212
+ }
213
+ .timeline-segment:hover {
214
+ background: rgba(108, 138, 255, 0.55);
215
+ }
216
+ .timeline-playhead {
217
+ position: absolute;
218
+ top: 0;
219
+ width: 2px;
220
+ height: 100%;
221
+ background: var(--danger);
222
+ pointer-events: none;
223
+ transition: left 0.1s linear;
224
+ }
225
+ .timeline-ticks {
226
+ display: flex;
227
+ justify-content: space-between;
228
+ font-size: 11px;
229
+ color: var(--text2);
230
+ margin-top: 4px;
231
+ padding: 0 2px;
232
+ }
233
+
234
+ /* Results */
235
+ .results-grid {
236
+ display: grid;
237
+ grid-template-columns: 1fr 1fr;
238
+ gap: 16px;
239
+ margin-top: 16px;
240
+ }
241
+ .result-item {
242
+ background: var(--surface2);
243
+ border-radius: 8px;
244
+ padding: 14px;
245
+ }
246
+ .result-item .label {
247
+ font-size: 12px;
248
+ color: var(--text2);
249
+ margin-bottom: 4px;
250
+ }
251
+ .result-item .value {
252
+ font-size: 18px;
253
+ font-weight: 700;
254
+ font-family: 'SF Mono', 'Fira Code', monospace;
255
+ }
256
+
257
+ /* Segments list */
258
+ .segments-list { margin-top: 16px; }
259
+ .segment-chip {
260
+ display: inline-flex;
261
+ align-items: center;
262
+ gap: 8px;
263
+ padding: 8px 14px;
264
+ margin: 4px;
265
+ background: var(--surface2);
266
+ border: 1px solid var(--border);
267
+ border-radius: 8px;
268
+ cursor: pointer;
269
+ transition: all 0.15s;
270
+ font-size: 14px;
271
+ }
272
+ .segment-chip:hover {
273
+ border-color: var(--accent);
274
+ background: rgba(108, 138, 255, 0.1);
275
+ }
276
+ .segment-chip .time {
277
+ font-family: 'SF Mono', 'Fira Code', monospace;
278
+ font-weight: 600;
279
+ color: var(--accent);
280
+ }
281
+ .segment-chip .norm {
282
+ color: var(--text2);
283
+ font-size: 12px;
284
+ }
285
+
286
+ /* Status */
287
+ .status {
288
+ text-align: center;
289
+ padding: 20px;
290
+ color: var(--text2);
291
+ font-size: 14px;
292
+ }
293
+ .status.loading { color: var(--accent); }
294
+ .status.error { color: var(--danger); }
295
+
296
+ .spinner {
297
+ display: inline-block;
298
+ width: 20px;
299
+ height: 20px;
300
+ border: 2px solid var(--border);
301
+ border-top-color: var(--accent);
302
+ border-radius: 50%;
303
+ animation: spin 0.8s linear infinite;
304
+ margin-right: 8px;
305
+ vertical-align: middle;
306
+ }
307
+ @keyframes spin { to { transform: rotate(360deg); } }
308
+
309
+ /* Raw output */
310
+ .raw-output {
311
+ margin-top: 12px;
312
+ padding: 12px;
313
+ background: var(--surface2);
314
+ border-radius: 8px;
315
+ font-family: 'SF Mono', 'Fira Code', monospace;
316
+ font-size: 14px;
317
+ word-break: break-all;
318
+ }
319
+
320
+ /* History */
321
+ .history-item {
322
+ padding: 12px;
323
+ background: var(--surface2);
324
+ border-radius: 8px;
325
+ margin-bottom: 8px;
326
+ cursor: pointer;
327
+ transition: all 0.15s;
328
+ }
329
+ .history-item:hover { border-left: 3px solid var(--accent); }
330
+ .history-query { font-weight: 600; font-size: 14px; }
331
+ .history-result { color: var(--text2); font-size: 13px; margin-top: 4px; }
332
+
333
+ .hidden { display: none; }
334
+
335
+ /* Settings row */
336
+ .settings-row {
337
+ display: flex;
338
+ gap: 16px;
339
+ margin-top: 12px;
340
+ font-size: 13px;
341
+ }
342
+ .settings-row label { color: var(--text2); }
343
+ .settings-row input[type="number"] {
344
+ width: 70px;
345
+ padding: 4px 8px;
346
+ border: 1px solid var(--border);
347
+ border-radius: 4px;
348
+ background: var(--surface2);
349
+ color: var(--text);
350
+ font-size: 13px;
351
+ }
352
+ </style>
353
+ </head>
354
+ <body>
355
+
356
+ <div class="container">
357
+ <header>
358
+ <h1>Vidi1.5-9B MLX</h1>
359
+ <p>Video Temporal Grounding β€” find <em>when</em> things happen in your video</p>
360
+ </header>
361
+
362
+ <!-- Upload + Query -->
363
+ <div class="card">
364
+ <h2>Input</h2>
365
+ <div class="upload-zone" id="uploadZone">
366
+ <input type="file" id="videoInput" accept="video/*">
367
+ <div id="uploadPrompt">
368
+ <div class="upload-icon">&#x1F3AC;</div>
369
+ <div>Drop video here or click to upload</div>
370
+ <div class="upload-text">MP4, MOV, AVI β€” up to 500 MB</div>
371
+ </div>
372
+ <div id="fileInfo" class="file-info hidden"></div>
373
+ </div>
374
+
375
+ <div class="query-row">
376
+ <input type="text" class="query-input" id="queryInput"
377
+ placeholder="Describe what to find, e.g. &quot;a person singing&quot; or &quot;guitar playing&quot;"
378
+ autocomplete="off">
379
+ <button class="btn btn-primary" id="runBtn" disabled>Analyze</button>
380
+ </div>
381
+
382
+ <div class="quick-queries">
383
+ <button class="quick-btn" data-q="a person talking">person talking</button>
384
+ <button class="quick-btn" data-q="a person singing">person singing</button>
385
+ <button class="quick-btn" data-q="an instrument being played">instrument played</button>
386
+ <button class="quick-btn" data-q="a close-up of a face">face close-up</button>
387
+ <button class="quick-btn" data-q="an audience or crowd">audience/crowd</button>
388
+ <button class="quick-btn" data-q="text or titles on screen">text on screen</button>
389
+ </div>
390
+
391
+ <div class="settings-row">
392
+ <label>FPS: <input type="number" id="fpsInput" value="1" min="0.5" max="10" step="0.5"></label>
393
+ <label>Max frames: <input type="number" id="maxFramesInput" value="128" min="1" max="512" step="1"></label>
394
+ </div>
395
+ </div>
396
+
397
+ <!-- Status -->
398
+ <div id="status" class="status hidden"></div>
399
+
400
+ <!-- Video Player + Timeline -->
401
+ <div id="playerCard" class="card hidden">
402
+ <h2>Video Player</h2>
403
+ <div class="video-container">
404
+ <video id="videoPlayer" controls></video>
405
+ </div>
406
+ <div class="timeline-container">
407
+ <div class="timeline-label">Timeline β€” <span id="durationLabel">0:00</span></div>
408
+ <div class="timeline" id="timeline">
409
+ <div class="timeline-playhead" id="playhead"></div>
410
+ </div>
411
+ <div class="timeline-ticks" id="timelineTicks"></div>
412
+ </div>
413
+ </div>
414
+
415
+ <!-- Results -->
416
+ <div id="resultsCard" class="card hidden">
417
+ <h2>Results</h2>
418
+
419
+ <div class="results-grid">
420
+ <div class="result-item">
421
+ <div class="label">Segments Found</div>
422
+ <div class="value" id="segCount">β€”</div>
423
+ </div>
424
+ <div class="result-item">
425
+ <div class="label">Speed</div>
426
+ <div class="value" id="speed">β€”</div>
427
+ </div>
428
+ <div class="result-item">
429
+ <div class="label">Preprocess</div>
430
+ <div class="value" id="tPreprocess">β€”</div>
431
+ </div>
432
+ <div class="result-item">
433
+ <div class="label">Generate</div>
434
+ <div class="value" id="tGenerate">β€”</div>
435
+ </div>
436
+ </div>
437
+
438
+ <div class="segments-list" id="segmentsList"></div>
439
+
440
+ <div>
441
+ <div class="timeline-label" style="margin-top:16px;">Raw Model Output</div>
442
+ <div class="raw-output" id="rawOutput"></div>
443
+ </div>
444
+ </div>
445
+
446
+ <!-- History -->
447
+ <div id="historyCard" class="card hidden">
448
+ <h2>Query History</h2>
449
+ <div id="historyList"></div>
450
+ </div>
451
+ </div>
452
+
453
+ <script>
454
+ const $ = s => document.querySelector(s);
455
+ const $$ = s => document.querySelectorAll(s);
456
+
457
+ // State
458
+ let currentFile = null;
459
+ let videoDuration = 0;
460
+ let currentSegments = [];
461
+ let queryHistory = [];
462
+
463
+ // Elements
464
+ const uploadZone = $('#uploadZone');
465
+ const videoInput = $('#videoInput');
466
+ const queryInput = $('#queryInput');
467
+ const runBtn = $('#runBtn');
468
+ const videoPlayer = $('#videoPlayer');
469
+ const timeline = $('#timeline');
470
+ const playhead = $('#playhead');
471
+ const status = $('#status');
472
+
473
+ // Upload handling
474
+ videoInput.addEventListener('change', handleFileSelect);
475
+ uploadZone.addEventListener('dragover', e => {
476
+ e.preventDefault();
477
+ uploadZone.classList.add('drag-over');
478
+ });
479
+ uploadZone.addEventListener('dragleave', () => uploadZone.classList.remove('drag-over'));
480
+ uploadZone.addEventListener('drop', e => {
481
+ e.preventDefault();
482
+ uploadZone.classList.remove('drag-over');
483
+ if (e.dataTransfer.files.length) {
484
+ videoInput.files = e.dataTransfer.files;
485
+ handleFileSelect();
486
+ }
487
+ });
488
+
489
+ async function handleFileSelect() {
490
+ const file = videoInput.files[0];
491
+ if (!file) return;
492
+
493
+ showStatus('Uploading video...', 'loading');
494
+
495
+ const form = new FormData();
496
+ form.append('video', file);
497
+
498
+ try {
499
+ const resp = await fetch('/upload', { method: 'POST', body: form });
500
+ const data = await resp.json();
501
+ if (data.error) throw new Error(data.error);
502
+
503
+ currentFile = data.filename;
504
+ videoDuration = data.duration;
505
+
506
+ // Update UI
507
+ uploadZone.classList.add('has-file');
508
+ $('#uploadPrompt').classList.add('hidden');
509
+ $('#fileInfo').classList.remove('hidden');
510
+ $('#fileInfo').innerHTML = `
511
+ <span class="name">${file.name}</span>
512
+ <span class="meta">${data.size_mb} MB &middot; ${formatTime(data.duration)}</span>
513
+ `;
514
+
515
+ // Setup video player
516
+ videoPlayer.src = `/uploads/${data.filename}`;
517
+ $('#playerCard').classList.remove('hidden');
518
+ $('#durationLabel').textContent = formatTime(data.duration);
519
+ buildTimelineTicks(data.duration);
520
+
521
+ runBtn.disabled = false;
522
+ hideStatus();
523
+ } catch (err) {
524
+ showStatus('Upload failed: ' + err.message, 'error');
525
+ }
526
+ }
527
+
528
+ // Query
529
+ runBtn.addEventListener('click', runInference);
530
+ queryInput.addEventListener('keydown', e => {
531
+ if (e.key === 'Enter' && !runBtn.disabled && queryInput.value.trim()) runInference();
532
+ });
533
+
534
+ $$('.quick-btn').forEach(btn => {
535
+ btn.addEventListener('click', () => {
536
+ queryInput.value = btn.dataset.q;
537
+ if (currentFile) runInference();
538
+ });
539
+ });
540
+
541
+ async function runInference() {
542
+ const query = queryInput.value.trim();
543
+ if (!query || !currentFile) return;
544
+
545
+ runBtn.disabled = true;
546
+ showStatus('<span class="spinner"></span> Analyzing video... this may take 1-2 minutes', 'loading');
547
+
548
+ try {
549
+ const resp = await fetch('/infer', {
550
+ method: 'POST',
551
+ headers: { 'Content-Type': 'application/json' },
552
+ body: JSON.stringify({
553
+ filename: currentFile,
554
+ query: query,
555
+ fps: parseFloat($('#fpsInput').value) || 1.0,
556
+ max_frames: parseInt($('#maxFramesInput').value) || 128,
557
+ }),
558
+ });
559
+ const data = await resp.json();
560
+ if (data.error) throw new Error(data.error);
561
+
562
+ currentSegments = data.segments;
563
+ displayResults(data, query);
564
+ addToHistory(query, data);
565
+ hideStatus();
566
+ } catch (err) {
567
+ showStatus('Inference failed: ' + err.message, 'error');
568
+ } finally {
569
+ runBtn.disabled = false;
570
+ }
571
+ }
572
+
573
+ function displayResults(data, query) {
574
+ $('#resultsCard').classList.remove('hidden');
575
+ $('#segCount').textContent = data.segments.length;
576
+ $('#speed').textContent = `${data.tokens_per_sec} tok/s`;
577
+ $('#tPreprocess').textContent = `${data.time_preprocess}s`;
578
+ $('#tGenerate').textContent = `${data.time_generate}s`;
579
+ $('#rawOutput').textContent = data.raw_output;
580
+
581
+ // Segments on timeline
582
+ timeline.querySelectorAll('.timeline-segment').forEach(el => el.remove());
583
+ data.segments.forEach((seg, i) => {
584
+ const left = (seg.start / videoDuration) * 100;
585
+ const width = Math.max(((seg.end - seg.start) / videoDuration) * 100, 0.5);
586
+ const el = document.createElement('div');
587
+ el.className = 'timeline-segment';
588
+ el.style.left = `${left}%`;
589
+ el.style.width = `${width}%`;
590
+ el.title = `${formatTime(seg.start)} - ${formatTime(seg.end)}`;
591
+ el.textContent = width > 5 ? `${formatTime(seg.start)}` : '';
592
+ el.addEventListener('click', e => {
593
+ e.stopPropagation();
594
+ videoPlayer.currentTime = seg.start;
595
+ videoPlayer.play();
596
+ });
597
+ timeline.appendChild(el);
598
+ });
599
+
600
+ // Segment chips
601
+ const list = $('#segmentsList');
602
+ list.innerHTML = '';
603
+ data.segments.forEach((seg, i) => {
604
+ const chip = document.createElement('div');
605
+ chip.className = 'segment-chip';
606
+ chip.innerHTML = `
607
+ <span class="time">${formatTime(seg.start)} - ${formatTime(seg.end)}</span>
608
+ <span class="norm">(${seg.raw})</span>
609
+ `;
610
+ chip.addEventListener('click', () => {
611
+ videoPlayer.currentTime = seg.start;
612
+ videoPlayer.play();
613
+ });
614
+ list.appendChild(chip);
615
+ });
616
+ }
617
+
618
+ function addToHistory(query, data) {
619
+ queryHistory.unshift({ query, data });
620
+ const card = $('#historyCard');
621
+ card.classList.remove('hidden');
622
+ const list = $('#historyList');
623
+
624
+ const item = document.createElement('div');
625
+ item.className = 'history-item';
626
+ item.innerHTML = `
627
+ <div class="history-query">${escapeHtml(query)}</div>
628
+ <div class="history-result">${data.segments.length} segments: ${data.raw_output}</div>
629
+ `;
630
+ item.addEventListener('click', () => {
631
+ currentSegments = data.segments;
632
+ displayResults(data, query);
633
+ });
634
+ list.prepend(item);
635
+ }
636
+
637
+ // Playhead tracking
638
+ videoPlayer.addEventListener('timeupdate', () => {
639
+ if (videoDuration > 0) {
640
+ const pct = (videoPlayer.currentTime / videoDuration) * 100;
641
+ playhead.style.left = `${pct}%`;
642
+ }
643
+ });
644
+
645
+ // Click on timeline to seek
646
+ timeline.addEventListener('click', e => {
647
+ const rect = timeline.getBoundingClientRect();
648
+ const pct = (e.clientX - rect.left) / rect.width;
649
+ videoPlayer.currentTime = pct * videoDuration;
650
+ });
651
+
652
+ // Timeline ticks
653
+ function buildTimelineTicks(duration) {
654
+ const ticks = $('#timelineTicks');
655
+ ticks.innerHTML = '';
656
+ const numTicks = Math.min(Math.ceil(duration / 10), 12);
657
+ const interval = duration / numTicks;
658
+ for (let i = 0; i <= numTicks; i++) {
659
+ const span = document.createElement('span');
660
+ span.textContent = formatTime(i * interval);
661
+ ticks.appendChild(span);
662
+ }
663
+ }
664
+
665
+ // Helpers
666
+ function formatTime(s) {
667
+ const m = Math.floor(s / 60);
668
+ const sec = Math.floor(s % 60);
669
+ return `${m}:${String(sec).padStart(2, '0')}`;
670
+ }
671
+
672
+ function showStatus(msg, type) {
673
+ status.className = `status ${type}`;
674
+ status.innerHTML = msg;
675
+ status.classList.remove('hidden');
676
+ }
677
+
678
+ function hideStatus() {
679
+ status.classList.add('hidden');
680
+ }
681
+
682
+ function escapeHtml(s) {
683
+ const d = document.createElement('div');
684
+ d.textContent = s;
685
+ return d.innerHTML;
686
+ }
687
+ </script>
688
+
689
+ </body>
690
+ </html>
web_app.py ADDED
@@ -0,0 +1,192 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Web UI for Vidi1.5-9B MLX β€” upload video, enter query, see temporal grounding results.
3
+
4
+ Usage:
5
+ python web_app.py [--model-path ./] [--port 7860]
6
+ """
7
+
8
+ import argparse
9
+ import json
10
+ import os
11
+ import re
12
+ import time
13
+ from pathlib import Path
14
+
15
+ from flask import Flask, render_template, request, jsonify, send_from_directory
16
+
17
+ app = Flask(__name__, template_folder="templates", static_folder="static")
18
+ app.config["MAX_CONTENT_LENGTH"] = 500 * 1024 * 1024 # 500 MB max upload
19
+
20
+ UPLOAD_DIR = Path("/tmp/vidi_uploads")
21
+ UPLOAD_DIR.mkdir(exist_ok=True)
22
+
23
+ # Global model references (loaded once at startup)
24
+ engine = None
25
+ preprocessor = None
26
+
27
+
28
+ def load_model_global(model_path: str):
29
+ """Load Vidi model once at startup."""
30
+ global engine, preprocessor
31
+
32
+ print("Loading config ...")
33
+ from mlx_vidi.run import load_config, load_model
34
+ from mlx_vidi.preprocessing import VidiPreprocessor
35
+
36
+ config = load_config(model_path)
37
+ print("Building model ...")
38
+ engine = load_model(model_path, config)
39
+ print("Loading preprocessor ...")
40
+ preprocessor = VidiPreprocessor(model_path)
41
+ print("Model ready!")
42
+
43
+
44
+ def parse_timestamps(text: str, duration: float) -> list:
45
+ """Parse model output like '0.21-0.22, 0.46-0.47' into absolute timestamps.
46
+
47
+ Returns list of dicts: [{"start": 12.6, "end": 13.2, "raw": "0.21-0.22"}, ...]
48
+ """
49
+ segments = []
50
+ # Match patterns like 0.21-0.22 or 0.210-0.220
51
+ pattern = r"(\d+\.\d+)\s*-\s*(\d+\.\d+)"
52
+ for m in re.finditer(pattern, text):
53
+ start_norm = float(m.group(1))
54
+ end_norm = float(m.group(2))
55
+ segments.append({
56
+ "start": round(start_norm * duration, 2),
57
+ "end": round(end_norm * duration, 2),
58
+ "start_norm": start_norm,
59
+ "end_norm": end_norm,
60
+ "raw": m.group(0),
61
+ })
62
+ return segments
63
+
64
+
65
+ def get_video_duration(video_path: str) -> float:
66
+ """Get video duration in seconds using ffprobe."""
67
+ import subprocess
68
+ try:
69
+ result = subprocess.run(
70
+ ["ffprobe", "-v", "quiet", "-show_entries", "format=duration",
71
+ "-of", "default=noprint_wrappers=1:nokey=1", video_path],
72
+ capture_output=True, text=True, check=True,
73
+ )
74
+ return float(result.stdout.strip())
75
+ except Exception:
76
+ return 0.0
77
+
78
+
79
+ # ── Routes ──────────────────────────────────────────────────────────────────
80
+
81
+ @app.route("/")
82
+ def index():
83
+ return render_template("index.html")
84
+
85
+
86
+ @app.route("/uploads/<path:filename>")
87
+ def serve_upload(filename):
88
+ return send_from_directory(UPLOAD_DIR, filename)
89
+
90
+
91
+ @app.route("/upload", methods=["POST"])
92
+ def upload_video():
93
+ """Handle video file upload."""
94
+ if "video" not in request.files:
95
+ return jsonify({"error": "No video file provided"}), 400
96
+
97
+ f = request.files["video"]
98
+ if f.filename == "":
99
+ return jsonify({"error": "Empty filename"}), 400
100
+
101
+ # Save with timestamp to avoid collisions
102
+ ext = Path(f.filename).suffix or ".mp4"
103
+ saved_name = f"video_{int(time.time())}{ext}"
104
+ save_path = UPLOAD_DIR / saved_name
105
+ f.save(str(save_path))
106
+
107
+ duration = get_video_duration(str(save_path))
108
+
109
+ return jsonify({
110
+ "filename": saved_name,
111
+ "duration": duration,
112
+ "size_mb": round(save_path.stat().st_size / 1024 / 1024, 1),
113
+ })
114
+
115
+
116
+ @app.route("/infer", methods=["POST"])
117
+ def infer():
118
+ """Run temporal grounding inference."""
119
+ data = request.get_json()
120
+ filename = data.get("filename")
121
+ query = data.get("query", "").strip()
122
+ fps = data.get("fps", 1.0)
123
+ max_frames = data.get("max_frames", 128)
124
+
125
+ if not filename or not query:
126
+ return jsonify({"error": "Missing filename or query"}), 400
127
+
128
+ video_path = str(UPLOAD_DIR / filename)
129
+ if not os.path.exists(video_path):
130
+ return jsonify({"error": "Video file not found"}), 404
131
+
132
+ duration = get_video_duration(video_path)
133
+
134
+ # Wrap query in Vidi's expected format
135
+ full_query = f"During which time segments in the video can we see {query}?"
136
+
137
+ try:
138
+ t0 = time.time()
139
+
140
+ # Preprocess
141
+ inputs = preprocessor.prepare_video(
142
+ video_path, full_query, fps=fps, max_frames=max_frames
143
+ )
144
+
145
+ t_preprocess = time.time() - t0
146
+
147
+ # Generate
148
+ t1 = time.time()
149
+ token_ids = engine.generate(
150
+ input_ids=inputs["input_ids"],
151
+ pixel_values=inputs["pixel_values"],
152
+ mel_features=inputs["mel_features"],
153
+ audio_sizes=inputs.get("audio_sizes", []),
154
+ max_tokens=512,
155
+ temperature=0.0,
156
+ )
157
+ t_generate = time.time() - t1
158
+
159
+ # Decode
160
+ text = preprocessor.tokenizer.decode(token_ids, skip_special_tokens=True)
161
+ segments = parse_timestamps(text, duration)
162
+
163
+ return jsonify({
164
+ "raw_output": text,
165
+ "query": query,
166
+ "full_query": full_query,
167
+ "duration": duration,
168
+ "segments": segments,
169
+ "num_frames": inputs["num_frames"],
170
+ "num_tokens": len(token_ids),
171
+ "time_preprocess": round(t_preprocess, 2),
172
+ "time_generate": round(t_generate, 2),
173
+ "tokens_per_sec": round(len(token_ids) / t_generate, 1) if t_generate > 0 else 0,
174
+ })
175
+
176
+ except Exception as e:
177
+ import traceback
178
+ traceback.print_exc()
179
+ return jsonify({"error": str(e)}), 500
180
+
181
+
182
+ # ── Main ────────────────────────────────────────────────────────────────────
183
+
184
+ if __name__ == "__main__":
185
+ parser = argparse.ArgumentParser()
186
+ parser.add_argument("--model-path", default=".", help="Path to MLX model directory")
187
+ parser.add_argument("--port", type=int, default=7860)
188
+ parser.add_argument("--host", default="127.0.0.1")
189
+ args = parser.parse_args()
190
+
191
+ load_model_global(args.model_path)
192
+ app.run(host=args.host, port=args.port, debug=False)