prithivMLmods commited on
Commit
7b2b6fa
·
verified ·
1 Parent(s): 5371ebf

update app

Browse files
Files changed (1) hide show
  1. app.py +173 -749
app.py CHANGED
@@ -1,15 +1,19 @@
1
- import gradio as gr
 
 
 
 
 
 
2
  import torch
3
  import spaces
4
  import numpy as np
 
5
  import supervision as sv
6
  from typing import Iterable
7
- from gradio.themes import Soft
8
- from gradio.themes.utils import colors, fonts, sizes
9
- import json
10
- import ast
11
- import re
12
- from PIL import Image, ImageDraw, ImageFont
13
  from threading import Thread
14
  from transformers import (
15
  Qwen3_5ForConditionalGeneration,
@@ -17,328 +21,29 @@ from transformers import (
17
  TextIteratorStreamer,
18
  )
19
 
20
-
21
  # ------------------------------------------------------------------
22
- # Theme / Colors
23
  # ------------------------------------------------------------------
24
- colors.steel_blue = colors.Color(
25
- name="steel_blue",
26
- c50="#EBF3F8", c100="#D3E5F0", c200="#A8CCE1", c300="#7DB3D2",
27
- c400="#529AC3", c500="#4682B4", c600="#3E72A0", c700="#36638C",
28
- c800="#2E5378", c900="#264364", c950="#1E3450",
29
- )
30
-
31
-
32
- class SteelBlueTheme(Soft):
33
- def __init__(
34
- self,
35
- *,
36
- primary_hue: colors.Color | str = colors.gray,
37
- secondary_hue: colors.Color | str = colors.steel_blue,
38
- neutral_hue: colors.Color | str = colors.slate,
39
- text_size: sizes.Size | str = sizes.text_lg,
40
- font: fonts.Font | str | Iterable[fonts.Font | str] = (
41
- fonts.GoogleFont("Outfit"), "Arial", "sans-serif",
42
- ),
43
- font_mono: fonts.Font | str | Iterable[fonts.Font | str] = (
44
- fonts.GoogleFont("IBM Plex Mono"), "ui-monospace", "monospace",
45
- ),
46
- ):
47
- super().__init__(
48
- primary_hue=primary_hue, secondary_hue=secondary_hue,
49
- neutral_hue=neutral_hue, text_size=text_size,
50
- font=font, font_mono=font_mono,
51
- )
52
- super().set(
53
- background_fill_primary="*primary_50",
54
- background_fill_primary_dark="*primary_900",
55
- body_background_fill="linear-gradient(135deg, *primary_200, *primary_100)",
56
- body_background_fill_dark="linear-gradient(135deg, *primary_900, *primary_800)",
57
- button_primary_text_color="white",
58
- button_primary_text_color_hover="white",
59
- button_primary_background_fill="linear-gradient(90deg, *secondary_500, *secondary_600)",
60
- button_primary_background_fill_hover="linear-gradient(90deg, *secondary_600, *secondary_700)",
61
- button_primary_background_fill_dark="linear-gradient(90deg, *secondary_600, *secondary_800)",
62
- button_primary_background_fill_hover_dark="linear-gradient(90deg, *secondary_500, *secondary_500)",
63
- button_secondary_text_color="black",
64
- button_secondary_text_color_hover="white",
65
- button_secondary_background_fill="linear-gradient(90deg, *primary_300, *primary_300)",
66
- button_secondary_background_fill_hover="linear-gradient(90deg, *primary_400, *primary_400)",
67
- button_secondary_background_fill_dark="linear-gradient(90deg, *primary_500, *primary_600)",
68
- button_secondary_background_fill_hover_dark="linear-gradient(90deg, *primary_500, *primary_500)",
69
- slider_color="*secondary_500",
70
- slider_color_dark="*secondary_600",
71
- block_title_text_weight="600",
72
- block_border_width="3px",
73
- block_shadow="*shadow_drop_lg",
74
- button_primary_shadow="*shadow_drop_lg",
75
- button_large_padding="11px",
76
- color_accent_soft="*primary_100",
77
- block_label_background_fill="*primary_200",
78
- )
79
-
80
-
81
- steel_blue_theme = SteelBlueTheme()
82
-
83
-
84
- css = r"""
85
- @import url('https://fonts.googleapis.com/css2?family=Outfit:wght@300;400;500;600;700;800&family=IBM+Plex+Mono:wght@400;500;600&display=swap');
86
-
87
- body, .gradio-container { font-family: 'Outfit', sans-serif !important; }
88
- footer { display: none !important; }
89
-
90
- /* -- App Header -- */
91
- .app-header {
92
- background: linear-gradient(135deg, #1E3450 0%, #264364 30%, #3E72A0 70%, #4682B4 100%);
93
- border-radius: 16px; padding: 32px 40px; margin-bottom: 24px;
94
- position: relative; overflow: hidden;
95
- box-shadow: 0 8px 32px rgba(30,52,80,0.25);
96
- }
97
- .app-header::before {
98
- content:''; position:absolute; top:-50%; right:-20%;
99
- width:400px; height:400px;
100
- background:radial-gradient(circle,rgba(255,255,255,0.06) 0%,transparent 70%);
101
- border-radius:50%;
102
- }
103
- .app-header::after {
104
- content:''; position:absolute; bottom:-30%; left:-10%;
105
- width:300px; height:300px;
106
- background:radial-gradient(circle,rgba(70,130,180,0.15) 0%,transparent 70%);
107
- border-radius:50%;
108
- }
109
- .header-content {
110
- display:flex; align-items:center; gap:24px;
111
- position:relative; z-index:1;
112
- }
113
- .header-icon-wrap {
114
- width:64px; height:64px; background:rgba(255,255,255,0.12);
115
- border-radius:16px; display:flex; align-items:center; justify-content:center;
116
- flex-shrink:0; backdrop-filter:blur(8px); border:1px solid rgba(255,255,255,0.15);
117
- }
118
- .header-icon-wrap svg { width:36px; height:36px; color:rgba(255,255,255,0.9); }
119
- .header-text h1 {
120
- font-family:'Outfit',sans-serif; font-size:2rem; font-weight:700;
121
- color:#fff; margin:0 0 8px 0; letter-spacing:-0.02em; line-height:1.2;
122
- }
123
- .header-meta { display:flex; align-items:center; gap:12px; flex-wrap:wrap; }
124
- .meta-badge {
125
- display:inline-flex; align-items:center; gap:6px;
126
- background:rgba(255,255,255,0.12); color:rgba(255,255,255,0.9);
127
- padding:4px 12px; border-radius:20px;
128
- font-family:'IBM Plex Mono',monospace; font-size:0.8rem; font-weight:500;
129
- border:1px solid rgba(255,255,255,0.1); backdrop-filter:blur(4px);
130
- }
131
- .meta-badge svg { width:14px; height:14px; }
132
- .meta-sep {
133
- width:4px; height:4px; background:rgba(255,255,255,0.35);
134
- border-radius:50%; flex-shrink:0;
135
- }
136
- .meta-cap { color:rgba(255,255,255,0.65); font-size:0.85rem; font-weight:400; }
137
-
138
- /* -- Tab transitions -- */
139
- .gradio-tabitem { animation: tabFadeIn 0.35s ease-out; }
140
- @keyframes tabFadeIn {
141
- from { opacity:0; transform:translateY(6px); }
142
- to { opacity:1; transform:translateY(0); }
143
- }
144
-
145
- /* -- Tab Intro Panels -- */
146
- .tab-intro {
147
- display:flex; align-items:flex-start; gap:16px;
148
- background:linear-gradient(135deg,rgba(70,130,180,0.06),rgba(70,130,180,0.02));
149
- border:1px solid rgba(70,130,180,0.15); border-left:4px solid #4682B4;
150
- border-radius:10px; padding:18px 22px; margin-bottom:20px;
151
- }
152
- .dark .tab-intro {
153
- background:linear-gradient(135deg,rgba(70,130,180,0.1),rgba(70,130,180,0.04));
154
- border-color:rgba(70,130,180,0.25);
155
- }
156
- .intro-icon {
157
- width:40px; height:40px; background:rgba(70,130,180,0.1);
158
- border-radius:10px; display:flex; align-items:center; justify-content:center;
159
- flex-shrink:0; margin-top:2px;
160
- }
161
- .intro-icon svg { width:22px; height:22px; color:#4682B4; }
162
- .dark .intro-icon svg { color:#7DB3D2; }
163
- .intro-text { flex:1; }
164
- .intro-text p { margin:0; color:#2E5378; font-size:0.95rem; line-height:1.6; }
165
- .dark .intro-text p { color:#A8CCE1; }
166
- .intro-text p.intro-sub { color:#64748b; font-size:0.85rem; margin-top:4px; }
167
- .dark .intro-text p.intro-sub { color:#94a3b8; }
168
-
169
- /* -- Section Headers -- */
170
- .section-heading {
171
- display:flex; align-items:center; gap:14px;
172
- margin:22px 0 14px 0; padding:0 2px;
173
- }
174
- .heading-icon {
175
- width:32px; height:32px;
176
- background:linear-gradient(135deg,#4682B4,#3E72A0);
177
- border-radius:8px; display:flex; align-items:center; justify-content:center;
178
- flex-shrink:0; box-shadow:0 2px 8px rgba(70,130,180,0.2);
179
- }
180
- .heading-icon svg { width:18px; height:18px; color:#fff; }
181
- .heading-label {
182
- font-family:'Outfit',sans-serif; font-weight:600; font-size:1.05rem;
183
- color:#1E3450; letter-spacing:-0.01em;
184
- }
185
- .dark .heading-label { color:#D3E5F0; }
186
- .heading-line {
187
- flex:1; height:1px;
188
- background:linear-gradient(90deg,rgba(70,130,180,0.2),transparent);
189
- }
190
-
191
- /* -- Status Indicators -- */
192
- .status-indicator {
193
- display:flex; align-items:center; gap:10px;
194
- padding:10px 16px; margin-top:10px;
195
- background:rgba(70,130,180,0.04); border:1px solid rgba(70,130,180,0.12);
196
- border-radius:8px;
197
- }
198
- .dark .status-indicator {
199
- background:rgba(70,130,180,0.08); border-color:rgba(70,130,180,0.2);
200
- }
201
- .status-dot {
202
- width:8px; height:8px; background:#22c55e;
203
- border-radius:50%; flex-shrink:0;
204
- animation:statusPulse 2s ease-in-out infinite;
205
- }
206
- @keyframes statusPulse {
207
- 0%,100% { opacity:1; box-shadow:0 0 0 0 rgba(34,197,94,0.4); }
208
- 50% { opacity:0.7; box-shadow:0 0 0 4px rgba(34,197,94,0); }
209
- }
210
- .status-text { font-size:0.85rem; color:#64748b; font-style:italic; }
211
- .dark .status-text { color:#94a3b8; }
212
-
213
- /* -- Card Labels -- */
214
- .card-label {
215
- display:flex; align-items:center; gap:8px;
216
- font-family:'Outfit',sans-serif; font-weight:600; font-size:0.8rem;
217
- text-transform:uppercase; letter-spacing:0.06em; color:#4682B4;
218
- margin-bottom:14px; padding-bottom:10px;
219
- border-bottom:1px solid rgba(70,130,180,0.1);
220
- }
221
- .dark .card-label { color:#7DB3D2; border-bottom-color:rgba(70,130,180,0.2); }
222
- .card-label svg { width:16px; height:16px; }
223
-
224
- /* -- Buttons -- */
225
- .primary {
226
- border-radius:10px !important; font-weight:600 !important;
227
- letter-spacing:0.02em !important; transition:all 0.25s ease !important;
228
- font-family:'Outfit',sans-serif !important;
229
- }
230
- .primary:hover {
231
- transform:translateY(-2px) !important;
232
- box-shadow:0 6px 20px rgba(70,130,180,0.3) !important;
233
- }
234
- .primary:active { transform:translateY(0) !important; }
235
-
236
- /* -- Textbox -- */
237
- .gradio-textbox textarea {
238
- font-family:'IBM Plex Mono',monospace !important;
239
- font-size:0.92rem !important; line-height:1.7 !important;
240
- border-radius:8px !important;
241
- }
242
-
243
- /* -- Accordion -- */
244
- .gradio-accordion {
245
- border-radius:10px !important; border:1px solid rgba(70,130,180,0.15) !important;
246
- }
247
- .gradio-accordion>.label-wrap { border-radius:10px !important; }
248
-
249
- /* -- Labels -- */
250
- label { font-weight:600 !important; font-family:'Outfit',sans-serif !important; }
251
-
252
- /* -- Slider -- */
253
- .gradio-slider input[type="range"] { accent-color:#4682B4 !important; }
254
-
255
- /* -- Scrollbar -- */
256
- ::-webkit-scrollbar { width:8px; height:8px; }
257
- ::-webkit-scrollbar-track { background:rgba(70,130,180,0.04); border-radius:4px; }
258
- ::-webkit-scrollbar-thumb { background:linear-gradient(135deg,#4682B4,#3E72A0); border-radius:4px; }
259
- ::-webkit-scrollbar-thumb:hover { background:linear-gradient(135deg,#3E72A0,#2E5378); }
260
-
261
- /* -- Gallery -- */
262
- .gradio-gallery { border-radius:10px !important; }
263
-
264
- /* -- Divider -- */
265
- .section-divider {
266
- height:1px; background:linear-gradient(90deg,transparent,rgba(70,130,180,0.2),transparent);
267
- margin:16px 0; border:none;
268
- }
269
-
270
- /* -- Settings panel -- */
271
- .settings-note {
272
- font-size:0.8rem; color:#64748b; margin-top:-4px; margin-bottom:8px;
273
- }
274
- .dark .settings-note { color:#94a3b8; }
275
-
276
- /* -- Responsive -- */
277
- @media (max-width: 768px) {
278
- .app-header { padding: 20px 24px; }
279
- .header-text h1 { font-size: 1.5rem; }
280
- .header-content { flex-direction: column; align-items: flex-start; gap: 16px; }
281
- .header-meta { gap: 8px; }
282
- }
283
- """
284
-
285
-
286
- SVG_BRAIN = '<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" d="M9.813 15.904 9 18.75l-.813-2.846a4.5 4.5 0 0 0-3.09-3.09L2.25 12l2.846-.813a4.5 4.5 0 0 0 3.09-3.09L9 5.25l.813 2.846a4.5 4.5 0 0 0 3.09 3.09L15.75 12l-2.846.813a4.5 4.5 0 0 0-3.09 3.09ZM18.259 8.715 18 9.75l-.259-1.035a3.375 3.375 0 0 0-2.455-2.456L14.25 6l1.036-.259a3.375 3.375 0 0 0 2.455-2.456L18 2.25l.259 1.035a3.375 3.375 0 0 0 2.456 2.456L21.75 6l-1.035.259a3.375 3.375 0 0 0-2.456 2.456ZM16.894 20.567 16.5 21.75l-.394-1.183a2.25 2.25 0 0 0-1.423-1.423L13.5 18.75l1.183-.394a2.25 2.25 0 0 0 1.423-1.423l.394-1.183.394 1.183a2.25 2.25 0 0 0 1.423 1.423l1.183.394-1.183.394a2.25 2.25 0 0 0-1.423 1.423Z"/></svg>'
287
- SVG_DETECT = '<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" d="M7.5 3.75H6A2.25 2.25 0 0 0 3.75 6v1.5M16.5 3.75H18A2.25 2.25 0 0 1 20.25 6v1.5m0 9V18A2.25 2.25 0 0 1 18 20.25h-1.5m-9 0H6A2.25 2.25 0 0 1 3.75 18v-1.5M15 12a3 3 0 1 1-6 0 3 3 0 0 1 6 0Z"/></svg>'
288
- SVG_POINT = '<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" d="M15 10.5a3 3 0 1 1-6 0 3 3 0 0 1 6 0Z"/><path stroke-linecap="round" stroke-linejoin="round" d="M19.5 10.5c0 7.142-7.5 11.25-7.5 11.25S4.5 17.642 4.5 10.5a7.5 7.5 0 1 1 15 0Z"/></svg>'
289
- SVG_ROUTE = '<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" d="M15 10.5a3 3 0 1 1-6 0 3 3 0 0 1 6 0Z"/><path stroke-linecap="round" stroke-linejoin="round" d="M19.5 10.5c0 7.142-7.5 11.25-7.5 11.25S4.5 17.642 4.5 10.5a7.5 7.5 0 1 1 15 0Z"/></svg>'
290
- SVG_CHIP = '<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" d="M8.25 3v1.5M4.5 8.25H3m18 0h-1.5M4.5 12H3m18 0h-1.5m-15 3.75H3m18 0h-1.5M8.25 19.5V21M12 3v1.5m0 15V21m3.75-18v1.5m0 15V21m-9-1.5h10.5a2.25 2.25 0 0 0 2.25-2.25V6.75a2.25 2.25 0 0 0-2.25-2.25H6.75A2.25 2.25 0 0 0 4.5 6.75v10.5a2.25 2.25 0 0 0 2.25 2.25Z"/></svg>'
291
- SVG_GEAR = '<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" d="M9.594 3.94c.09-.542.56-.94 1.11-.94h2.593c.55 0 1.02.398 1.11.94l.213 1.281c.063.374.313.686.645.87.074.04.147.083.22.127.324.196.72.257 1.075.124l1.217-.456a1.125 1.125 0 0 1 1.37.49l1.296 2.247a1.125 1.125 0 0 1-.26 1.431l-1.003.827c-.293.24-.438.613-.431.992a6.759 6.759 0 0 1 0 .255c-.007.378.138.75.43.99l1.005.828c.424.35.534.954.26 1.43l-1.298 2.247a1.125 1.125 0 0 1-1.369.491l-1.217-.456c-.355-.133-.75-.072-1.076.124a6.57 6.57 0 0 1-.22.128c-.331.183-.581.495-.644.869l-.213 1.281c-.09.543-.56.94-1.11.94h-2.594c-.55 0-1.019-.398-1.11-.94l-.213-1.281c-.062-.374-.312-.686-.644-.87a6.52 6.52 0 0 1-.22-.127c-.325-.196-.72-.257-1.076-.124l-1.217.456a1.125 1.125 0 0 1-1.369-.49l-1.297-2.247a1.125 1.125 0 0 1 .26-1.431l1.004-.827c.292-.24.437-.613.43-.992a6.932 6.932 0 0 1 0-.255c.007-.378-.138-.75-.43-.99l-1.004-.828a1.125 1.125 0 0 1-.26-1.43l1.297-2.247a1.125 1.125 0 0 1 1.37-.491l1.216.456c.356.133.751.072 1.076-.124.072-.044.146-.087.22-.128.332-.183.582-.495.644-.869l.214-1.28Z"/><path stroke-linecap="round" stroke-linejoin="round" d="M15 12a3 3 0 1 1-6 0 3 3 0 0 1 6 0Z"/></svg>'
292
-
293
  DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
294
- DTYPE = (
295
- torch.bfloat16
296
- if torch.cuda.is_available() and torch.cuda.is_bf16_supported()
297
- else torch.float16
298
- )
299
-
300
- # ------------------------------------------------------------------
301
- # Model — swapped to Qwen3.8-27B (same Qwen3_5ForConditionalGeneration arch)
302
- # ------------------------------------------------------------------
303
  MODEL_NAME = "Qwen/Qwen3.8-27B"
304
 
305
- # Fixed GPU duration steps (seconds) — index-driven slider
306
- GPU_DURATIONS = [60, 90, 120, 150, 180, 250, 300]
307
- DEFAULT_GPU_DURATION_IDX = 1
308
-
309
- # ------------------------------------------------------------------
310
- # Example galleries — 4 examples per category, images 1.png -> 4.png
311
- # Drop matching files into an "examples/" folder next to this script.
312
- # ------------------------------------------------------------------
313
-
314
- # (image, prompt) — Object Detection tab
315
- DETECT_EXAMPLES = [
316
- ["examples/1.jpg", "Detect the yellow car that is parked."],
317
- ]
318
-
319
- # (image, prompt) — Point Localization tab
320
- POINT_EXAMPLES = [
321
- ["examples/2.jpg", "Point to all the red cars."],
322
- ]
323
-
324
- # (image, prompt) — Spatial Guidance tab
325
- SPATIAL_EXAMPLES = [
326
- ["examples/3.jpg", "Map a path from the door to the lamp."],
327
- ]
328
-
329
  BRIGHT_YELLOW = sv.Color(r=255, g=230, b=0)
330
  DARK_OUTLINE = sv.Color(r=40, g=40, b=40)
331
  BLACK = sv.Color(r=0, g=0, b=0)
332
  WHITE = sv.Color(r=255, g=255, b=255)
333
 
334
  # Spatial path colors
335
- SPATIAL_LINE = (46, 130, 180) # steel blue
336
- SPATIAL_DOT = (46, 130, 180)
337
  SPATIAL_RING = (255, 255, 255)
338
- SPATIAL_LABEL_BG = (30, 52, 80)
339
  SPATIAL_LABEL_TXT = (255, 255, 255)
340
- SPATIAL_ARROW = (230, 149, 0) # warm accent for direction
341
 
 
 
 
342
  print(f"Loading model: {MODEL_NAME} ...")
343
  qwen_model = Qwen3_5ForConditionalGeneration.from_pretrained(
344
  MODEL_NAME, torch_dtype=DTYPE, device_map=DEVICE, attn_implementation="kernels-community/flash-attn2@v3",
@@ -346,20 +51,61 @@ qwen_model = Qwen3_5ForConditionalGeneration.from_pretrained(
346
  qwen_processor = AutoProcessor.from_pretrained(MODEL_NAME)
347
  print("Model loaded.")
348
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
349
 
350
  # ------------------------------------------------------------------
351
- # Robust JSON Parsing & Data Extraction
352
  # ------------------------------------------------------------------
353
  def safe_parse_json(text: str):
354
- """Thoroughly extract JSON arrays/objects from raw model text."""
355
- # Remove markdown code blocks
356
  text = re.sub(r"```(json)?", "", text).strip()
357
-
358
- # Find the first JSON list or dict in the text
359
  match = re.search(r'(\[.*\]|\{.*\})', text, re.DOTALL)
360
  if match:
361
  json_str = match.group(1)
362
- # Clean up trailing commas that break JSON parsers
363
  json_str_clean = re.sub(r',\s*([}\]])', r'\1', json_str)
364
  try:
365
  return json.loads(json_str_clean)
@@ -368,8 +114,6 @@ def safe_parse_json(text: str):
368
  return ast.literal_eval(json_str_clean)
369
  except Exception:
370
  pass
371
-
372
- # Fallback to whole text
373
  text_clean = re.sub(r',\s*([}\]])', r'\1', text)
374
  try:
375
  return json.loads(text_clean)
@@ -381,26 +125,20 @@ def safe_parse_json(text: str):
381
  pass
382
  return []
383
 
384
-
385
  def _extract_point(item: dict):
386
- """Find point coordinates regardless of key naming variations."""
387
- if not isinstance(item, dict):
388
- return None
389
  for k in ["point_2d", "point", "points", "coordinate", "coordinates", "xy"]:
390
  if k in item and isinstance(item[k], (list, tuple)) and len(item[k]) == 2:
391
  return item[k]
392
  return None
393
 
394
  def _extract_bbox(item: dict):
395
- """Find bbox coordinates regardless of key naming variations."""
396
- if not isinstance(item, dict):
397
- return None
398
  for k in ["bbox_2d", "bbox", "box", "bounding_box", "xyxy"]:
399
  if k in item and isinstance(item[k], (list, tuple)) and len(item[k]) == 4:
400
  return item[k]
401
  return None
402
 
403
-
404
  def _load_font(size: int = 16):
405
  size = max(6, int(size))
406
  try:
@@ -411,33 +149,23 @@ def _load_font(size: int = 16):
411
  except (IOError, OSError):
412
  return ImageFont.load_default()
413
 
 
 
 
 
414
 
415
- # ------------------------------------------------------------------
416
- # Annotation (sizes are now configurable)
417
- # ------------------------------------------------------------------
418
- def annotate_image(
419
- image: Image.Image,
420
- result: dict,
421
- point_radius: int = 6,
422
- box_thickness: int = 2,
423
- text_scale: float = 0.5,
424
- ):
425
- """Annotate image with bounding boxes (Detect) or keypoints (Point)."""
426
- if not isinstance(image, Image.Image) or not isinstance(result, dict):
427
- return image
428
  image = image.convert("RGB")
429
  ow, oh = image.size
430
-
431
  point_radius = max(1, int(point_radius))
432
  box_thickness = max(1, int(box_thickness))
433
  text_scale = max(0.1, float(text_scale))
434
  text_thickness = max(1, round(text_scale * 2))
435
 
436
- # --- Point localization ---
437
  if "points" in result and result["points"]:
438
  pts = [[int(p["x"] * ow), int(p["y"] * oh)] for p in result["points"]]
439
- if not pts:
440
- return image
441
  kp = sv.KeyPoints(xy=np.array(pts).reshape(1, -1, 2))
442
  scene = np.array(image.copy())
443
  scene = sv.VertexAnnotator(radius=point_radius + 3, color=DARK_OUTLINE).annotate(scene=scene, key_points=kp)
@@ -459,7 +187,6 @@ def annotate_image(
459
  ).annotate(scene=scene, detections=sv.Detections(xyxy=np.array(tb)), labels=vl)
460
  return Image.fromarray(scene)
461
 
462
- # --- Object detection ---
463
  if "objects" in result and result["objects"]:
464
  boxes, labels = [], []
465
  for obj in result["objects"]:
@@ -468,8 +195,7 @@ def annotate_image(
468
  obj.get("x_max", 0.0) * ow, obj.get("y_max", 0.0) * oh,
469
  ])
470
  labels.append(obj.get("label", "object"))
471
- if not boxes:
472
- return image
473
  scene = np.array(image.copy())
474
  h, w = scene.shape[:2]
475
  masks = np.zeros((len(boxes), h, w), dtype=bool)
@@ -478,8 +204,7 @@ def annotate_image(
478
  x2, y2 = min(w, int(box[2])), min(h, int(box[3]))
479
  masks[i, y1:y2, x1:x2] = True
480
  dets = sv.Detections(xyxy=np.array(boxes), mask=masks)
481
- if len(dets) == 0:
482
- return image
483
  scene = sv.MaskAnnotator(color=BRIGHT_YELLOW, opacity=0.18, color_lookup=sv.ColorLookup.INDEX).annotate(scene=scene, detections=dets)
484
  scene = sv.BoxAnnotator(color=BRIGHT_YELLOW, thickness=box_thickness, color_lookup=sv.ColorLookup.INDEX).annotate(scene=scene, detections=dets)
485
  scene = sv.LabelAnnotator(
@@ -489,61 +214,41 @@ def annotate_image(
489
  return Image.fromarray(scene)
490
  return image
491
 
492
-
493
- def annotate_spatial_path(
494
- image: Image.Image,
495
- result: dict,
496
- dot_radius: int = 6,
497
- line_width: int = 4,
498
- text_scale: float = 0.5,
499
- ):
500
- """
501
- Draw a spatial map: connecting dots (waypoints) that form a path/route
502
- between mapped objects in the image.
503
- """
504
- if not isinstance(image, Image.Image) or not isinstance(result, dict):
505
- return image
506
  image = image.convert("RGB")
507
  w, h = image.size
508
- if "points" not in result or not result["points"]:
509
- return image
510
 
511
  dot_radius = max(1, int(dot_radius))
512
  line_width = max(1, int(line_width))
513
  text_scale = max(0.1, float(text_scale))
514
 
515
  draw = ImageDraw.Draw(image, "RGBA")
516
- font_label = _load_font(16 * text_scale * 2) # text_scale ~0.5 -> 16px baseline
517
  font_num = _load_font(14 * text_scale * 2)
518
 
519
  points = result["points"]
520
  pts = [(int(p["x"] * w), int(p["y"] * h)) for p in points]
521
  labels = [p.get("label", f"P{i+1}") for i, p in enumerate(points)]
522
 
523
- scale_ratio = dot_radius / 8.0 # relative to original default size
524
 
525
- # --- 1) Draw connecting path lines (with subtle glow) ---
526
  if len(pts) >= 2:
527
- # outer soft glow
528
  for i in range(len(pts) - 1):
529
  draw.line([pts[i], pts[i+1]], fill=SPATIAL_LINE + (60,), width=line_width + 6)
530
- # main path
531
  for i in range(len(pts) - 1):
532
  draw.line([pts[i], pts[i+1]], fill=SPATIAL_LINE, width=line_width)
533
 
534
- # --- 2) Arrowheads to indicate direction ---
535
  for i in range(len(pts) - 1):
536
  x1, y1 = pts[i]
537
  x2, y2 = pts[i+1]
538
  dx, dy = x2 - x1, y2 - y1
539
  length = (dx * dx + dy * dy) ** 0.5
540
- if length < 12:
541
- continue
542
  ux, uy = dx / length, dy / length
543
- # arrow base near the destination point
544
  offset = 18 * scale_ratio
545
  bx, by = x2 - ux * offset, y2 - uy * offset
546
- # perpendicular
547
  px, py = -uy, ux
548
  aw, ah = 7 * scale_ratio, 9 * scale_ratio
549
  p1 = (bx + px * aw, by + py * aw)
@@ -551,83 +256,72 @@ def annotate_spatial_path(
551
  p3 = (bx + ux * ah, by + uy * ah)
552
  draw.polygon([p1, p2, p3], fill=SPATIAL_ARROW)
553
 
554
- # --- 3) Draw waypoint dots ---
555
  for i, (cx, cy) in enumerate(pts):
556
  halo_r = dot_radius + 8
557
  ring_r = dot_radius + 3
558
- # outer halo
559
  draw.ellipse((cx - halo_r, cy - halo_r, cx + halo_r, cy + halo_r), fill=SPATIAL_LINE + (50,))
560
- # white ring
561
- draw.ellipse((cx - ring_r, cy - ring_r, cx + ring_r, cy + ring_r),
562
- outline=SPATIAL_RING, width=max(1, round(3 * scale_ratio)))
563
- # solid inner dot
564
- draw.ellipse((cx - dot_radius, cy - dot_radius, cx + dot_radius, cy + dot_radius),
565
- fill=SPATIAL_DOT, outline=SPATIAL_DOT)
566
- # number badge
567
  num_text = str(i + 1)
568
  nbbox = draw.textbbox((0, 0), num_text, font=font_num)
569
  nw = nbbox[2] - nbbox[0]
570
  nh = nbbox[3] - nbbox[1]
571
  nx, ny = cx - nw // 2, cy - nh // 2 - 1
572
- # tiny number on the dot
573
  draw.text((nx, ny), num_text, fill=SPATIAL_RING, font=font_num)
574
 
575
- # --- 4) Labels next to each waypoint ---
576
  for i, (cx, cy) in enumerate(pts):
577
  label = f"{i+1}. {labels[i]}"
578
  bbox = draw.textbbox((0, 0), label, font=font_label)
579
  tw, th = bbox[2] - bbox[0], bbox[3] - bbox[1]
580
  lx, ly = cx + dot_radius + 10, cy - th - 8
581
- # background pill
582
  pad = 5
583
- draw.rectangle(
584
- (lx - pad, ly - pad, lx + tw + pad, ly + th + pad),
585
- fill=SPATIAL_LABEL_BG,
586
- outline=SPATIAL_LINE, width=1,
587
- )
588
  draw.text((lx, ly), label, fill=SPATIAL_LABEL_TXT, font=font_label)
589
 
590
- # --- 5) Legend / summary footer ---
591
  n_pts = len(pts)
592
  legend_text = f"Spatial map · {n_pts} waypoints · path length {len(pts)-1} segments"
593
  legend_font = _load_font(13 * text_scale * 2)
594
  bbox = draw.textbbox((0, 0), legend_text, font=legend_font)
595
  tw, th = bbox[2] - bbox[0], bbox[3] - bbox[1]
596
  fx, fy = 10, h - th - 22
597
- draw.rectangle((fx - 8, fy - 6, fx + tw + 16, fy + th + 10),
598
- fill=SPATIAL_LABEL_BG + (220,))
599
  draw.text((fx, fy), legend_text, fill=SPATIAL_LABEL_TXT, font=legend_font)
600
 
601
  return image
602
 
603
-
604
- # ------------------------------------------------------------------
605
- # Dynamic GPU duration
606
- # ------------------------------------------------------------------
607
- def get_gpu_duration(image, category, prompt, point_radius, box_thickness, text_scale, gpu_duration_seconds):
608
- """
609
- ZeroGPU allows the `duration` argument of @spaces.GPU to be a callable
610
- that receives the same arguments as the decorated function and returns
611
- the number of seconds to request for that specific call. This lets the
612
- GPU Duration slider in the UI directly control the allocation per run.
613
- """
614
- try:
615
- return int(gpu_duration_seconds)
616
- except (TypeError, ValueError):
617
- return GPU_DURATIONS[DEFAULT_GPU_DURATION_IDX]
618
-
619
-
620
  # ------------------------------------------------------------------
621
- # Main pipeline
622
  # ------------------------------------------------------------------
623
- @spaces.GPU(size="xlarge", duration=get_gpu_duration)
624
- def process_inputs(image, category, prompt, point_radius, box_thickness, text_scale, gpu_duration_seconds):
625
- if image is None:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
626
  raise gr.Error("Please upload an image.")
627
- if not prompt or not prompt.strip():
628
  raise gr.Error("Please provide a prompt.")
629
- image = image.convert("RGB")
630
- image.thumbnail((512, 512))
 
 
 
 
 
 
 
631
 
632
  if category == "Detect":
633
  full_prompt = (
@@ -655,7 +349,7 @@ def process_inputs(image, category, prompt, point_radius, box_thickness, text_sc
655
  messages = [{
656
  "role": "user",
657
  "content": [
658
- {"type": "image", "image": image},
659
  {"type": "text", "text": full_prompt},
660
  ],
661
  }]
@@ -663,7 +357,7 @@ def process_inputs(image, category, prompt, point_radius, box_thickness, text_sc
663
  messages, tokenize=False, add_generation_prompt=True
664
  )
665
  inputs = qwen_processor(
666
- text=[text], images=[image],
667
  return_tensors="pt", padding=True,
668
  ).to(qwen_model.device)
669
 
@@ -678,35 +372,29 @@ def process_inputs(image, category, prompt, point_radius, box_thickness, text_sc
678
  streamer=streamer,
679
  max_new_tokens=2048,
680
  use_cache=True,
681
- do_sample=False, # Deterministic for reliable JSON
682
  ),
683
  )
684
  thread.start()
685
  full_text = ""
686
  for tok in streamer:
687
  full_text += tok
688
- yield image, full_text
689
  thread.join()
690
 
691
- # ----- Post-process for each category -----
 
 
692
  if category == "Point":
693
  parsed = safe_parse_json(full_text)
694
- # Handle dict containing the list
695
  if isinstance(parsed, dict):
696
- list_found = False
697
  for k in ["points", "keypoints", "point"]:
698
  if k in parsed and isinstance(parsed[k], list):
699
  parsed = parsed[k]
700
- list_found = True
701
  break
702
- if not list_found:
703
- # fallback to first list value found
704
  for v in parsed.values():
705
- if isinstance(v, list):
706
- parsed = v
707
- break
708
- else:
709
- parsed = []
710
 
711
  result = {"points": []}
712
  if isinstance(parsed, list):
@@ -714,36 +402,25 @@ def process_inputs(image, category, prompt, point_radius, box_thickness, text_sc
714
  pt = _extract_point(item)
715
  if pt:
716
  x, y = pt
717
- result["points"].append({
718
- "label": item.get("label", ""),
719
- "x": x / 1000.0, "y": y / 1000.0,
720
- })
721
 
722
  if result["points"]:
723
- yield annotate_image(
724
- image.copy(), result,
725
- point_radius=point_radius, box_thickness=box_thickness, text_scale=text_scale,
726
- ), json.dumps(result, indent=2)
727
  else:
728
- yield image.copy(), f"Could not extract any points.\nRaw model output:\n{full_text}"
729
 
730
  elif category == "Detect":
731
  parsed = safe_parse_json(full_text)
732
- # Handle dict containing the list
733
  if isinstance(parsed, dict):
734
- list_found = False
735
  for k in ["objects", "detections", "bboxes", "boxes", "results"]:
736
  if k in parsed and isinstance(parsed[k], list):
737
  parsed = parsed[k]
738
- list_found = True
739
  break
740
- if not list_found:
741
  for v in parsed.values():
742
- if isinstance(v, list):
743
- parsed = v
744
- break
745
- else:
746
- parsed = []
747
 
748
  result = {"objects": []}
749
  if isinstance(parsed, list):
@@ -758,30 +435,22 @@ def process_inputs(image, category, prompt, point_radius, box_thickness, text_sc
758
  })
759
 
760
  if result["objects"]:
761
- yield annotate_image(
762
- image.copy(), result,
763
- point_radius=point_radius, box_thickness=box_thickness, text_scale=text_scale,
764
- ), json.dumps(result, indent=2)
765
  else:
766
- yield image.copy(), f"Could not extract any objects.\nRaw model output:\n{full_text}"
767
 
768
  elif category == "Spatial":
769
  parsed = safe_parse_json(full_text)
770
- # Handle dict containing the list
771
  if isinstance(parsed, dict):
772
- list_found = False
773
  for k in ["points", "waypoints", "path", "route", "nodes", "map"]:
774
  if k in parsed and isinstance(parsed[k], list):
775
  parsed = parsed[k]
776
- list_found = True
777
  break
778
- if not list_found:
779
  for v in parsed.values():
780
- if isinstance(v, list):
781
- parsed = v
782
- break
783
- else:
784
- parsed = []
785
 
786
  result = {"points": []}
787
  if isinstance(parsed, list):
@@ -789,293 +458,48 @@ def process_inputs(image, category, prompt, point_radius, box_thickness, text_sc
789
  pt = _extract_point(item)
790
  if pt:
791
  x, y = pt
792
- result["points"].append({
793
- "label": item.get("label", "waypoint"),
794
- "x": x / 1000.0, "y": y / 1000.0,
795
- })
796
 
797
  if result["points"]:
798
- wp_lines = "\n".join(
799
- f" {i+1}. {p['label']} → ({p['x']:.3f}, {p['y']:.3f})"
800
- for i, p in enumerate(result["points"])
801
- )
802
- summary = (
803
  f"Spatial map generated.\n"
804
  f"Waypoints ({len(result['points'])}):\n{wp_lines}\n"
805
  f"Path segments: {max(0, len(result['points']) - 1)}"
806
  )
807
- yield annotate_spatial_path(
808
- image.copy(), result,
809
- dot_radius=point_radius, line_width=box_thickness * 2, text_scale=text_scale,
810
- ), summary
811
  else:
812
- yield image.copy(), f"Could not extract any spatial waypoints.\nRaw model output:\n{full_text}"
813
-
814
-
815
- def on_gpu_duration_change(idx):
816
- seconds = GPU_DURATIONS[int(idx)]
817
- return f"**GPU budget:** {seconds}s requested per run", seconds
818
-
819
-
820
- # ------------------------------------------------------------------
821
- # HTML building blocks
822
- # ------------------------------------------------------------------
823
- def html_header():
824
- return f"""
825
- <div class="app-header">
826
- <div class="header-content">
827
- <div class="header-icon-wrap">{SVG_BRAIN}</div>
828
- <div class="header-text">
829
- <h1>Qwen3.8 27B &mdash; Object-Detection</h1>
830
- <div class="header-meta">
831
- <span class="meta-badge">{SVG_CHIP} {MODEL_NAME}</span>
832
- <span class="meta-sep"></span>
833
- <span class="meta-cap">Object Detection</span>
834
- <span class="meta-sep"></span>
835
- <span class="meta-cap">Point Localization</span>
836
- <span class="meta-sep"></span>
837
- <span class="meta-cap">Spatial Guidance</span>
838
- </div>
839
- </div>
840
- </div>
841
- </div>
842
- """
843
-
844
 
845
- def html_tab_intro(icon_svg, title, description, detail=""):
846
- sub = f'<p class="intro-sub">{detail}</p>' if detail else ""
847
- return f"""
848
- <div class="tab-intro">
849
- <div class="intro-icon">{icon_svg}</div>
850
- <div class="intro-text">
851
- <p><strong>{title}</strong> &mdash; {description}</p>
852
- {sub}
853
- </div>
854
- </div>
855
- """
856
-
857
-
858
- def html_status_indicator(text):
859
- return f"""
860
- <div class="status-indicator">
861
- <span class="status-dot"></span>
862
- <span class="status-text">{text}</span>
863
- </div>
864
- """
865
-
866
-
867
- def html_divider():
868
- return '<div class="section-divider"></div>'
869
-
870
-
871
- # ------------------------------------------------------------------
872
- # Gradio UI
873
- # ------------------------------------------------------------------
874
- with gr.Blocks() as demo:
875
-
876
- gr.HTML(html_header())
877
-
878
- # ================================================================
879
- # Global Visualization & Performance Settings
880
- # (shared across all tabs)
881
- # ================================================================
882
- with gr.Accordion("Visualization & Performance Settings", open=False):
883
- gr.HTML(html_tab_intro(
884
- SVG_GEAR,
885
- "Settings",
886
- "Tune how bounding boxes / points / waypoints are drawn, and how much "
887
- "GPU time is requested for each run.",
888
- ))
889
- with gr.Row():
890
- point_radius_slider = gr.Slider(
891
- minimum=2, maximum=20, step=1, value=4,
892
- label="Point / Waypoint Radius (px)",
893
- info="Size of point markers and spatial waypoint dots",
894
- )
895
- box_thickness_slider = gr.Slider(
896
- minimum=1, maximum=8, step=1, value=1,
897
- label="Bounding Box Thickness (px)",
898
- info="Also scales the spatial path line width",
899
- )
900
- text_scale_slider = gr.Slider(
901
- minimum=0.2, maximum=1.5, step=0.05, value=0.4,
902
- label="Label Text Scale",
903
- info="Size of labels drawn on the image",
904
- )
905
- with gr.Row():
906
- gpu_duration_slider = gr.Slider(
907
- minimum=0, maximum=len(GPU_DURATIONS) - 1, step=1,
908
- value=DEFAULT_GPU_DURATION_IDX,
909
- label="GPU Duration",
910
- info="Steps: 60 · 90 · 120 · 150 · 180 · 250 · 300 seconds",
911
- )
912
- gpu_duration_label = gr.Markdown(
913
- f"**GPU budget:** {GPU_DURATIONS[DEFAULT_GPU_DURATION_IDX]}s requested per run"
914
- )
915
- gpu_duration_state = gr.State(GPU_DURATIONS[DEFAULT_GPU_DURATION_IDX])
916
-
917
- gpu_duration_slider.change(
918
- fn=on_gpu_duration_change,
919
- inputs=[gpu_duration_slider],
920
- outputs=[gpu_duration_label, gpu_duration_state],
921
- )
922
-
923
- with gr.Tabs():
924
-
925
- # =============================================================
926
- # Object Detection Tab
927
- # =============================================================
928
- with gr.Tab("Object Detection"):
929
- gr.HTML(html_tab_intro(
930
- SVG_DETECT,
931
- "Object Detection",
932
- "Draw bounding boxes around the objects you describe. "
933
- "Just tell the model what to find and it returns precise coordinates.",
934
- "The app overlays masks and boxes automatically.",
935
- ))
936
- with gr.Row():
937
- with gr.Column():
938
- det_img = gr.Image(type="pil", label="Upload Image", height=350)
939
- det_prompt = gr.Textbox(
940
- label="What to detect",
941
- placeholder="e.g., all cars, the people, the red sign",
942
- lines=3,
943
- )
944
- det_btn = gr.Button("Detect Objects", variant="primary")
945
- gr.HTML(html_divider())
946
- gr.Examples(
947
- examples=DETECT_EXAMPLES,
948
- inputs=[det_img, det_prompt],
949
- label="Examples",
950
- )
951
- with gr.Column(scale=2):
952
- det_out_img = gr.Image(label="Detected Objects", height=380)
953
- det_out_text = gr.Textbox(label="Detection Result", lines=10, interactive=True)
954
- gr.HTML(html_status_indicator("Bounding boxes are drawn automatically."))
955
-
956
- def _det_wrapper(image, prompt, point_radius, box_thickness, text_scale, gpu_duration_seconds):
957
- outputs = None
958
- for outputs in process_inputs(
959
- image, "Detect", prompt,
960
- point_radius, box_thickness, text_scale, gpu_duration_seconds,
961
- ):
962
- pass
963
- return outputs if outputs else (image, "")
964
-
965
- det_btn.click(
966
- fn=_det_wrapper,
967
- inputs=[
968
- det_img, det_prompt,
969
- point_radius_slider, box_thickness_slider, text_scale_slider,
970
- gpu_duration_state,
971
- ],
972
- outputs=[det_out_img, det_out_text],
973
- )
974
-
975
- # =============================================================
976
- # Point Localization Tab
977
- # =============================================================
978
- with gr.Tab("Point Localization"):
979
- gr.HTML(html_tab_intro(
980
- SVG_POINT,
981
- "Point Localization",
982
- "Locate specific points of interest in the image — the tip of an object, "
983
- "a particular feature, or anything you can describe.",
984
- "Returns precise 2D coordinates highlighted on the image.",
985
- ))
986
- with gr.Row():
987
- with gr.Column():
988
- pt_img = gr.Image(type="pil", label="Upload Image", height=350)
989
- pt_prompt = gr.Textbox(
990
- label="What to locate",
991
- placeholder="e.g., the tip of the pen / the gun held by the person",
992
- lines=3,
993
- )
994
- pt_btn = gr.Button("Locate Point", variant="primary")
995
- gr.HTML(html_divider())
996
- gr.Examples(
997
- examples=POINT_EXAMPLES,
998
- inputs=[pt_img, pt_prompt],
999
- label="Examples",
1000
- )
1001
- with gr.Column(scale=2):
1002
- pt_out_img = gr.Image(label="Localized Point", height=380)
1003
- pt_out_text = gr.Textbox(label="Localization Result", lines=10, interactive=True)
1004
- gr.HTML(html_status_indicator("Points are highlighted with bright markers."))
1005
-
1006
- def _pt_wrapper(image, prompt, point_radius, box_thickness, text_scale, gpu_duration_seconds):
1007
- outputs = None
1008
- for outputs in process_inputs(
1009
- image, "Point", prompt,
1010
- point_radius, box_thickness, text_scale, gpu_duration_seconds,
1011
- ):
1012
- pass
1013
- return outputs if outputs else (image, "")
1014
-
1015
- pt_btn.click(
1016
- fn=_pt_wrapper,
1017
- inputs=[
1018
- pt_img, pt_prompt,
1019
- point_radius_slider, box_thickness_slider, text_scale_slider,
1020
- gpu_duration_state,
1021
- ],
1022
- outputs=[pt_out_img, pt_out_text],
1023
- )
1024
-
1025
- # =============================================================
1026
- # Spatial Guidance Tab
1027
- # =============================================================
1028
- with gr.Tab("Spatial Guidance"):
1029
- gr.HTML(html_tab_intro(
1030
- SVG_ROUTE,
1031
- "Spatial Guidance",
1032
- "Map a path or spatial route between objects in the image. "
1033
- "The model identifies key waypoints, and the app draws connecting dots, "
1034
- "directional arrows, and labels to produce a spatial map.",
1035
- "Use cases: navigation paths, object-to-object routes, spatial relationships.",
1036
- ))
1037
- with gr.Row():
1038
- with gr.Column():
1039
- sp_img = gr.Image(type="pil", label="Upload Image", height=350)
1040
- sp_prompt = gr.Textbox(
1041
- label="Describe the path / spatial map",
1042
- placeholder="e.g., Map a path from the door to the window / "
1043
- "spatial route between the chair and the desk",
1044
- lines=3,
1045
- )
1046
- sp_btn = gr.Button("Generate Spatial Map", variant="primary")
1047
- gr.HTML(html_divider())
1048
- gr.Examples(
1049
- examples=SPATIAL_EXAMPLES,
1050
- inputs=[sp_img, sp_prompt],
1051
- label="Examples",
1052
- )
1053
- with gr.Column(scale=2):
1054
- sp_out_img = gr.Image(label="Spatial Map Output", height=380)
1055
- sp_out_text = gr.Textbox(label="Spatial Summary", lines=10, interactive=True)
1056
- gr.HTML(html_status_indicator(
1057
- "Waypoints are connected with directional arrows to form a clear path."
1058
- ))
1059
-
1060
- def _sp_wrapper(image, prompt, point_radius, box_thickness, text_scale, gpu_duration_seconds):
1061
- outputs = None
1062
- for outputs in process_inputs(
1063
- image, "Spatial", prompt,
1064
- point_radius, box_thickness, text_scale, gpu_duration_seconds,
1065
- ):
1066
- pass
1067
- return outputs if outputs else (image, "")
1068
-
1069
- sp_btn.click(
1070
- fn=_sp_wrapper,
1071
- inputs=[
1072
- sp_img, sp_prompt,
1073
- point_radius_slider, box_thickness_slider, text_scale_slider,
1074
- gpu_duration_state,
1075
- ],
1076
- outputs=[sp_out_img, sp_out_text],
1077
- )
1078
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1079
 
1080
  if __name__ == "__main__":
1081
- demo.launch(css=css, theme=steel_blue_theme, show_error=True, ssr_mode=False)
 
1
+ import os
2
+ import gc
3
+ import base64
4
+ import json
5
+ import ast
6
+ import re
7
+ from io import BytesIO
8
  import torch
9
  import spaces
10
  import numpy as np
11
+ from PIL import Image, ImageDraw, ImageFont
12
  import supervision as sv
13
  from typing import Iterable
14
+ import gradio as gr
15
+ from gradio import Server
16
+ from fastapi.responses import HTMLResponse
 
 
 
17
  from threading import Thread
18
  from transformers import (
19
  Qwen3_5ForConditionalGeneration,
 
21
  TextIteratorStreamer,
22
  )
23
 
 
24
  # ------------------------------------------------------------------
25
+ # Config & Constants
26
  # ------------------------------------------------------------------
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27
  DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
28
+ DTYPE = torch.bfloat16 if torch.cuda.is_available() and torch.cuda.is_bf16_supported() else torch.float16
 
 
 
 
 
 
 
 
29
  MODEL_NAME = "Qwen/Qwen3.8-27B"
30
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
31
  BRIGHT_YELLOW = sv.Color(r=255, g=230, b=0)
32
  DARK_OUTLINE = sv.Color(r=40, g=40, b=40)
33
  BLACK = sv.Color(r=0, g=0, b=0)
34
  WHITE = sv.Color(r=255, g=255, b=255)
35
 
36
  # Spatial path colors
37
+ SPATIAL_LINE = (255, 69, 0) # OrangeRed
38
+ SPATIAL_DOT = (255, 69, 0)
39
  SPATIAL_RING = (255, 255, 255)
40
+ SPATIAL_LABEL_BG = (80, 20, 0)
41
  SPATIAL_LABEL_TXT = (255, 255, 255)
42
+ SPATIAL_ARROW = (255, 165, 0) # Orange for direction
43
 
44
+ # ------------------------------------------------------------------
45
+ # Model Loading
46
+ # ------------------------------------------------------------------
47
  print(f"Loading model: {MODEL_NAME} ...")
48
  qwen_model = Qwen3_5ForConditionalGeneration.from_pretrained(
49
  MODEL_NAME, torch_dtype=DTYPE, device_map=DEVICE, attn_implementation="kernels-community/flash-attn2@v3",
 
51
  qwen_processor = AutoProcessor.from_pretrained(MODEL_NAME)
52
  print("Model loaded.")
53
 
54
+ # ------------------------------------------------------------------
55
+ # Examples Config
56
+ # ------------------------------------------------------------------
57
+ EXAMPLES_CONFIG = [
58
+ {"image": "examples/1.jpg", "prompt": "Detect the yellow car that is parked.", "mode": "Detect"},
59
+ {"image": "examples/2.jpg", "prompt": "Point to all the red cars.", "mode": "Point"},
60
+ {"image": "examples/3.jpg", "prompt": "Map a path from the door to the lamp.", "mode": "Spatial"},
61
+ ]
62
+
63
+ def make_thumb_b64(path, max_dim=220):
64
+ if not os.path.exists(path):
65
+ return ""
66
+ try:
67
+ img = Image.open(path).convert("RGB")
68
+ img.thumbnail((max_dim, max_dim), Image.LANCZOS)
69
+ buf = BytesIO()
70
+ img.save(buf, format="JPEG", quality=65)
71
+ return f"data:image/jpeg;base64,{base64.b64encode(buf.getvalue()).decode()}"
72
+ except Exception as e:
73
+ return ""
74
+
75
+ def encode_full_image(path):
76
+ if not os.path.exists(path):
77
+ return ""
78
+ try:
79
+ with open(path, "rb") as f:
80
+ data = f.read()
81
+ ext = path.rsplit(".", 1)[-1].lower()
82
+ mime = {"jpg": "image/jpeg", "jpeg": "image/jpeg", "png": "image/png", "webp": "image/webp"}.get(ext, "image/jpeg")
83
+ return f"data:{mime};base64,{base64.b64encode(data).decode()}"
84
+ except Exception as e:
85
+ return ""
86
+
87
+ def build_client_config():
88
+ examples = []
89
+ for i, ex in enumerate(EXAMPLES_CONFIG):
90
+ examples.append({
91
+ "idx": i,
92
+ "thumb": make_thumb_b64(ex["image"]),
93
+ "prompt": ex["prompt"],
94
+ "mode": ex["mode"],
95
+ })
96
+ return {"examples": examples, "modes": ["Detect", "Point", "Spatial"], "default_mode": "Detect"}
97
+
98
+ print("Building client config…")
99
+ CLIENT_CONFIG = build_client_config()
100
 
101
  # ------------------------------------------------------------------
102
+ # Helpers
103
  # ------------------------------------------------------------------
104
  def safe_parse_json(text: str):
 
 
105
  text = re.sub(r"```(json)?", "", text).strip()
 
 
106
  match = re.search(r'(\[.*\]|\{.*\})', text, re.DOTALL)
107
  if match:
108
  json_str = match.group(1)
 
109
  json_str_clean = re.sub(r',\s*([}\]])', r'\1', json_str)
110
  try:
111
  return json.loads(json_str_clean)
 
114
  return ast.literal_eval(json_str_clean)
115
  except Exception:
116
  pass
 
 
117
  text_clean = re.sub(r',\s*([}\]])', r'\1', text)
118
  try:
119
  return json.loads(text_clean)
 
125
  pass
126
  return []
127
 
 
128
  def _extract_point(item: dict):
129
+ if not isinstance(item, dict): return None
 
 
130
  for k in ["point_2d", "point", "points", "coordinate", "coordinates", "xy"]:
131
  if k in item and isinstance(item[k], (list, tuple)) and len(item[k]) == 2:
132
  return item[k]
133
  return None
134
 
135
  def _extract_bbox(item: dict):
136
+ if not isinstance(item, dict): return None
 
 
137
  for k in ["bbox_2d", "bbox", "box", "bounding_box", "xyxy"]:
138
  if k in item and isinstance(item[k], (list, tuple)) and len(item[k]) == 4:
139
  return item[k]
140
  return None
141
 
 
142
  def _load_font(size: int = 16):
143
  size = max(6, int(size))
144
  try:
 
149
  except (IOError, OSError):
150
  return ImageFont.load_default()
151
 
152
+ def pil_to_b64_png(image: Image.Image) -> str:
153
+ buf = BytesIO()
154
+ image.save(buf, format="PNG")
155
+ return f"data:image/png;base64,{base64.b64encode(buf.getvalue()).decode()}"
156
 
157
+ def annotate_image(image: Image.Image, result: dict, point_radius: int = 6, box_thickness: int = 2, text_scale: float = 0.5):
158
+ if not isinstance(image, Image.Image) or not isinstance(result, dict): return image
 
 
 
 
 
 
 
 
 
 
 
159
  image = image.convert("RGB")
160
  ow, oh = image.size
 
161
  point_radius = max(1, int(point_radius))
162
  box_thickness = max(1, int(box_thickness))
163
  text_scale = max(0.1, float(text_scale))
164
  text_thickness = max(1, round(text_scale * 2))
165
 
 
166
  if "points" in result and result["points"]:
167
  pts = [[int(p["x"] * ow), int(p["y"] * oh)] for p in result["points"]]
168
+ if not pts: return image
 
169
  kp = sv.KeyPoints(xy=np.array(pts).reshape(1, -1, 2))
170
  scene = np.array(image.copy())
171
  scene = sv.VertexAnnotator(radius=point_radius + 3, color=DARK_OUTLINE).annotate(scene=scene, key_points=kp)
 
187
  ).annotate(scene=scene, detections=sv.Detections(xyxy=np.array(tb)), labels=vl)
188
  return Image.fromarray(scene)
189
 
 
190
  if "objects" in result and result["objects"]:
191
  boxes, labels = [], []
192
  for obj in result["objects"]:
 
195
  obj.get("x_max", 0.0) * ow, obj.get("y_max", 0.0) * oh,
196
  ])
197
  labels.append(obj.get("label", "object"))
198
+ if not boxes: return image
 
199
  scene = np.array(image.copy())
200
  h, w = scene.shape[:2]
201
  masks = np.zeros((len(boxes), h, w), dtype=bool)
 
204
  x2, y2 = min(w, int(box[2])), min(h, int(box[3]))
205
  masks[i, y1:y2, x1:x2] = True
206
  dets = sv.Detections(xyxy=np.array(boxes), mask=masks)
207
+ if len(dets) == 0: return image
 
208
  scene = sv.MaskAnnotator(color=BRIGHT_YELLOW, opacity=0.18, color_lookup=sv.ColorLookup.INDEX).annotate(scene=scene, detections=dets)
209
  scene = sv.BoxAnnotator(color=BRIGHT_YELLOW, thickness=box_thickness, color_lookup=sv.ColorLookup.INDEX).annotate(scene=scene, detections=dets)
210
  scene = sv.LabelAnnotator(
 
214
  return Image.fromarray(scene)
215
  return image
216
 
217
+ def annotate_spatial_path(image: Image.Image, result: dict, dot_radius: int = 6, line_width: int = 4, text_scale: float = 0.5):
218
+ if not isinstance(image, Image.Image) or not isinstance(result, dict): return image
 
 
 
 
 
 
 
 
 
 
 
 
219
  image = image.convert("RGB")
220
  w, h = image.size
221
+ if "points" not in result or not result["points"]: return image
 
222
 
223
  dot_radius = max(1, int(dot_radius))
224
  line_width = max(1, int(line_width))
225
  text_scale = max(0.1, float(text_scale))
226
 
227
  draw = ImageDraw.Draw(image, "RGBA")
228
+ font_label = _load_font(16 * text_scale * 2)
229
  font_num = _load_font(14 * text_scale * 2)
230
 
231
  points = result["points"]
232
  pts = [(int(p["x"] * w), int(p["y"] * h)) for p in points]
233
  labels = [p.get("label", f"P{i+1}") for i, p in enumerate(points)]
234
 
235
+ scale_ratio = dot_radius / 8.0
236
 
 
237
  if len(pts) >= 2:
 
238
  for i in range(len(pts) - 1):
239
  draw.line([pts[i], pts[i+1]], fill=SPATIAL_LINE + (60,), width=line_width + 6)
 
240
  for i in range(len(pts) - 1):
241
  draw.line([pts[i], pts[i+1]], fill=SPATIAL_LINE, width=line_width)
242
 
 
243
  for i in range(len(pts) - 1):
244
  x1, y1 = pts[i]
245
  x2, y2 = pts[i+1]
246
  dx, dy = x2 - x1, y2 - y1
247
  length = (dx * dx + dy * dy) ** 0.5
248
+ if length < 12: continue
 
249
  ux, uy = dx / length, dy / length
 
250
  offset = 18 * scale_ratio
251
  bx, by = x2 - ux * offset, y2 - uy * offset
 
252
  px, py = -uy, ux
253
  aw, ah = 7 * scale_ratio, 9 * scale_ratio
254
  p1 = (bx + px * aw, by + py * aw)
 
256
  p3 = (bx + ux * ah, by + uy * ah)
257
  draw.polygon([p1, p2, p3], fill=SPATIAL_ARROW)
258
 
 
259
  for i, (cx, cy) in enumerate(pts):
260
  halo_r = dot_radius + 8
261
  ring_r = dot_radius + 3
 
262
  draw.ellipse((cx - halo_r, cy - halo_r, cx + halo_r, cy + halo_r), fill=SPATIAL_LINE + (50,))
263
+ draw.ellipse((cx - ring_r, cy - ring_r, cx + ring_r, cy + ring_r), outline=SPATIAL_RING, width=max(1, round(3 * scale_ratio)))
264
+ draw.ellipse((cx - dot_radius, cy - dot_radius, cx + dot_radius, cy + dot_radius), fill=SPATIAL_DOT, outline=SPATIAL_DOT)
 
 
 
 
 
265
  num_text = str(i + 1)
266
  nbbox = draw.textbbox((0, 0), num_text, font=font_num)
267
  nw = nbbox[2] - nbbox[0]
268
  nh = nbbox[3] - nbbox[1]
269
  nx, ny = cx - nw // 2, cy - nh // 2 - 1
 
270
  draw.text((nx, ny), num_text, fill=SPATIAL_RING, font=font_num)
271
 
 
272
  for i, (cx, cy) in enumerate(pts):
273
  label = f"{i+1}. {labels[i]}"
274
  bbox = draw.textbbox((0, 0), label, font=font_label)
275
  tw, th = bbox[2] - bbox[0], bbox[3] - bbox[1]
276
  lx, ly = cx + dot_radius + 10, cy - th - 8
 
277
  pad = 5
278
+ draw.rectangle((lx - pad, ly - pad, lx + tw + pad, ly + th + pad), fill=SPATIAL_LABEL_BG, outline=SPATIAL_LINE, width=1)
 
 
 
 
279
  draw.text((lx, ly), label, fill=SPATIAL_LABEL_TXT, font=font_label)
280
 
 
281
  n_pts = len(pts)
282
  legend_text = f"Spatial map · {n_pts} waypoints · path length {len(pts)-1} segments"
283
  legend_font = _load_font(13 * text_scale * 2)
284
  bbox = draw.textbbox((0, 0), legend_text, font=legend_font)
285
  tw, th = bbox[2] - bbox[0], bbox[3] - bbox[1]
286
  fx, fy = 10, h - th - 22
287
+ draw.rectangle((fx - 8, fy - 6, fx + tw + 16, fy + th + 10), fill=SPATIAL_LABEL_BG + (220,))
 
288
  draw.text((fx, fy), legend_text, fill=SPATIAL_LABEL_TXT, font=legend_font)
289
 
290
  return image
291
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
292
  # ------------------------------------------------------------------
293
+ # Gradio Server (Server mode): FastAPI + Gradio queue/API engine
294
  # ------------------------------------------------------------------
295
+ app = Server(title="Qwen3.8-27B-Object-Detection")
296
+
297
+ @app.mcp.tool(name="run_inference")
298
+ @app.api(name="run_inference")
299
+ @spaces.GPU(size="xlarge", duration=90)
300
+ def infer(
301
+ image_b64: str,
302
+ mode: str,
303
+ prompt: str,
304
+ point_radius: int,
305
+ box_thickness: int,
306
+ text_scale: float,
307
+ ) -> dict:
308
+ """Runs object detection, point localization, or spatial mapping."""
309
+ gc.collect()
310
+ torch.cuda.empty_cache()
311
+
312
+ if not image_b64:
313
  raise gr.Error("Please upload an image.")
314
+ if not prompt or prompt.strip() == "":
315
  raise gr.Error("Please provide a prompt.")
316
+
317
+ try:
318
+ header, data = image_b64.split(",", 1)
319
+ pil_image = Image.open(BytesIO(base64.b64decode(data))).convert("RGB")
320
+ except Exception as e:
321
+ raise gr.Error(f"Invalid image data: {e}")
322
+
323
+ pil_image.thumbnail((512, 512))
324
+ category = mode
325
 
326
  if category == "Detect":
327
  full_prompt = (
 
349
  messages = [{
350
  "role": "user",
351
  "content": [
352
+ {"type": "image", "image": pil_image},
353
  {"type": "text", "text": full_prompt},
354
  ],
355
  }]
 
357
  messages, tokenize=False, add_generation_prompt=True
358
  )
359
  inputs = qwen_processor(
360
+ text=[text], images=[pil_image],
361
  return_tensors="pt", padding=True,
362
  ).to(qwen_model.device)
363
 
 
372
  streamer=streamer,
373
  max_new_tokens=2048,
374
  use_cache=True,
375
+ do_sample=False,
376
  ),
377
  )
378
  thread.start()
379
  full_text = ""
380
  for tok in streamer:
381
  full_text += tok
 
382
  thread.join()
383
 
384
+ result_text = full_text
385
+ result_image = pil_image.copy()
386
+
387
  if category == "Point":
388
  parsed = safe_parse_json(full_text)
 
389
  if isinstance(parsed, dict):
 
390
  for k in ["points", "keypoints", "point"]:
391
  if k in parsed and isinstance(parsed[k], list):
392
  parsed = parsed[k]
 
393
  break
394
+ else:
 
395
  for v in parsed.values():
396
+ if isinstance(v, list): parsed = v; break
397
+ else: parsed = []
 
 
 
398
 
399
  result = {"points": []}
400
  if isinstance(parsed, list):
 
402
  pt = _extract_point(item)
403
  if pt:
404
  x, y = pt
405
+ result["points"].append({"label": item.get("label", ""), "x": x / 1000.0, "y": y / 1000.0})
 
 
 
406
 
407
  if result["points"]:
408
+ result_image = annotate_image(pil_image.copy(), result, point_radius, box_thickness, text_scale)
409
+ result_text = json.dumps(result, indent=2)
 
 
410
  else:
411
+ result_text = f"Could not extract any points.\nRaw model output:\n{full_text}"
412
 
413
  elif category == "Detect":
414
  parsed = safe_parse_json(full_text)
 
415
  if isinstance(parsed, dict):
 
416
  for k in ["objects", "detections", "bboxes", "boxes", "results"]:
417
  if k in parsed and isinstance(parsed[k], list):
418
  parsed = parsed[k]
 
419
  break
420
+ else:
421
  for v in parsed.values():
422
+ if isinstance(v, list): parsed = v; break
423
+ else: parsed = []
 
 
 
424
 
425
  result = {"objects": []}
426
  if isinstance(parsed, list):
 
435
  })
436
 
437
  if result["objects"]:
438
+ result_image = annotate_image(pil_image.copy(), result, point_radius, box_thickness, text_scale)
439
+ result_text = json.dumps(result, indent=2)
 
 
440
  else:
441
+ result_text = f"Could not extract any objects.\nRaw model output:\n{full_text}"
442
 
443
  elif category == "Spatial":
444
  parsed = safe_parse_json(full_text)
 
445
  if isinstance(parsed, dict):
 
446
  for k in ["points", "waypoints", "path", "route", "nodes", "map"]:
447
  if k in parsed and isinstance(parsed[k], list):
448
  parsed = parsed[k]
 
449
  break
450
+ else:
451
  for v in parsed.values():
452
+ if isinstance(v, list): parsed = v; break
453
+ else: parsed = []
 
 
 
454
 
455
  result = {"points": []}
456
  if isinstance(parsed, list):
 
458
  pt = _extract_point(item)
459
  if pt:
460
  x, y = pt
461
+ result["points"].append({"label": item.get("label", "waypoint"), "x": x / 1000.0, "y": y / 1000.0})
 
 
 
462
 
463
  if result["points"]:
464
+ wp_lines = "\n".join(f" {i+1}. {p['label']} → ({p['x']:.3f}, {p['y']:.3f})" for i, p in enumerate(result["points"]))
465
+ result_text = (
 
 
 
466
  f"Spatial map generated.\n"
467
  f"Waypoints ({len(result['points'])}):\n{wp_lines}\n"
468
  f"Path segments: {max(0, len(result['points']) - 1)}"
469
  )
470
+ result_image = annotate_spatial_path(pil_image.copy(), result, point_radius, box_thickness * 2, text_scale)
 
 
 
471
  else:
472
+ result_text = f"Could not extract any spatial waypoints.\nRaw model output:\n{full_text}"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
473
 
474
+ return {"image": pil_to_b64_png(result_image), "text": result_text}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
475
 
476
+ @app.api(name="load_example", queue=False)
477
+ def load_example(idx: float) -> dict:
478
+ try:
479
+ i = int(idx)
480
+ except (ValueError, TypeError):
481
+ i = -1
482
+ if i < 0 or i >= len(EXAMPLES_CONFIG):
483
+ return {"image": "", "prompt": "", "mode": "Detect", "name": "", "status": "error"}
484
+ ex = EXAMPLES_CONFIG[i]
485
+ b64 = encode_full_image(ex["image"])
486
+ return {
487
+ "image": b64,
488
+ "prompt": ex["prompt"],
489
+ "mode": ex["mode"],
490
+ "name": os.path.basename(ex["image"]),
491
+ "status": "ok" if b64 else "error"
492
+ }
493
+
494
+ @app.get("/api/config")
495
+ def client_config():
496
+ return CLIENT_CONFIG
497
+
498
+ @app.get("/", response_class=HTMLResponse)
499
+ async def homepage():
500
+ html_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "index.html")
501
+ with open(html_path, "r", encoding="utf-8") as f:
502
+ return f.read()
503
 
504
  if __name__ == "__main__":
505
+ app.launch(show_error=True, mcp_server=True)