John Ho Claude Opus 4.8 (1M context) commited on
Commit ·
4330d70
1
Parent(s): abcd56e
Rewrite MCP tool docstrings so schema survives Gradio's parser
Browse filesMove the output schema of process_image/process_video into the docstring
description prose (the only section Gradio's MCP parser leaves intact),
name the bounding-box coordinate formats explicitly with a reference URL,
keep each Args entry on a single line, drop the mangled Returns sections,
and add return type hints. Verified against a live MCP schema dump.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
app.py
CHANGED
|
@@ -37,18 +37,17 @@ def process_image(
|
|
| 37 |
bboxes: Union[list, str] = None,
|
| 38 |
points: Union[list, str] = None,
|
| 39 |
point_labels: Union[list, str] = None,
|
| 40 |
-
):
|
| 41 |
-
"""
|
| 42 |
-
|
|
|
|
| 43 |
|
| 44 |
Args:
|
| 45 |
-
im: Pillow Image
|
| 46 |
-
variant: SAM2 model variant
|
| 47 |
-
bboxes: bounding boxes
|
| 48 |
-
points: points
|
| 49 |
-
point_labels: list of
|
| 50 |
-
Returns:
|
| 51 |
-
list: a list of masks in the form of bit64 encoded strings
|
| 52 |
"""
|
| 53 |
# input validation
|
| 54 |
has_bboxes = type(bboxes) != type(None) and bboxes != ""
|
|
@@ -93,19 +92,18 @@ def process_video(
|
|
| 93 |
drop_masks: bool = False,
|
| 94 |
ref_frame_idx: int = 0,
|
| 95 |
async_frame_load: bool = True,
|
| 96 |
-
):
|
| 97 |
-
"""
|
| 98 |
-
|
|
|
|
| 99 |
|
| 100 |
Args:
|
| 101 |
-
video_path: path to video
|
| 102 |
-
variant:
|
| 103 |
-
masks:
|
| 104 |
-
drop_masks:
|
| 105 |
-
ref_frame_idx: the
|
| 106 |
-
async_frame_load:
|
| 107 |
-
Returns:
|
| 108 |
-
list: a list of tracked objects expressed as a list of dictionary [{"frame":..., "track_id":..., "x":..., "y":...,"w":...,"h":...,"conf":..., "mask_b64":...},...]
|
| 109 |
"""
|
| 110 |
model = load_vid_model(variant=variant)
|
| 111 |
masks = json.loads(masks) if isinstance(masks, str) else masks
|
|
|
|
| 37 |
bboxes: Union[list, str] = None,
|
| 38 |
points: Union[list, str] = None,
|
| 39 |
point_labels: Union[list, str] = None,
|
| 40 |
+
) -> list[str]:
|
| 41 |
+
"""SAM2 image segmentation: segment objects in an image from box and/or point prompts and return their masks.
|
| 42 |
+
|
| 43 |
+
Provide prompts as bounding boxes and/or points (points require point_labels). Returns a JSON list of segmentation masks, one mask per prompt -- one mask per bounding box, or a single mask for a points-only prompt -- in the same order as the input prompts. Each mask is a base64-encoded 1-bit (binary) PNG string; when decoded the PNG has the same width and height as the original input image, with pixel value 1 inside the object and 0 elsewhere.
|
| 44 |
|
| 45 |
Args:
|
| 46 |
+
im: The RGB image to segment, as a Pillow Image.
|
| 47 |
+
variant: SAM2 model variant to load; one of "tiny", "small", "base_plus", or "large".
|
| 48 |
+
bboxes: Object bounding boxes to segment, as a JSON list of dicts [{"x0":..., "y0":..., "x1":..., "y1":...}, ...] in ABSOLUTE PIXEL coordinates of the input image (top-left corner x0/y0, bottom-right corner x1/y1) -- the albumentations "pascal_voc" format; either bboxes or points must be provided.
|
| 49 |
+
points: Object points to segment, as a JSON list of dicts [{"x":..., "y":...}, ...] in ABSOLUTE PIXEL coordinates of the input image; requires point_labels.
|
| 50 |
+
point_labels: JSON list of integers, one per point, where 1 marks a foreground (include) point and 0 a background (exclude) point.
|
|
|
|
|
|
|
| 51 |
"""
|
| 52 |
# input validation
|
| 53 |
has_bboxes = type(bboxes) != type(None) and bboxes != ""
|
|
|
|
| 92 |
drop_masks: bool = False,
|
| 93 |
ref_frame_idx: int = 0,
|
| 94 |
async_frame_load: bool = True,
|
| 95 |
+
) -> list[dict]:
|
| 96 |
+
"""SAM2 video segmentation: track objects through a video given their segmentation masks on a reference frame.
|
| 97 |
+
|
| 98 |
+
Seed the objects to track by supplying their masks on the reference frame (ref_frame_idx); SAM2 then propagates each object forward -- and, when ref_frame_idx is nonzero, also backward -- through every frame. Returns a JSON list of per-frame detections, one entry per tracked object per frame in which it appears (frames where an object is absent are skipped). Each detection is a dict with: "frame" (integer frame index); "track_id" (integer object id, stable across frames); "x", "y", "w", "h" (the object's bounding box as top-left-x, top-left-y, width, height, each NORMALIZED to 0.0-1.0 by dividing by the frame width or height -- this is the albumentations "coco" layout [x_min, y_min, width, height] but normalized to 0-1 rather than absolute pixels, and it is NOT [x_min, y_min, x_max, y_max]; box formats documented at https://albumentations.ai/docs/3-basic-usage/bounding-boxes-augmentations/#bounding-box-formats ); "conf" (always 1); and, unless drop_masks is true, "mask_b64" (a base64-encoded 1-bit PNG string the same width and height as the video frame, 1 inside the object and 0 elsewhere).
|
| 99 |
|
| 100 |
Args:
|
| 101 |
+
video_path: Filesystem path to the input video file.
|
| 102 |
+
variant: SAM2 model variant to load; one of "tiny", "small", "base_plus", or "large".
|
| 103 |
+
masks: JSON list of base64-encoded 1-bit PNG masks for the reference frame, one per object to track, e.g. ["b'iVBORw0KGgo...'", ...]; the b'...' literal wrapper is accepted and stripped.
|
| 104 |
+
drop_masks: When true, omit the "mask_b64" field from every detection so only bounding-box information is returned.
|
| 105 |
+
ref_frame_idx: Frame index the provided masks correspond to; a nonzero value triggers bidirectional tracking (forward and backward from this frame).
|
| 106 |
+
async_frame_load: When true, load video frames in parallel with propagation to reduce inference time.
|
|
|
|
|
|
|
| 107 |
"""
|
| 108 |
model = load_vid_model(variant=variant)
|
| 109 |
masks = json.loads(masks) if isinstance(masks, str) else masks
|