import { client, handle_file } from "https://cdn.jsdelivr.net/npm/@gradio/client/dist/index.min.js"; let clientInstance = null; const state = { unique: { files: [], framesDir: null, frameCount: 0, fps: 1.0, frameIdx: 0, points: [] }, generic: { files: [] } }; const $ = (id) => document.getElementById(id); function samplingMode(prefix) { const el = $(`${prefix}-sampling`); return el ? el.value : "1 FPS (Space limit)"; } function escapeHtml(value) { return String(value ?? "") .replaceAll("&", "&") .replaceAll("<", "<") .replaceAll(">", ">") .replaceAll('"', """) .replaceAll("'", "'"); } function nowTime() { return new Date().toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", second: "2-digit" }); } function statusMarkup(stateName, message) { const dot = stateName === "err" ? "idle" : stateName; const color = stateName === "err" ? ' style="background:var(--sl-signal)"' : ""; return `
${escapeHtml(message)} ${nowTime()}
`; } function setStatus(prefix, stateName, message) { $(`${prefix}-status`).innerHTML = statusMarkup(stateName, message); } function setBackendHtml(prefix, meta) { if (meta?.status_html) { $(`${prefix}-status`).innerHTML = meta.status_html; } if (meta?.stats_html) { $(`${prefix}-stats`).innerHTML = meta.stats_html; } } async function extractGenericServerPreview(file) { const img = $("generic-input-img"); const empty = $("generic-result-empty"); try { const api = await getClient(); const result = await api.predict("/preview_generic", { input_files: wrapFiles([file]) }); const [frameFile, meta] = result.data; const url = fileUrl(frameFile); if (meta && meta.success && url) { img.onload = () => empty.classList.add("hidden"); img.src = url; img.classList.remove("hidden"); empty.classList.add("hidden"); return; } } catch (e) { console.error("Server preview failed:", e); } empty.textContent = `${file.name} — preview not available (it will still be processed).`; empty.classList.remove("hidden"); } function setGenericPreview(file) { const video = $("generic-input"); const img = $("generic-input-img"); const empty = $("generic-result-empty"); // leaving any previous result behind $("generic-result").classList.add("hidden"); $("generic-download").classList.add("hidden"); $("generic-stats").classList.add("hidden"); empty.textContent = "Drop input to preview."; if (!file) { video.classList.add("hidden"); img.classList.add("hidden"); video.removeAttribute("src"); img.removeAttribute("src"); empty.classList.remove("hidden"); return; } const url = URL.createObjectURL(file); if ((file.type || "").startsWith("video")) { video.muted = true; video.src = url; video.classList.remove("hidden"); img.classList.add("hidden"); video.load(); // preload="metadata" won't paint a frame on its own — seek a hair to // force the browser to decode and show the first frame as a thumbnail. video.addEventListener("loadedmetadata", function onMeta() { try { video.currentTime = Math.min(0.1, (video.duration || 1) / 2); } catch (e) {} }, { once: true }); // Some codecs (MPEG-4 Part 2, HEVC…) can't be decoded by the browser. // Fall back to a first frame extracted server-side with OpenCV (like Unique). video.addEventListener("error", function onErr() { video.classList.add("hidden"); empty.textContent = "Loading preview…"; empty.classList.remove("hidden"); extractGenericServerPreview(file); }, { once: true }); } else { img.src = url; img.classList.remove("hidden"); video.classList.add("hidden"); } empty.classList.add("hidden"); $("generic-head-label").textContent = "Input · preview"; } function showUniqueView(view) { const result = view === "result"; $("unique-setup-view").classList.toggle("hidden", result); $("unique-result-view").classList.toggle("hidden", !result); if (!result) { requestAnimationFrame(drawUniquePoints); } } async function getClient() { if (!clientInstance) { clientInstance = await client(window.location.origin); } return clientInstance; } function wrapFiles(files) { return Array.from(files || []).map((file) => handle_file(file)); } function fileUrl(fileObj) { return fileObj?.url || fileObj?.path || ""; } function setButtonBusy(button, busy, label) { if (!button.dataset.label) { button.dataset.label = button.querySelector("span:last-child")?.textContent || button.textContent; } button.disabled = busy; const text = button.querySelector("span:last-child"); if (text) { text.textContent = busy ? label : button.dataset.label; } } function setupTabs() { document.querySelectorAll(".tab").forEach((button) => { button.addEventListener("click", () => { const target = button.dataset.tab; document.querySelectorAll(".tab").forEach((tab) => tab.classList.toggle("active", tab === button)); document.querySelectorAll(".tab-panel").forEach((panel) => { panel.classList.toggle("active", panel.id === `${target}-panel`); }); requestAnimationFrame(drawUniquePoints); }); }); } function setupFileDrop(prefix) { const drop = $(`${prefix}-drop`); const input = $(`${prefix}-file`); const label = $(`${prefix}-file-label`); function assignFiles(files) { const list = Array.from(files || []); state[prefix].files = list; drop.classList.toggle("has-file", list.length > 0); if (list.length === 0) { label.textContent = "Drop input here or browse"; } else if (list.length === 1) { label.textContent = list[0].name; } else { label.textContent = `${list.length} files selected`; } if (prefix === "unique") { clearUniqueReference(); setStatus("unique", "idle", "Input loaded. Run Extract Frames."); } else { setGenericPreview(list[0]); setStatus("generic", "idle", "Input loaded. Ready to run segmentation."); } } input.addEventListener("change", () => assignFiles(input.files)); ["dragenter", "dragover"].forEach((eventName) => { drop.addEventListener(eventName, (event) => { event.preventDefault(); drop.classList.add("drag"); }); }); ["dragleave", "drop"].forEach((eventName) => { drop.addEventListener(eventName, (event) => { event.preventDefault(); drop.classList.remove("drag"); }); }); drop.addEventListener("drop", (event) => { assignFiles(event.dataTransfer.files); }); } function clearUniqueReference() { state.unique.framesDir = null; state.unique.frameCount = 0; state.unique.fps = 1.0; state.unique.frameIdx = 0; state.unique.points = []; $("unique-slider-wrap").classList.add("hidden"); $("unique-ref-hint").classList.remove("hidden"); $("unique-frame-slider").value = "0"; $("unique-frame-slider").max = "0"; $("unique-frame-count").textContent = "0 / 0"; $("unique-reference").classList.add("empty"); $("unique-reference").querySelector(".empty-state").classList.remove("hidden"); $("unique-reference-img").classList.add("hidden"); $("unique-reference-img").removeAttribute("src"); $("unique-reference").classList.remove("running"); showUniqueView("setup"); drawUniquePoints(); } function setReferenceImage(fileObj) { const url = fileUrl(fileObj); const image = $("unique-reference-img"); const box = $("unique-reference"); if (!url) { clearUniqueReference(); return; } image.src = url; image.classList.remove("hidden"); box.classList.remove("empty"); box.querySelector(".empty-state").classList.add("hidden"); image.onload = drawUniquePoints; } function updateFrameCounter() { const count = state.unique.frameCount || 0; $("unique-frame-count").textContent = count ? `${state.unique.frameIdx + 1} / ${count}` : "0 / 0"; } async function extractUniqueFrames() { if (!state.unique.files.length) { setStatus("unique", "idle", "Drop input before extracting frames."); return; } const button = $("unique-extract"); setButtonBusy(button, true, "Extracting"); setStatus("unique", "live", "Extracting frames..."); try { const api = await getClient(); const result = await api.predict("/extract_unique", { input_files: wrapFiles(state.unique.files), sampling: samplingMode("unique") }); const [frameFile, meta] = result.data; setBackendHtml("unique", meta); if (meta?.success) { state.unique.framesDir = meta.frames_dir; state.unique.frameCount = meta.frame_count || 0; state.unique.fps = meta.fps || 1.0; state.unique.frameIdx = 0; state.unique.points = []; $("unique-frame-slider").max = String(Math.max(0, state.unique.frameCount - 1)); $("unique-frame-slider").value = "0"; $("unique-slider-wrap").classList.remove("hidden"); $("unique-ref-hint").classList.add("hidden"); updateFrameCounter(); setReferenceImage(frameFile); } } catch (error) { console.error(error); setStatus("unique", "err", error.message || "Frame extraction failed."); } finally { setButtonBusy(button, false); } } async function loadReferenceFrame(frameIdx) { if (!state.unique.framesDir) { return; } state.unique.frameIdx = Number(frameIdx) || 0; state.unique.points = []; showUniqueView("setup"); updateFrameCounter(); drawUniquePoints(); try { const api = await getClient(); const result = await api.predict("/reference_frame", { frames_dir: state.unique.framesDir, frame_idx: state.unique.frameIdx }); const [frameFile, meta] = result.data; if (meta?.success) { setReferenceImage(frameFile); } else { setStatus("unique", "err", meta?.error || "Reference frame not found."); } } catch (error) { console.error(error); setStatus("unique", "err", error.message || "Could not load reference frame."); } } function displayedImageRect() { const image = $("unique-reference-img"); const box = $("unique-reference"); if (image.classList.contains("hidden") || !image.naturalWidth) { return null; } const imageRect = image.getBoundingClientRect(); const boxRect = box.getBoundingClientRect(); return { image, box, width: imageRect.width, height: imageRect.height, left: imageRect.left - boxRect.left, top: imageRect.top - boxRect.top, naturalWidth: image.naturalWidth, naturalHeight: image.naturalHeight }; } function drawUniquePoints() { const canvas = $("unique-point-canvas"); const rect = displayedImageRect(); if (!rect) { canvas.width = 0; canvas.height = 0; return; } const dpr = window.devicePixelRatio || 1; canvas.style.left = `${rect.left}px`; canvas.style.top = `${rect.top}px`; canvas.style.width = `${rect.width}px`; canvas.style.height = `${rect.height}px`; canvas.width = Math.max(1, Math.round(rect.width * dpr)); canvas.height = Math.max(1, Math.round(rect.height * dpr)); const ctx = canvas.getContext("2d"); ctx.setTransform(dpr, 0, 0, dpr, 0, 0); ctx.clearRect(0, 0, rect.width, rect.height); ctx.strokeStyle = "#315DCE"; ctx.lineWidth = 2; state.unique.points.forEach(([x, y]) => { const px = (x / rect.naturalWidth) * rect.width; const py = (y / rect.naturalHeight) * rect.height; ctx.beginPath(); ctx.moveTo(px - 10, py); ctx.lineTo(px + 10, py); ctx.moveTo(px, py - 10); ctx.lineTo(px, py + 10); ctx.stroke(); }); } function addUniquePoint(event) { const rect = displayedImageRect(); if (!rect) { return; } const imageRect = rect.image.getBoundingClientRect(); const x = Math.round(((event.clientX - imageRect.left) / imageRect.width) * rect.naturalWidth); const y = Math.round(((event.clientY - imageRect.top) / imageRect.height) * rect.naturalHeight); state.unique.points.push([x, y]); $("unique-text").value = ""; drawUniquePoints(); } function showVideo(prefix, fileObj) { const url = fileUrl(fileObj); const video = $(`${prefix}-result`); const empty = $(`${prefix}-result-empty`); const download = $(`${prefix}-download`); if (!url) { video.classList.add("hidden"); empty.classList.remove("hidden"); download.classList.add("hidden"); return; } video.src = url; video.classList.remove("hidden"); empty.classList.add("hidden"); download.href = url; download.classList.remove("hidden"); } async function runUniqueSegmentation() { const button = $("unique-run"); setButtonBusy(button, true, "Running"); setStatus("unique", "live", "Running segmentation..."); $("unique-reference").classList.add("running"); try { const api = await getClient(); const result = await api.predict("/run_unique", { input_files: wrapFiles(state.unique.files), points: state.unique.points, prompt: $("unique-text").value, encoder_size: $("unique-encoder").value, offload: $("unique-offload").checked, frame_idx: state.unique.frameIdx, frames_dir_cache: state.unique.framesDir, original_fps: state.unique.fps }); const [videoFile, meta] = result.data; setBackendHtml("unique", meta); showVideo("unique", videoFile); showUniqueView("result"); } catch (error) { console.error(error); setStatus("unique", "err", error.message || "Segmentation failed."); } finally { $("unique-reference").classList.remove("running"); setButtonBusy(button, false); } } async function runGenericSegmentation() { if (!state.generic.files.length) { setStatus("generic", "idle", "Drop input before running segmentation."); return; } const button = $("generic-run"); setButtonBusy(button, true, "Running"); setStatus("generic", "live", "Running segmentation..."); $("generic-result-box").classList.add("running"); try { const api = await getClient(); const result = await api.predict("/run_generic", { input_files: wrapFiles(state.generic.files), category: $("generic-category").value, accept: Number($("generic-accept").value), reject: Number($("generic-reject").value), vlm_model: $("generic-vlm").value, sampling: samplingMode("generic") }); const [videoFile, meta] = result.data; setBackendHtml("generic", meta); $("generic-input").classList.add("hidden"); $("generic-input-img").classList.add("hidden"); $("generic-head-label").textContent = "Result · masks overlaid per frame"; showVideo("generic", videoFile); $("generic-stats").classList.remove("hidden"); } catch (error) { console.error(error); setStatus("generic", "err", error.message || "Segmentation failed."); } finally { $("generic-result-box").classList.remove("running"); setButtonBusy(button, false); } } function setupRanges() { const accept = $("generic-accept"); const reject = $("generic-reject"); accept.addEventListener("input", () => { $("generic-accept-value").textContent = Number(accept.value).toFixed(2); }); reject.addEventListener("input", () => { $("generic-reject-value").textContent = Number(reject.value).toFixed(2); }); const frameSlider = $("unique-frame-slider"); frameSlider.addEventListener("input", () => { state.unique.frameIdx = Number(frameSlider.value) || 0; updateFrameCounter(); }); frameSlider.addEventListener("change", () => loadReferenceFrame(frameSlider.value)); } async function loadExample(btn) { const prefix = btn.dataset.prefix; const src = btn.dataset.src; let fields = {}; try { fields = JSON.parse(btn.dataset.fields || "{}"); } catch (e) {} btn.disabled = true; try { const resp = await fetch(src); const blob = await resp.blob(); const file = new File([blob], src.split("/").pop(), { type: blob.type || "video/mp4" }); const input = $(`${prefix}-file`); const dt = new DataTransfer(); dt.items.add(file); input.files = dt.files; input.dispatchEvent(new Event("change")); // reuse the normal upload flow Object.entries(fields).forEach(([id, val]) => { const el = $(id); if (!el) return; el.value = val; el.dispatchEvent(new Event("input")); }); } catch (e) { console.error("Example load failed:", e); } finally { btn.disabled = false; } } function setupActions() { $("unique-extract").addEventListener("click", extractUniqueFrames); $("unique-run").addEventListener("click", runUniqueSegmentation); $("generic-run").addEventListener("click", runGenericSegmentation); $("unique-reference-img").addEventListener("click", addUniquePoint); $("unique-clear-points").addEventListener("click", () => { state.unique.points = []; drawUniquePoints(); }); $("unique-back").addEventListener("click", () => showUniqueView("setup")); document.querySelectorAll(".example-card").forEach((b) => b.addEventListener("click", () => loadExample(b))); window.addEventListener("resize", drawUniquePoints); } setupTabs(); setupFileDrop("unique"); setupFileDrop("generic"); setupRanges(); setupActions();