""" AI 3D Generation Pipeline — Hugging Face Space (ZeroGPU) ======================================================== A production-ready Gradio application that generates high-fidelity, manifold 3D models from multi-angle images + text instructions. Pipeline: 1. Background removal (RMBG-1.4) 2. Multi-image composite / best-view selection 3. TripoSG 3D reconstruction 4. Manifold repair + scale normalization 5. GLB/OBJ/STL export Designed for Hugging Face ZeroGPU (A100-40GB). """ from __future__ import annotations import os import io import re import sys import json import uuid import time import shutil import base64 import logging import tempfile import zipfile from pathlib import Path from typing import Optional, List, Tuple, Dict import gradio as gr import numpy as np import torch from PIL import Image # HuggingFace ZeroGPU decorator try: import spaces HAS_SPACES = True except ImportError: HAS_SPACES = False # Fallback no-op decorator for local dev class spaces: @staticmethod def GPU(fn=None, duration=None): if fn is None: return lambda f: f return fn # Mesh processing import trimesh # --------------------------------------------------------------------------- # Configuration # --------------------------------------------------------------------------- MODEL_ID = "VAST-AI/TripoSG" RMBG_MODEL_ID = "briaai/RMBG-1.4" MAX_FACES = 90000 DEFAULT_FACES = 50000 OUTPUT_DIR = Path(tempfile.gettempdir()) / "triposg_output" OUTPUT_DIR.mkdir(parents=True, exist_ok=True) logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- # Global model references (loaded once) # --------------------------------------------------------------------------- triposg_pipeline = None rmbg_model = None device = "cuda" if torch.cuda.is_available() else "cpu" def load_models(): """Load TripoSG + RMBG models into GPU memory.""" global triposg_pipeline, rmbg_model if triposg_pipeline is not None: return # already loaded logger.info("Loading TripoSG pipeline from %s ...", MODEL_ID) try: # TripoSG uses a custom pipeline; import from the repo sys.path.insert(0, os.path.dirname(__file__)) # Method 1: Try to use the HuggingFace model directly from huggingface_hub import snapshot_download model_path = snapshot_download(MODEL_ID) logger.info("Model downloaded to: %s", model_path) # Import TripoSG inference utilities # The model uses a custom pipeline - we import it dynamically sys.path.insert(0, model_path) try: from triposg import TripoSGPipeline triposg_pipeline = TripoSGPipeline.from_pretrained(model_path).to(device) except ImportError: # Alternative: use the inference script approach logger.warning("Direct import failed, using CLI inference fallback") triposg_pipeline = {"model_path": model_path, "type": "cli"} except Exception as e: logger.error("Failed to load TripoSG: %s", e) triposg_pipeline = None # Load background remover logger.info("Loading RMBG background remover...") try: from transformers import pipeline as hf_pipeline rmbg_model = hf_pipeline("image-segmentation", model=RMBG_MODEL_ID, device=device) logger.info("RMBG loaded successfully") except Exception as e: logger.warning("RMBG not available, will skip background removal: %s", e) rmbg_model = None # --------------------------------------------------------------------------- # Image Processing Utilities # --------------------------------------------------------------------------- def remove_background(image: Image.Image) -> Image.Image: """Remove background from an image using RMBG-1.4.""" if rmbg_model is None: return image try: results = rmbg_model(image) # Find the mask and apply it for result in results: if result.get("label") in ["foreground", "person", "object"]: mask = result["mask"] image = image.convert("RGBA") image.putalpha(mask) return image # If no specific label, use the first result if results: mask = results[0]["mask"] image = image.convert("RGBA") image.putalpha(mask) return image except Exception as e: logger.warning("Background removal failed: %s", e) return image def select_best_view(images: list[Image.Image]) -> Image.Image: """ From multiple input images, select the best single view for reconstruction. Heuristic: pick the largest image (highest resolution = most detail). In a production pipeline, you would use a multi-view reconstruction model. """ if not images: raise ValueError("No images provided") if len(images) == 1: return images[0] # Score images by resolution and sharpness scored = [] for img in images: w, h = img.size resolution_score = w * h # Simple Laplacian variance for sharpness gray = np.array(img.convert("L"), dtype=np.float32) laplacian = np.abs(np.roll(gray, 1, 0) + np.roll(gray, -1, 0) + np.roll(gray, 1, 1) + np.roll(gray, -1, 1) - 4 * gray) sharpness = float(np.var(laplacian)) scored.append((img, resolution_score * 0.3 + sharpness * 0.7)) scored.sort(key=lambda x: x[1], reverse=True) logger.info("Selected best view (score=%.1f) from %d images", scored[0][1], len(images)) return scored[0][0] def create_composite_view(images: list[Image.Image], max_images: int = 4) -> Image.Image: """ Create a multi-view composite image (2x2 grid) from multiple input images. Some models work better with a single composite showing multiple angles. """ images = images[:max_images] n = len(images) if n == 1: return images[0] # Resize all to same size target_size = 512 resized = [img.resize((target_size, target_size), Image.LANCZOS) for img in images] if n == 2: composite = Image.new("RGB", (target_size * 2, target_size)) composite.paste(resized[0], (0, 0)) composite.paste(resized[1], (target_size, 0)) elif n <= 4: cols, rows = 2, 2 composite = Image.new("RGB", (target_size * cols, target_size * rows), (255, 255, 255)) for i, img in enumerate(resized): x = (i % cols) * target_size y = (i // cols) * target_size composite.paste(img, (x, y)) else: composite = resized[0] return composite # --------------------------------------------------------------------------- # Text-to-Dimension Extraction # --------------------------------------------------------------------------- def extract_dimensions_from_prompt(prompt: str) -> dict: """ Parse dimensional info from text prompt. Supports patterns like: - "10cm x 5cm x 3cm" - "base is 10cm" - "height: 200mm" - "scale 1:10" Returns dict with extracted measurements. """ dims = {} # Match "NxMxP" patterns (cm, mm, m, inches) pattern_3d = r'(\d+(?:\.\d+)?)\s*(?:x|×|by)\s*(\d+(?:\.\d+)?)\s*(?:x|×|by)\s*(\d+(?:\.\d+)?)\s*(cm|mm|m|in|inch|inches)?' match = re.search(pattern_3d, prompt, re.IGNORECASE) if match: dims["width"] = float(match.group(1)) dims["depth"] = float(match.group(2)) dims["height"] = float(match.group(3)) dims["unit"] = (match.group(4) or "cm").lower().rstrip("es").rstrip("ch") return dims # Match individual dimension patterns for dim_name in ["width", "height", "depth", "length", "base", "diameter", "radius"]: pattern = rf'{dim_name}\s*(?:is|:|\=)?\s*(\d+(?:\.\d+)?)\s*(cm|mm|m|in|inch|inches)?' match = re.search(pattern, prompt, re.IGNORECASE) if match: dims[dim_name] = float(match.group(1)) dims["unit"] = (match.group(2) or "cm").lower().rstrip("es").rstrip("ch") # Match scale ratios scale_pattern = r'(?:scale|ratio)\s*(?:is|:|\=)?\s*1\s*:\s*(\d+(?:\.\d+)?)' match = re.search(scale_pattern, prompt, re.IGNORECASE) if match: dims["scale_ratio"] = float(match.group(1)) return dims # --------------------------------------------------------------------------- # Mesh Post-Processing (Manifold + Scale) # --------------------------------------------------------------------------- def repair_mesh(mesh: trimesh.Trimesh) -> trimesh.Trimesh: """Make a mesh manifold (watertight) for 3D printing.""" logger.info("Repairing mesh: %d vertices, %d faces", len(mesh.vertices), len(mesh.faces)) # Fix normals trimesh.repair.fix_normals(mesh) trimesh.repair.fix_inversion(mesh) trimesh.repair.fix_winding(mesh) # Fill holes if not mesh.is_watertight: mesh.fill_holes() logger.info("Filled holes → watertight: %s", mesh.is_watertight) # Remove degenerate faces mask = mesh.nondegenerate_faces() if not mask.all(): mesh.update_faces(mask) logger.info("Removed %d degenerate faces", (~mask).sum()) # Merge close vertices mesh.merge_vertices(merge_tex=True, merge_norm=True) return mesh def scale_mesh(mesh: trimesh.Trimesh, dimensions: dict) -> trimesh.Trimesh: """ Scale mesh based on extracted dimensions. The model outputs are in arbitrary units — we map the largest extent to the largest provided dimension. """ if not dimensions: return mesh # Convert everything to millimeters for consistency unit_to_mm = {"mm": 1.0, "cm": 10.0, "m": 1000.0, "in": 25.4} unit = dimensions.get("unit", "cm") multiplier = unit_to_mm.get(unit, 10.0) # Find the target size target_dims = [] for key in ["width", "height", "depth", "length", "base", "diameter"]: if key in dimensions: target_dims.append(dimensions[key] * multiplier) # convert to mm if not target_dims: if "scale_ratio" in dimensions: # Apply scale ratio directly scale_factor = 1.0 / dimensions["scale_ratio"] mesh.apply_scale(scale_factor) return mesh return mesh # Scale the mesh so its largest extent matches the largest target dimension current_extents = mesh.extents max_current = max(current_extents) max_target = max(target_dims) if max_current > 0: scale_factor = max_target / max_current mesh.apply_scale(scale_factor) logger.info("Scaled mesh by %.3f (target: %.1fmm)", scale_factor, max_target) return mesh def segment_mesh_parts(mesh: trimesh.Trimesh) -> list[trimesh.Trimesh]: """ Attempt to split a mesh into separate connected components (modular parts). This is useful for complex objects where parts should be independently editable. """ try: components = mesh.split(only_watertight=False) if len(components) > 1: logger.info("Mesh split into %d modular parts", len(components)) return components else: logger.info("Mesh is a single connected component") return [mesh] except Exception as e: logger.warning("Mesh segmentation failed: %s", e) return [mesh] # --------------------------------------------------------------------------- # Export Utilities # --------------------------------------------------------------------------- def export_mesh(mesh: trimesh.Trimesh, fmt: str = "glb", parts: list = None) -> str: """ Export mesh to file. If parts are provided, creates a ZIP with individual files. """ job_id = str(uuid.uuid4())[:8] output_base = OUTPUT_DIR / job_id output_base.mkdir(parents=True, exist_ok=True) if parts and len(parts) > 1: # Export each part separately and zip them part_files = [] for i, part in enumerate(parts): fname = f"part_{i+1:02d}.{fmt}" fpath = output_base / fname part.export(str(fpath)) part_files.append(str(fpath)) # Also export the combined model combined_path = output_base / f"combined.{fmt}" mesh.export(str(combined_path)) part_files.append(str(combined_path)) # Create zip zip_path = output_base / f"model_parts_{job_id}.zip" with zipfile.ZipFile(str(zip_path), "w") as zf: for pf in part_files: zf.write(pf, os.path.basename(pf)) logger.info("Exported %d parts + combined to %s", len(parts), zip_path) return str(combined_path), str(zip_path) else: fpath = output_base / f"model.{fmt}" mesh.export(str(fpath)) logger.info("Exported single mesh to %s", fpath) return str(fpath), str(fpath) # --------------------------------------------------------------------------- # Main Generation Pipeline # --------------------------------------------------------------------------- @spaces.GPU(duration=300) def generate_3d_model( input_images: list, text_prompt: str, num_faces: int = DEFAULT_FACES, export_format: str = "glb", enable_modular: bool = False, progress=gr.Progress(track_tqdm=True), ): """ End-to-end 3D generation pipeline. Args: input_images: List of uploaded image file paths text_prompt: Text instructions (dimensions, details, style) num_faces: Target face count for the mesh export_format: Output format (glb, obj, stl) enable_modular: Whether to split into modular parts """ if not input_images and not text_prompt: raise gr.Error("Please provide at least one image or a text prompt.") start_time = time.time() status_log = [] def log(msg): status_log.append(msg) logger.info(msg) # --- Step 1: Load models --- log("🔄 Loading models...") load_models() if triposg_pipeline is None: raise gr.Error("Model failed to load. Please try again or check the Space logs.") # --- Step 2: Process images --- log("🖼️ Processing input images...") pil_images = [] if input_images: for img_path in input_images: if isinstance(img_path, str): img = Image.open(img_path).convert("RGB") elif hasattr(img_path, "name"): img = Image.open(img_path.name).convert("RGB") else: img = img_path.convert("RGB") if isinstance(img_path, Image.Image) else None if img: pil_images.append(img) # --- Step 3: Remove backgrounds --- log("✂️ Removing backgrounds...") clean_images = [remove_background(img) for img in pil_images] # --- Step 4: Select best view / create composite --- if clean_images: if len(clean_images) == 1: primary_image = clean_images[0] log("📷 Using single image input") else: primary_image = select_best_view(clean_images) log(f"📷 Selected best view from {len(clean_images)} images") else: primary_image = None # --- Step 5: Extract dimensions from prompt --- dimensions = {} if text_prompt: dimensions = extract_dimensions_from_prompt(text_prompt) if dimensions: log(f"📏 Extracted dimensions: {dimensions}") # --- Step 6: Run TripoSG inference --- log("🧠 Running 3D reconstruction (this may take 1-3 minutes)...") try: if isinstance(triposg_pipeline, dict) and triposg_pipeline.get("type") == "cli": # CLI fallback: save image, run inference script model_path = triposg_pipeline["model_path"] tmp_img = OUTPUT_DIR / f"input_{uuid.uuid4().hex[:6]}.png" if primary_image: primary_image.save(str(tmp_img)) else: # Create a blank image if only text prompt blank = Image.new("RGB", (512, 512), (200, 200, 200)) blank.save(str(tmp_img)) tmp_out = OUTPUT_DIR / f"output_{uuid.uuid4().hex[:6]}.glb" import subprocess cmd = [ sys.executable, "-m", "scripts.inference_triposg", "--image-input", str(tmp_img), "--output-path", str(tmp_out), "--faces", str(num_faces), ] env = os.environ.copy() env["PYTHONPATH"] = model_path + ":" + env.get("PYTHONPATH", "") result = subprocess.run(cmd, capture_output=True, text=True, timeout=600, cwd=model_path, env=env) if result.returncode != 0: logger.error("Inference stderr: %s", result.stderr) raise RuntimeError(f"Inference failed: {result.stderr[-500:]}") mesh = trimesh.load(str(tmp_out)) else: # Direct pipeline call if primary_image: output = triposg_pipeline( primary_image, num_faces=num_faces, ) # The output format depends on the pipeline version if hasattr(output, "meshes"): mesh = output.meshes[0] elif isinstance(output, trimesh.Trimesh): mesh = output else: # Try to load from file path mesh = trimesh.load(output) else: raise gr.Error("Image input is required for 3D generation.") except Exception as e: raise gr.Error(f"3D reconstruction failed: {str(e)}") # --- Step 7: Post-process mesh --- log("🔧 Repairing mesh for manifold output...") mesh = repair_mesh(mesh) # --- Step 8: Apply dimensional scaling --- if dimensions: log("📐 Applying dimensional scaling...") mesh = scale_mesh(mesh, dimensions) # --- Step 9: Modular segmentation --- parts = None if enable_modular: log("🧩 Segmenting into modular parts...") parts = segment_mesh_parts(mesh) # --- Step 10: Export --- log(f"💾 Exporting as .{export_format}...") preview_path, download_path = export_mesh(mesh, export_format, parts) elapsed = time.time() - start_time log(f"✅ Complete in {elapsed:.1f}s — manifold: {mesh.is_watertight}, " f"vertices: {len(mesh.vertices)}, faces: {len(mesh.faces)}") status_text = "\n".join(status_log) return preview_path, download_path, status_text # --------------------------------------------------------------------------- # Gradio API Endpoint (for n8n / external calls) # --------------------------------------------------------------------------- @spaces.GPU(duration=300) def api_generate( image_b64_list: str, prompt: str, faces: int = DEFAULT_FACES, fmt: str = "glb", modular: bool = False, ): """ JSON API endpoint for programmatic access (n8n, curl, etc.). Args: image_b64_list: JSON array of base64-encoded images prompt: Text instructions faces: Target face count fmt: Export format (glb/obj/stl) modular: Split into parts Returns: Tuple of (model file path, download file path, status text) """ # Decode images from base64 images_data = json.loads(image_b64_list) if image_b64_list else [] tmp_files = [] for i, b64_str in enumerate(images_data): # Handle data URIs if "," in b64_str: b64_str = b64_str.split(",", 1)[1] img_bytes = base64.b64decode(b64_str) tmp_path = OUTPUT_DIR / f"api_input_{uuid.uuid4().hex[:6]}.png" tmp_path.write_bytes(img_bytes) tmp_files.append(str(tmp_path)) return generate_3d_model( input_images=tmp_files, text_prompt=prompt, num_faces=faces, export_format=fmt, enable_modular=modular, ) # --------------------------------------------------------------------------- # Gradio UI # --------------------------------------------------------------------------- HEADER_MD = """ # 🚀 AI 3D Model Generator — Print-Ready Pipeline Generate **high-fidelity, manifold 3D models** from multiple images + text instructions. Upload photos from different angles, add dimensional requirements, and get a **watertight mesh** ready for 3D printing or game engines. **Pipeline**: Background Removal → Multi-View Selection → TripoSG Reconstruction → Manifold Repair → Scale Calibration → Export > 💡 **Tip**: For best results, provide 2-4 images of the object from different angles on a clean background. """ with gr.Blocks( title="AI 3D Model Generator", theme=gr.themes.Soft( primary_hue=gr.themes.colors.indigo, secondary_hue=gr.themes.colors.purple, ), css=""" .gradio-container { max-width: 1200px !important; } .status-box textarea { font-family: monospace; font-size: 12px; } """, ) as demo: gr.Markdown(HEADER_MD) with gr.Row(): # ----- Left Column: Inputs ----- with gr.Column(scale=1): gr.Markdown("### 📥 Input") input_images = gr.File( label="Upload Images (multi-angle)", file_count="multiple", file_types=["image"], type="filepath", ) text_prompt = gr.Textbox( label="Text Instructions", placeholder=( "Describe details, dimensions, materials...\n" "e.g.: Industrial bracket, base is 10cm × 5cm, " "matte steel finish, M6 bolt holes on each corner" ), lines=4, ) with gr.Row(): num_faces = gr.Slider( minimum=5000, maximum=MAX_FACES, value=DEFAULT_FACES, step=5000, label="Target Face Count", ) export_format = gr.Dropdown( choices=["glb", "obj", "stl"], value="glb", label="Export Format", ) enable_modular = gr.Checkbox( label="🧩 Split into modular parts (experimental)", value=False, ) generate_btn = gr.Button( "🚀 Generate 3D Model", variant="primary", size="lg", ) # ----- Right Column: Outputs ----- with gr.Column(scale=1): gr.Markdown("### 📤 Output") model_preview = gr.Model3D( label="3D Model Preview", clear_color=[0.1, 0.1, 0.15, 1.0], ) download_file = gr.File(label="⬇️ Download Model") status_output = gr.Textbox( label="Pipeline Status", lines=10, interactive=False, elem_classes=["status-box"], ) # ----- Hidden API interface ----- with gr.Row(visible=False): api_image_input = gr.Textbox(label="Base64 Images JSON") api_prompt_input = gr.Textbox(label="Prompt") api_faces_input = gr.Number(label="Faces", value=DEFAULT_FACES) api_fmt_input = gr.Textbox(label="Format", value="glb") api_modular_input = gr.Checkbox(label="Modular", value=False) api_model_output = gr.File(label="Model") api_download_output = gr.File(label="Download") api_status_output = gr.Textbox(label="Status") # ----- Event handlers ----- generate_btn.click( fn=generate_3d_model, inputs=[input_images, text_prompt, num_faces, export_format, enable_modular], outputs=[model_preview, download_file, status_output], api_name="generate", ) # --------------------------------------------------------------------------- # Launch # --------------------------------------------------------------------------- demo.queue(max_size=5) demo.launch( show_api=True, # Expose /api/ endpoints for n8n )