#!/usr/bin/env python3 """Build face-rendered, solidified, and canonical 3D views for one result.""" from __future__ import annotations import argparse import hashlib import json import os import sys from pathlib import Path from typing import Any import cv2 import numpy as np import trimesh from PIL import Image, ImageDraw PROJECT_ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(PROJECT_ROOT)) from tools.build_accessibility_solid_mesh_showcase import ( # noqa: E402 CONTINUOUS_SURFACE_CATEGORIES, build_canonical_staircase_prism_mesh, build_continuous_stair_mesh, build_solid_mesh, clean_target_mask, continuous_depth_for_category, fit_plane, plane_depth_for_pixels, prepare_texture, read_json, read_mask, render_mesh_frame, trusted_stair_edges_from_manifest, write_ply, write_turntable_gif, ) from accessibilityamodal.reconstruct import camera_intrinsics, pixels_to_points # noqa: E402 def continuous_presentation_mask( target_mask: np.ndarray, *, maximum_hull_expansion_ratio: float = 1.8, ) -> tuple[np.ndarray, dict[str, Any]]: """Return a hole-free, category-constrained surface presentation mask. The semantic/review masks remain untouched on disk. This mask is used only for the nonmetric solid visualization so that segmentation notches do not become vertical person-shaped walls. A modest convex hull is accepted for perspective walkway footprints; strongly concave routes keep their outer contour and only close internal holes. """ target = np.asarray(target_mask, dtype=bool) if target.ndim != 2: raise ValueError("target_mask must be a 2D mask") if maximum_hull_expansion_ratio < 1: raise ValueError("maximum_hull_expansion_ratio must be at least 1") pixels = int(target.sum()) if pixels == 0: return target.copy(), { "policy": "unchanged_empty_target", "source_pixel_count": 0, "presentation_pixel_count": 0, "hull_expansion_ratio": None, } external = np.zeros_like(target, dtype=np.uint8) contours, _ = cv2.findContours( target.astype(np.uint8), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE, ) cv2.drawContours(external, contours, -1, 1, cv2.FILLED) external_mask = external.astype(bool) ys, xs = np.where(external_mask) points = np.column_stack([xs, ys]).astype(np.int32) hull = np.zeros_like(external, dtype=np.uint8) if points.shape[0] >= 3: cv2.fillConvexPoly(hull, cv2.convexHull(points), 1) else: hull[external_mask] = 1 hull_mask = hull.astype(bool) hull_ratio = int(hull_mask.sum()) / max(int(external_mask.sum()), 1) use_hull = hull_ratio <= maximum_hull_expansion_ratio presentation = hull_mask if use_hull else external_mask return presentation, { "policy": ( "bounded_convex_support_presentation" if use_hull else "external_contour_hole_fill_due_to_ambiguous_concavity" ), "source_pixel_count": pixels, "external_contour_pixel_count": int(external_mask.sum()), "presentation_pixel_count": int(presentation.sum()), "hull_expansion_ratio": round(hull_ratio, 8), "maximum_hull_expansion_ratio": float(maximum_hull_expansion_ratio), "semantic_mask_unchanged": True, "metric_geometry": False, "human_review_required": True, } def idealized_continuous_presentation_depth( depth: np.ndarray, presentation_mask: np.ndarray, visible_mask: np.ndarray | None, ) -> tuple[np.ndarray, dict[str, Any]]: """Fit one bounded support plane for a clean, nonmetric solid preview.""" source = np.asarray(depth, dtype=np.float32) target = np.asarray(presentation_mask, dtype=bool) visible = target if visible_mask is None else ( np.asarray(visible_mask, dtype=bool) & target ) h, w = source.shape fx, fy, cx, cy = camera_intrinsics(w, h, None, None, None, None) points = pixels_to_points(source, fx, fy, cx, cy) fit_mask = visible & np.isfinite(source) & (source > 0) plane = fit_plane(points[fit_mask]) if plane is None: fallback, metadata = continuous_depth_for_category( source, target, visible_mask, "walkway", ) return fallback, { **metadata, "presentation_depth_policy": "fallback_continuous_depth_plane_fit_failed", "human_review_required": True, } observed = source[fit_mask] low, high = np.percentile(observed, [1, 99]) lower_bound = max(1e-4, 0.50 * float(low)) upper_bound = max(lower_bound + 1e-4, 1.50 * float(high)) plane_depth = plane_depth_for_pixels( source.shape, plane, fx, fy, cx, cy, ) invalid = target & (~np.isfinite(plane_depth) | (plane_depth <= 0)) bounded_plane = np.clip(plane_depth, lower_bound, upper_bound) result = source.copy() result[target & ~invalid] = bounded_plane[target & ~invalid] if invalid.any(): result[invalid] = float(np.median(observed)) clamped = target & ( (plane_depth < lower_bound) | (plane_depth > upper_bound) ) return result, { "surface_model": "bounded_robust_single_plane_presentation", "fit_points": int(fit_mask.sum()), "plane_normal": [float(value) for value in plane[0]], "plane_d": float(plane[1]), "presentation_pixels": int(target.sum()), "invalid_plane_pixels_replaced": int(invalid.sum()), "bounded_plane_pixels": int(clamped.sum()), "depth_bounds": [lower_bound, upper_bound], "presentation_depth_policy": ( "visible_target_plane_extended_over_nonmetric_presentation_mask" ), "metric_geometry": False, "safe_passage_claim": False, "human_review_required": True, } def load_ply(path: Path) -> tuple[np.ndarray, np.ndarray, np.ndarray]: mesh = trimesh.load(path, process=False) if not isinstance(mesh, trimesh.Trimesh): raise ValueError(f"Expected one triangle mesh in {path}") vertices = np.asarray(mesh.vertices, dtype=np.float32) faces = np.asarray(mesh.faces, dtype=np.int64) vertex_colors = np.asarray(mesh.visual.vertex_colors) if vertex_colors.ndim == 2 and vertex_colors.shape[0] == len(vertices): colors = vertex_colors[:, :3].astype(np.uint8) else: colors = np.full((len(vertices), 3), (190, 190, 185), dtype=np.uint8) return vertices, colors, faces def export_glb( path: Path, vertices: np.ndarray, colors: np.ndarray, faces: np.ndarray, ) -> dict[str, Any]: rgba = np.concatenate( [colors.astype(np.uint8), np.full((len(colors), 1), 255, dtype=np.uint8)], axis=1, ) mesh = trimesh.Trimesh( vertices=vertices, faces=faces, vertex_colors=rgba, process=False, ) path.parent.mkdir(parents=True, exist_ok=True) mesh.export(path) return { "watertight": bool(mesh.is_watertight), "euler_number": int(mesh.euler_number), "bounds": np.asarray(mesh.bounds).round(6).tolist(), } def save_variant( output_dir: Path, name: str, title: str, vertices: np.ndarray, colors: np.ndarray, faces: np.ndarray, *, sample_id: str, category: str, frames: int, duration_ms: int, size: tuple[int, int], pitch: float, source_ply: Path | None = None, render_outputs: bool = True, ) -> dict[str, Any]: variant_dir = output_dir / name variant_dir.mkdir(parents=True, exist_ok=True) ply_path = source_ply or variant_dir / f"{name}.ply" if source_ply is None: write_ply(ply_path, vertices, colors, faces) gif_path = variant_dir / f"{name}_slow.gif" preview_path = variant_dir / f"{name}_preview.jpg" glb_path = variant_dir / f"{name}.glb" if render_outputs: write_turntable_gif( gif_path, vertices, colors, faces, sample_id=sample_id, category=f"{category} | {title}", frames=frames, size=size, pitch=pitch, duration_ms=duration_ms, ) render_mesh_frame( vertices, colors, faces, size=size, yaw=35.0, pitch=pitch, title=f"{sample_id} | {title}", ).save(preview_path, quality=94) glb_info = export_glb(glb_path, vertices, colors, faces) return { "name": name, "title": title, "vertices": int(len(vertices)), "faces": int(len(faces)), "ply": str(ply_path), "glb": str(glb_path), "gif": str(gif_path) if render_outputs else None, "preview": str(preview_path) if render_outputs else None, **glb_info, } def build_contact_sheet(path: Path, rows: list[dict[str, Any]]) -> None: previews = [Image.open(row["preview"]).convert("RGB") for row in rows] width = max(image.width for image in previews) label_height = 42 height = sum(image.height + label_height for image in previews) sheet = Image.new("RGB", (width, height), "white") draw = ImageDraw.Draw(sheet) top = 0 for image, row in zip(previews, rows): sheet.paste(image, (0, top + label_height)) draw.text( (10, top + 10), f"{row['title']} | {row['vertices']} vertices | {row['faces']} faces", fill=(20, 20, 20), ) top += image.height + label_height path.parent.mkdir(parents=True, exist_ok=True) sheet.save(path, quality=94) def portable_path(path: str | Path, anchor: Path) -> str: return Path(os.path.relpath(Path(path).resolve(), anchor.resolve())).as_posix() def portable_variant_row(row: dict[str, Any], anchor: Path) -> dict[str, Any]: portable = dict(row) for key in ("ply", "glb", "gif", "preview"): value = portable.get(key) if value: portable[key] = portable_path(value, anchor) return portable def sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for block in iter(lambda: handle.read(1024 * 1024), b""): digest.update(block) return digest.hexdigest() def portable_file_record(path: Path, anchor: Path) -> dict[str, Any]: resolved = path.expanduser().resolve() return { "path": portable_path(resolved, anchor), "sha256": sha256_file(resolved), "bytes": int(resolved.stat().st_size), } def resolve_declared_path(value: Any, base_dir: Path) -> Path | None: if not isinstance(value, str) or not value: return None path = Path(value).expanduser() return path if path.is_absolute() else base_dir / path def _portable_model_identity(value: Any) -> dict[str, Any] | None: if not isinstance(value, dict): return None allowed = { "model_id", "family", "encoder", "metric_training_dataset", "checkpoint_filename", "checkpoint_sha256", "selection_policy", "license", "third_party", "revision", "revision_review_required_before_public_release", } identity = { key: item for key, item in value.items() if key in allowed and isinstance(item, (str, int, float, bool, type(None))) } return identity or None def load_depth_context( geometry_dir: Path, ) -> tuple[dict[str, Any], Path | None]: """Read the depth sidecar without promoting monocular scale to truth.""" candidates = ( geometry_dir.parent / "depth_manifest.json", geometry_dir / "depth_manifest.json", ) manifest_path = next((path for path in candidates if path.is_file()), None) payload: dict[str, Any] = {} if manifest_path is not None: value = read_json(manifest_path) if not isinstance(value, dict): raise ValueError(f"Depth manifest must contain one JSON object: {manifest_path}") payload = value raw_model = payload.get("model") model = raw_model if isinstance(raw_model, str) else None if model and Path(model).is_absolute(): model = Path(model).name checkpoint = payload.get("checkpoint") checkpoint_path = resolve_declared_path( checkpoint, manifest_path.parent if manifest_path is not None else geometry_dir, ) checkpoint_name = ( Path(checkpoint).name if isinstance(checkpoint, str) and checkpoint else None ) checkpoint_sha256 = ( sha256_file(checkpoint_path) if checkpoint_path is not None and checkpoint_path.is_file() else None ) model_identity = _portable_model_identity(payload.get("model_identity")) if model_identity is None and model: # Older depth manifests predate the structured identity object. Keep # their declared model identifier instead of silently dropping it. model_identity = {"model_id": model} if ( model_identity is None and payload.get("engine") == "depth_anything_v2" and checkpoint_name == "depth_anything_v2_metric_hypersim_vitl.pth" ): model_identity = { "model_id": "Depth-Anything-V2-Metric-Hypersim-Large", "family": "Depth Anything V2", "encoder": payload.get("encoder") or "vitl", "metric_training_dataset": "Hypersim", "checkpoint_filename": checkpoint_name, "checkpoint_sha256": checkpoint_sha256, "selection_policy": ( "best_available_local_single_image_metric_depth_model" ), } metric_depth_value = payload.get("metric_depth") if metric_depth_value not in (None, True, False): raise ValueError( "depth_manifest.metric_depth must be a JSON boolean or null: " f"{manifest_path}" ) metric_style = metric_depth_value is True return { "manifest_available": manifest_path is not None, "engine": payload.get("engine"), "model": model, "model_identity": model_identity, "checkpoint_filename": checkpoint_name, "checkpoint_sha256": checkpoint_sha256, "encoder": payload.get("encoder"), "metric_depth_manifest_value": metric_depth_value, "metric_style_depth": metric_style, "maximum_model_depth": payload.get("max_depth") if metric_style else None, "calibrated_metric_truth": False, "scale_policy": ( "metric_style_monocular_output_not_calibrated_truth" if metric_style else "relative_or_unknown_depth_scale" ), }, manifest_path def build_argument_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--geometry-dir", required=True) parser.add_argument("--output-dir", required=True) parser.add_argument("--sample-id", default="single_mesh") parser.add_argument("--category", default="stairs") parser.add_argument("--frames", type=int, default=72) parser.add_argument("--duration-ms", type=int, default=140) parser.add_argument("--width", type=int, default=640) parser.add_argument("--height", type=int, default=480) parser.add_argument("--pitch", type=float, default=-25.0) parser.add_argument("--stride", type=int, default=6) parser.add_argument("--thickness-ratio", type=float, default=0.12) parser.add_argument("--stair-cross-samples", type=int, default=48) parser.add_argument("--stair-step-count", type=int, default=0) parser.add_argument( "--texture-image", default=None, help=( "Optional frozen RGB texture. When supplied it is required and takes " "strict precedence over every discovered completion/fallback image." ), ) parser.add_argument( "--hidden-tint", type=float, default=0.0, help="Hidden-region review tint in [0,1]. Default 0 preserves source texture.", ) parser.add_argument( "--include-depth-solid", action="store_true", help=( "Also render the experimental depth-extruded solid. It is disabled by " "default because long side-wall triangles are not publication quality." ), ) parser.add_argument( "--model-only", action="store_true", help=( "Build PLY/GLB presentation models without duplicate CPU GIFs; " "use the dedicated GPU turntable renderer for final review media." ), ) return parser def main() -> int: parser = build_argument_parser() args = parser.parse_args() geometry_dir = Path(args.geometry_dir).expanduser().resolve() output_dir = Path(args.output_dir).expanduser().resolve() texture_image = ( Path(args.texture_image).expanduser().resolve() if args.texture_image else None ) required = [ geometry_dir / "completed_mesh.ply", geometry_dir / "completed_depth.npy", geometry_dir / "amodal_target_mask.png", ] if texture_image is not None: required.append(texture_image) missing = [str(path) for path in required if not path.is_file()] if missing: parser.error("missing geometry input(s): " + ", ".join(missing)) if args.frames < 12 or args.duration_ms < 20: parser.error("use at least 12 frames and 20 ms per frame") if not 0.0 <= args.hidden_tint <= 1.0: parser.error("--hidden-tint must be in [0,1]") depth = np.load(geometry_dir / "completed_depth.npy").astype(np.float32) shape = depth.shape target_mask = clean_target_mask( read_mask(geometry_dir / "amodal_target_mask.png", shape), close_kernel=5, keep_largest=True, ) visible_path = geometry_dir / "target_visible_mask.png" visible_mask = read_mask(visible_path, shape) if visible_path.is_file() else None colors, texture_source = prepare_texture( geometry_dir.parent, geometry_dir, shape, hidden_tint=args.hidden_tint, texture_override=texture_image, ) selected_texture_path = Path(texture_source.split(" + ", 1)[0]).resolve() if ( texture_image is not None and selected_texture_path != texture_image ): raise RuntimeError( "Explicit --texture-image lost strict priority during texture selection" ) geometry_manifest_path = geometry_dir / "geometry_manifest.json" geometry_manifest = ( read_json(geometry_manifest_path) if geometry_manifest_path.is_file() else {} ) depth_context, depth_manifest_path = load_depth_context(geometry_dir) metric_style_depth = bool(depth_context["metric_style_depth"]) depth_manifest = ( read_json(depth_manifest_path) if depth_manifest_path is not None else {} ) manifest_depth_output = resolve_declared_path( depth_manifest.get("output_depth"), depth_manifest_path.parent if depth_manifest_path is not None else geometry_dir, ) geometry_depth_path = resolve_declared_path( geometry_manifest.get("depth_source"), geometry_manifest_path.parent, ) depth_context["geometry_depth_source_matches_manifest"] = ( geometry_depth_path.resolve() == manifest_depth_output.resolve() if geometry_depth_path is not None and manifest_depth_output is not None else None ) depth_context["manifest_output_depth_filename"] = ( manifest_depth_output.name if manifest_depth_output is not None else None ) depth_context["geometry_depth_source_filename"] = ( geometry_depth_path.name if geometry_depth_path is not None else None ) stair_edges, stair_edge_provenance = ( trusted_stair_edges_from_manifest(geometry_manifest) ) stair_slope = float(geometry_manifest.get("stair_edge_slope", 0.0)) size = (args.width, args.height) output_dir.mkdir(parents=True, exist_ok=True) variants: list[dict[str, Any]] = [] source_mesh_path = geometry_dir / "completed_mesh.ply" source_vertices, source_colors, source_faces = load_ply(source_mesh_path) variants.append( save_variant( output_dir, "01_surface_faces", "Original continuous triangle faces", source_vertices, source_colors, source_faces, sample_id=args.sample_id, category=args.category, frames=args.frames, duration_ms=args.duration_ms, size=size, pitch=args.pitch, source_ply=source_mesh_path, render_outputs=not args.model_only, ) ) if args.include_depth_solid: if args.category == "stairs" and stair_edges: solid_vertices, solid_colors, solid_faces, solid_metadata = build_continuous_stair_mesh( depth, colors, target_mask, stride=args.stride, thickness=None, thickness_ratio=args.thickness_ratio, support_mode="stair_sidewall", wedge_base_drop_ratio=0.12, stair_edges_y=stair_edges, stair_edge_slope=stair_slope, cross_samples=args.stair_cross_samples, ) solid_title = "Experimental depth-extruded stairs" else: solid_vertices, solid_colors, solid_faces, solid_metadata = build_solid_mesh( depth, colors, target_mask, stride=args.stride, max_depth_jump=None, thickness=None, thickness_ratio=args.thickness_ratio, support_mode="boundary", wedge_base_drop_ratio=0.12, category=args.category, stair_edges_y=stair_edges, ) solid_title = "Experimental depth-extruded surface" solid_row = save_variant( output_dir, "debug_depth_solid", solid_title, solid_vertices, solid_colors, solid_faces, sample_id=args.sample_id, category=args.category, frames=args.frames, duration_ms=args.duration_ms, size=size, pitch=args.pitch, render_outputs=not args.model_only, ) solid_row["construction"] = solid_metadata solid_row["publication_ready"] = False variants.append(solid_row) if args.category == "stairs": ( canonical_vertices, canonical_colors, canonical_faces, canonical_metadata, ) = build_canonical_staircase_prism_mesh( depth, colors, target_mask, stride=args.stride, thickness=None, thickness_ratio=args.thickness_ratio, stair_edges_y=stair_edges, stair_edge_slope=stair_slope, step_count=args.stair_step_count, metric_style_depth=metric_style_depth, ) canonical_row = save_variant( output_dir, "02_canonical_stairs", "Depth-scaled textured open-world staircase", canonical_vertices, canonical_colors, canonical_faces, sample_id=args.sample_id, category=args.category, frames=args.frames, duration_ms=args.duration_ms, size=size, pitch=args.pitch, render_outputs=not args.model_only, ) canonical_row["construction"] = canonical_metadata canonical_row["publication_ready"] = True variants.append(canonical_row) elif args.category in CONTINUOUS_SURFACE_CATEGORIES: presentation_mask, presentation_mask_metadata = ( continuous_presentation_mask(target_mask) ) continuous_depth, surface_metadata = idealized_continuous_presentation_depth( depth, presentation_mask, visible_mask, ) presentation_depth_bounds = surface_metadata.get("depth_bounds", [0.0, 1.0]) presentation_depth_jump_limit = max( 1.0, float(presentation_depth_bounds[1]) - float(presentation_depth_bounds[0]) + 1e-3, ) ( support_vertices, support_colors, support_faces, support_metadata, ) = build_solid_mesh( continuous_depth, colors, presentation_mask, stride=args.stride, max_depth_jump=presentation_depth_jump_limit, thickness=None, thickness_ratio=args.thickness_ratio, support_mode="vertical_slab", wedge_base_drop_ratio=0.12, category=args.category, stair_edges_y=None, ) support_row = save_variant( output_dir, "02_continuous_support_surface", "Category-constrained continuous support surface", support_vertices, support_colors, support_faces, sample_id=args.sample_id, category=args.category, frames=args.frames, duration_ms=args.duration_ms, size=size, pitch=args.pitch, render_outputs=not args.model_only, ) support_row["construction"] = { **support_metadata, **surface_metadata, "presentation_mask": presentation_mask_metadata, "role": "nonmetric_category_constrained_geometry_hypothesis", "safe_passage_claim": False, "human_review_required": True, } support_row["publication_ready"] = True variants.append(support_row) contact_sheet = output_dir / "3d_variants_contact_sheet.jpg" if not args.model_only: build_contact_sheet(contact_sheet, variants) texture_parts = texture_source.split(" + ") portable_texture_source = " + ".join( [portable_path(texture_parts[0], output_dir), *texture_parts[1:]] ) selected_texture_record = portable_file_record( selected_texture_path, output_dir, ) explicit_texture_record = ( portable_file_record(texture_image, output_dir) if texture_image is not None else None ) if depth_manifest_path is not None: depth_context["manifest"] = portable_path(depth_manifest_path, output_dir) else: depth_context["manifest"] = None portable_variants = [ portable_variant_row(row, output_dir) for row in variants ] canonical_construction = next( ( row.get("construction", {}) for row in variants if row.get("name") == "02_canonical_stairs" ), {}, ) manifest = { "schema_version": "accessibilityamodal_3d_presentation_variants_v2", "sample_id": args.sample_id, "category": args.category, "geometry_dir": portable_path(geometry_dir, output_dir), "texture_source": portable_texture_source, "texture": { "selection_policy": ( "explicit_frozen_texture_strict_priority" if texture_image is not None else "pipeline_texture_discovery" ), "explicit_frozen_texture": explicit_texture_record, "selected_texture": selected_texture_record, "selected_source_description": portable_texture_source, "hidden_tint": float(args.hidden_tint), "source_file_modified": False, "used_for_vertex_colors": True, }, "geometry": { "source_geometry_manifest": ( portable_path(geometry_manifest_path, output_dir) if geometry_manifest_path.is_file() else None ), "geometry_mode": geometry_manifest.get("geometry_mode"), "depth": depth_context, "metric_style_depth": metric_style_depth, "calibrated_metric_truth": False, "stair_edge_rows": stair_edges, "stair_edge_provenance": stair_edge_provenance, "stair_edge_source": ( geometry_manifest.get("stair_edge_source") or geometry_manifest.get("edge_source") ), "stair_step_count": canonical_construction.get("stair_step_count"), "stair_step_count_source": canonical_construction.get( "stair_step_count_source" ), "role": "open_world_accessibility_surface_hypothesis", }, "depth_model_identity": depth_context.get("model_identity"), "metric_style_depth": metric_style_depth, "frames": args.frames, "duration_ms": args.duration_ms, "seconds_per_rotation": round(args.frames * args.duration_ms / 1000.0, 3), "contact_sheet": ( portable_path(contact_sheet, output_dir) if not args.model_only else None ), "model_only": bool(args.model_only), "variants": portable_variants, "publication_variants": [ row["name"] for row in variants if row.get("publication_ready", True) ], "experimental_depth_solid_included": args.include_depth_solid, "depth_is_metric_ground_truth": False, "solid_models_are_visual_geometry_hypotheses": True, "open_world_semantics": { "scene_role": "local_accessibility_surface_patch", "freestanding_object_model": False, "context_landings": canonical_construction.get("context_landings"), "automatic_passability_claim": False, "human_review_required": True, }, "path_policy": "All filesystem paths are relative to manifest.json.", } (output_dir / "manifest.json").write_text( json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" ) print(json.dumps(manifest, ensure_ascii=False, indent=2)) return 0 if __name__ == "__main__": raise SystemExit(main())