| { |
| "benchmark_metadata": { |
| "name": "Warehouse Pallet Detection Benchmark Comparison", |
| "description": "Cross-study comparison of pallet detection models. No standard benchmark exists for this task, so results are collected from published literature on comparable (but not identical) warehouse pallet detection datasets.", |
| "created": "2026-03-08", |
| "notes": "Direct comparison is limited because each study uses its own private dataset with different imaging conditions, pallet types, and annotation guidelines. Values marked null were not reported in the source publication." |
| }, |
| "our_models": { |
| "description": "YOLOv11 nano and small variants share the same dataset, augmentation pipeline, and hyperparameters. Only the base model architecture differs. Medium, large, and extra-large variants were tested but showed no accuracy improvement over small.", |
| "classes": [ |
| "pallet" |
| ], |
| "num_classes": 1, |
| "input_resolutions": [ |
| 320, |
| 640, |
| 960, |
| 1280 |
| ], |
| "resolution_coverage": { |
| "nano": [320, 640, 960, 1280], |
| "small": [640, 1280] |
| }, |
| "framework": "Ultralytics", |
| "dataset": { |
| "name": "EDITools Warehouse Pallet Dataset", |
| "total_images": 5000, |
| "train_ratio": 0.8, |
| "val_ratio": 0.15, |
| "test_ratio": 0.05, |
| "note": "Uses full available dataset with no image cap. Exact counts depend on labeled images at training time.", |
| "source": "Real warehouse operational footage", |
| "labeling_method": "VLM auto-labeling (Qwen3.5 9B) + human review", |
| "label_format": "YOLO (class_id x_center y_center width height, normalized 0-1)", |
| "pallet_definition": "Complete unit: wooden skid base + all products stacked on top", |
| "inclusion_criteria": [ |
| "Foreground pallets fully within frame", |
| "Shape and location identifiable (blur acceptable)", |
| "Pallets on forklift forks if mostly visible" |
| ], |
| "exclusion_criteria": [ |
| "Partially cut off by image border", |
| "Background pallets too small to distinguish", |
| "Fully occluded by another pallet" |
| ], |
| "negative_examples": true, |
| "estimated_avg_pallets_per_positive_image": 2.6, |
| "train_images": 4000, |
| "val_images": 750, |
| "test_images": 250 |
| }, |
| "training": { |
| "epochs": 100, |
| "early_stopping_patience": 20, |
| "batch_size": 16, |
| "augmentation": { |
| "hsv_h": 0.015, |
| "hsv_s": 0.7, |
| "hsv_v": 0.4, |
| "degrees": 10.0, |
| "translate": 0.1, |
| "scale": 0.5, |
| "fliplr": 0.5, |
| "mosaic": 1.0 |
| } |
| }, |
| "variants": [ |
| { |
| "name": "YOLOv11n Warehouse Pallet Detector (320p)", |
| "variant": "nano", |
| "variant_code": "n", |
| "resolution": 320, |
| "base_model": "yolo11n.pt", |
| "parameters_millions": 2.6, |
| "model_size_mb": 5.2, |
| "repo_id": "EFFGRP/yolov11n-warehouse-pallets-320", |
| "best_for": "Ultra-edge microcontroller-class devices (ESP32-S3, Coral Edge TPU, RPi Zero, hobby/embedded projects)", |
| "results": { |
| "test_set": { |
| "map50": 0.6584, |
| "map50_95": 0.5426, |
| "precision": 0.6477, |
| "recall": 0.6023 |
| }, |
| "inference_time_ms": { |
| "gpu": null, |
| "cpu": null |
| } |
| } |
| }, |
| { |
| "name": "YOLOv11n Warehouse Pallet Detector (640p)", |
| "variant": "nano", |
| "variant_code": "n", |
| "resolution": 640, |
| "base_model": "yolo11n.pt", |
| "parameters_millions": 2.6, |
| "model_size_mb": 5.2, |
| "repo_id": "EFFGRP/yolov11n-warehouse-pallets-640", |
| "best_for": "Edge / real-time on low-power devices (Jetson Nano, RPi)", |
| "results": { |
| "test_set": { |
| "map50": 0.6739, |
| "map50_95": 0.5708, |
| "precision": 0.6582, |
| "recall": 0.6137 |
| }, |
| "inference_time_ms": { |
| "gpu": 5, |
| "cpu": 25 |
| } |
| } |
| }, |
| { |
| "name": "YOLOv11s Warehouse Pallet Detector (640p)", |
| "variant": "small", |
| "variant_code": "s", |
| "resolution": 640, |
| "base_model": "yolo11s.pt", |
| "parameters_millions": 9.4, |
| "model_size_mb": 18.3, |
| "repo_id": "EFFGRP/yolov11s-warehouse-pallets-640", |
| "best_for": "Balanced speed/accuracy for mid-range hardware", |
| "results": { |
| "test_set": { |
| "map50": 0.5918, |
| "map50_95": 0.4852, |
| "precision": 0.5988, |
| "recall": 0.5739 |
| }, |
| "inference_time_ms": { |
| "gpu": 7, |
| "cpu": 45 |
| } |
| } |
| }, |
| { |
| "name": "YOLOv11n Warehouse Pallet Detector (960p)", |
| "variant": "nano", |
| "variant_code": "n", |
| "resolution": 960, |
| "base_model": "yolo11n.pt", |
| "parameters_millions": 2.6, |
| "model_size_mb": 5.2, |
| "repo_id": "EFFGRP/yolov11n-warehouse-pallets-960", |
| "best_for": "Production middle-ground: Jetson Orin Nano, 1080p cameras downsampled, better small-object recall than 640p without 1280p latency", |
| "results": { |
| "test_set": { |
| "map50": 0.6693, |
| "map50_95": 0.571, |
| "precision": 0.6529, |
| "recall": 0.6078 |
| }, |
| "inference_time_ms": { |
| "gpu": null, |
| "cpu": null |
| } |
| } |
| }, |
| { |
| "name": "YOLOv11n Warehouse Pallet Detector (1280p)", |
| "variant": "nano", |
| "variant_code": "n", |
| "resolution": 1280, |
| "base_model": "yolo11n.pt", |
| "parameters_millions": 2.6, |
| "model_size_mb": 5.3, |
| "repo_id": "EFFGRP/yolov11n-warehouse-pallets-1280", |
| "best_for": "Edge with high-res cameras, small pallet detection", |
| "results": { |
| "test_set": { |
| "map50": 0.6618, |
| "map50_95": 0.5645, |
| "precision": 0.6539, |
| "recall": 0.5944 |
| }, |
| "inference_time_ms": { |
| "gpu": 18, |
| "cpu": 95 |
| } |
| } |
| }, |
| { |
| "name": "YOLOv11s Warehouse Pallet Detector (1280p)", |
| "variant": "small", |
| "variant_code": "s", |
| "resolution": 1280, |
| "base_model": "yolo11s.pt", |
| "parameters_millions": 9.4, |
| "model_size_mb": 18.4, |
| "repo_id": "EFFGRP/yolov11s-warehouse-pallets-1280", |
| "best_for": "Balanced speed/accuracy with high-res input", |
| "results": { |
| "test_set": { |
| "map50": 0.6801, |
| "map50_95": 0.5802, |
| "precision": 0.6465, |
| "recall": 0.6281 |
| }, |
| "inference_time_ms": { |
| "gpu": 25, |
| "cpu": 170 |
| } |
| } |
| } |
| ] |
| }, |
| "comparison_models": [ |
| { |
| "name": "YOLOv8 (Synthetic Data)", |
| "architecture": "YOLOv8", |
| "source": "Pallet Detection and Localisation From Synthetic Data", |
| "url": "https://arxiv.org/abs/2503.22965", |
| "year": 2025, |
| "dataset": { |
| "type": "Synthetic (Unity domain randomization)", |
| "total_images": null, |
| "notes": "Evaluated on real-world single-pallet scenes. High mAP reflects simpler evaluation scenarios (1-2 pallets per scene)." |
| }, |
| "results": { |
| "map50": 0.995, |
| "map50_95": null, |
| "precision": null, |
| "recall": null |
| }, |
| "comparison_notes": "Synthetic training data with evaluation on relatively simple real scenes. Not directly comparable to cluttered multi-pallet warehouse environments." |
| }, |
| { |
| "name": "YOLOv8 (Semi-Autonomous Forklift)", |
| "architecture": "YOLOv8", |
| "source": "Learning-Based Vision Systems for Semi-Autonomous Forklift Operation in Industrial Warehouse Environments", |
| "url": "https://arxiv.org/html/2511.06295", |
| "year": 2025, |
| "dataset": { |
| "type": "Custom warehouse + augmented real imagery", |
| "total_images": null, |
| "notes": "Optuna-driven hyperparameter optimization. Also tested pallet-hole detection." |
| }, |
| "results": { |
| "map50": 0.95, |
| "map50_95": null, |
| "precision": null, |
| "recall": null |
| }, |
| "comparison_notes": "Focused on forklift-mounted camera perspective with pallet and pallet-hole as separate tasks." |
| }, |
| { |
| "name": "YOLOv11 (Semi-Autonomous Forklift)", |
| "architecture": "YOLOv11", |
| "source": "Learning-Based Vision Systems for Semi-Autonomous Forklift Operation in Industrial Warehouse Environments", |
| "url": "https://arxiv.org/html/2511.06295", |
| "year": 2025, |
| "dataset": { |
| "type": "Custom warehouse + augmented real imagery", |
| "total_images": null, |
| "notes": "Same study as YOLOv8 variant. YOLOv11 showed superior precision and stable convergence." |
| }, |
| "results": { |
| "map50": null, |
| "map50_95": null, |
| "precision": 0.93, |
| "recall": null |
| }, |
| "comparison_notes": "Reported as having superior precision over YOLOv8 in the same study." |
| }, |
| { |
| "name": "Faster R-CNN", |
| "architecture": "Faster R-CNN", |
| "source": "A Comparison of Deep Learning Models for Pallet Detection in Industrial Warehouses", |
| "url": "https://ieeexplore.ieee.org/document/9266168/", |
| "year": 2020, |
| "dataset": { |
| "type": "Real industrial warehouse", |
| "total_images": 1344, |
| "notes": "RGB images with pallets on ground and on racks, various orientations and lighting." |
| }, |
| "results": { |
| "map50": 0.89, |
| "map50_95": null, |
| "precision": null, |
| "recall": null |
| }, |
| "comparison_notes": "Best performer in the 2020 comparative study. Two-stage detector, slower inference than YOLO." |
| }, |
| { |
| "name": "SSD", |
| "architecture": "SSD", |
| "source": "A Comparison of Deep Learning Models for Pallet Detection in Industrial Warehouses", |
| "url": "https://ieeexplore.ieee.org/document/9266168/", |
| "year": 2020, |
| "dataset": { |
| "type": "Real industrial warehouse", |
| "total_images": 1344, |
| "notes": "Same dataset as Faster R-CNN comparison." |
| }, |
| "results": { |
| "map50": 0.85, |
| "map50_95": null, |
| "precision": null, |
| "recall": null |
| }, |
| "comparison_notes": "Comparable to our model's mAP50 but on a different dataset and older architecture." |
| }, |
| { |
| "name": "YOLOv4", |
| "architecture": "YOLOv4", |
| "source": "A Comparison of Deep Learning Models for Pallet Detection in Industrial Warehouses", |
| "url": "https://ieeexplore.ieee.org/document/9266168/", |
| "year": 2020, |
| "dataset": { |
| "type": "Real industrial warehouse", |
| "total_images": 1344, |
| "notes": "Same dataset as Faster R-CNN and SSD comparison." |
| }, |
| "results": { |
| "map50": 0.82, |
| "map50_95": null, |
| "precision": null, |
| "recall": null |
| }, |
| "comparison_notes": "Lowest performer in the 2020 study. Significant architecture improvements in v8/v11." |
| }, |
| { |
| "name": "YOLOv8 + CBAM (Pallet Tracking)", |
| "architecture": "YOLOv8 + CBAM attention + DeepSORT", |
| "source": "Digitalization of Warehouse Management: Real-Time Pallet Tracking Using YOLO and CBAM-Enhanced Deep Learning Models", |
| "url": "https://link.springer.com/chapter/10.1007/978-3-031-91643-4_5", |
| "year": 2025, |
| "dataset": { |
| "type": "Custom warehouse", |
| "total_images": null, |
| "notes": "Focus on tracking visually similar pallets with DeepSORT. Detection + tracking pipeline." |
| }, |
| "results": { |
| "map50": null, |
| "map50_95": null, |
| "precision": null, |
| "recall": null |
| }, |
| "comparison_notes": "Tracking-focused work. Adds attention mechanism and multi-object tracking on top of YOLO detection." |
| }, |
| { |
| "name": "NVIDIA SDG Pallet Model", |
| "architecture": "Custom (NVIDIA, TensorRT optimized)", |
| "source": "NVIDIA SDG Pallet Model - Synthetic Data Generation", |
| "url": "https://github.com/NVIDIA-AI-IOT/sdg_pallet_model", |
| "year": 2024, |
| "dataset": { |
| "type": "Synthetic (NVIDIA Omniverse Replicator + OpenUSD SimReady Assets)", |
| "total_images": 25000, |
| "notes": "Trained on synthetic data. Detects pallet side face centers and corners for forklift docking. Supports wood, metal, and plastic pallets. Runs real-time on Jetson AGX Orin." |
| }, |
| "results": { |
| "map50": null, |
| "map50_95": null, |
| "precision": null, |
| "recall": null |
| }, |
| "comparison_notes": "Most production-ready alternative. Different task: detects pallet pocket/side-face keypoints for autonomous forklift docking, not full pallet unit bounding boxes. Includes pretrained ONNX model and TensorRT optimization." |
| }, |
| { |
| "name": "YOLOv8 (Synthetic Boost)", |
| "architecture": "YOLOv8", |
| "source": "Improving Pallet Detection Using Synthetic Data", |
| "url": "https://arxiv.org/html/2402.07098v1", |
| "year": 2024, |
| "dataset": { |
| "type": "Synthetic domain randomization + real data", |
| "total_images": null, |
| "notes": "Showed 69% mAP50 improvement on stacked pallets and 50% on racked pallets when augmenting with synthetic data." |
| }, |
| "results": { |
| "map50": null, |
| "map50_95": null, |
| "precision": null, |
| "recall": null |
| }, |
| "comparison_notes": "Demonstrates value of synthetic augmentation for pallet detection. Reports relative improvements rather than absolute metrics." |
| }, |
| { |
| "name": "AM-Mask R-CNN", |
| "architecture": "Mask R-CNN + Attention Mechanism", |
| "source": "Enhanced Pallet Detection: AM-Mask R-CNN", |
| "url": "https://www.tandfonline.com/doi/full/10.1080/0951192X.2025.2501587", |
| "year": 2025, |
| "dataset": { |
| "type": "Complex warehouse environments", |
| "total_images": null, |
| "notes": "Attention-enhanced Mask R-CNN for instance segmentation in challenging warehouse layouts." |
| }, |
| "results": { |
| "map50": null, |
| "map50_95": null, |
| "precision": null, |
| "recall": null |
| }, |
| "comparison_notes": "Instance segmentation approach (masks, not just boxes). Addresses complex occlusion scenarios but heavier inference cost than YOLO." |
| }, |
| { |
| "name": "YOLOv5 + ArUco Fiducial", |
| "architecture": "YOLOv5 + ArUco markers", |
| "source": "Pallet Detection and Distance Estimation with YOLO and Fiducial Marker Algorithm", |
| "url": "https://ieeexplore.ieee.org/document/10066999/", |
| "year": 2023, |
| "dataset": { |
| "type": "Custom with fiducial markers", |
| "total_images": null, |
| "notes": "Combined YOLO detection with ArUco fiducial markers for distance estimation. 24fps real-time, 2.28cm avg distance error." |
| }, |
| "results": { |
| "map50": null, |
| "map50_95": null, |
| "precision": 0.995, |
| "recall": null |
| }, |
| "comparison_notes": "High precision but uses fiducial markers on pallets (not marker-free). Distance estimation capability is unique." |
| } |
| ], |
| "existing_datasets": { |
| "hugging_face": [ |
| { |
| "name": "Francesco/lettuce-pallets", |
| "url": "https://huggingface.co/datasets/Francesco/lettuce-pallets", |
| "description": "Object detection dataset from Roboflow 100 (Nov 2022). Only directly pallet-related dataset on HF.", |
| "size": "Small" |
| }, |
| { |
| "name": "nvidia/PhysicalAI-Spatial-Intelligence-Warehouse", |
| "url": "https://huggingface.co/datasets/nvidia/PhysicalAI-Spatial-Intelligence-Warehouse", |
| "description": "Synthetic warehouse dataset by NVIDIA for 3D scene understanding. RGB-D images with object masks and spatial reasoning Q&A.", |
| "size": "Large" |
| }, |
| { |
| "name": "nvidia/PhysicalAI-SimReady-Warehouse-01", |
| "url": "https://huggingface.co/datasets/nvidia/PhysicalAI-SimReady-Warehouse-01", |
| "description": "753 OpenUSD 3D assets for IsaacSim depicting an industrial warehouse. ~15GB. For robotic manipulation.", |
| "size": "15GB" |
| } |
| ], |
| "roboflow": [ |
| { |
| "name": "Roboflow Pallets Category", |
| "url": "https://universe.roboflow.com/browse/logistics/pallets", |
| "description": "1,755+ community-contributed pallet images across multiple projects.", |
| "size": "1,755+ images" |
| }, |
| { |
| "name": "Pallet Detection by sundharesan-kumaresan", |
| "url": "https://universe.roboflow.com/sundharesan-kumaresan/pallet-detection-ith6b", |
| "description": "3,304 images with pre-trained model and API endpoint.", |
| "size": "3,304 images" |
| } |
| ], |
| "other": [ |
| { |
| "name": "Amazon ARMBench", |
| "url": "https://www.armbench.com/", |
| "description": "Large-scale warehouse manipulation benchmark. 450K+ segmentation labels, 50K+ images, 200K+ unique objects. CC BY 4.0.", |
| "size": "450K+ labels" |
| }, |
| { |
| "name": "EmaroLab PDT Dataset", |
| "url": "https://github.com/EmaroLab/PDT", |
| "description": "565 2D laser scans from real warehouse environments for pallet tracking with Faster R-CNN.", |
| "size": "565 scans" |
| } |
| ] |
| }, |
| "gap_analysis": { |
| "no_standard_benchmark": "Unlike COCO or Pascal VOC for general object detection, there is no established standard benchmark dataset for warehouse pallet detection. Each published study uses its own private dataset.", |
| "no_hf_model": "No dedicated pallet detection model existed on Hugging Face prior to this publication. The HF Hub has ~21 models tagged 'logistics' but none are pallet-specific.", |
| "dataset_variability": "Datasets differ in pallet definition (wooden base only vs. full unit with products), imaging conditions (static camera vs. forklift-mounted vs. drone), warehouse type (cold storage, dry goods, outdoor yard), and annotation granularity.", |
| "metrics_inconsistency": "Many studies report only mAP@0.5 without mAP@0.5:0.95, precision, or recall, making thorough comparison difficult.", |
| "synthetic_vs_real_gap": "Synthetic data models report very high mAP but real-world deployment performance is less documented.", |
| "pallet_type_gap": "No benchmark tests generalization across pallet types (wood, plastic, metal, EUR, US standard).", |
| "pocket_detection_gap": "Pallet pocket/hole detection for forklift docking has much lower accuracy (~72%) than pallet face detection (~95%) and is underserved.", |
| "edge_deployment_gap": "No standardized latency/accuracy benchmarks exist for pallet detection on edge devices (Jetson, etc.).", |
| "recommendation": "The community would benefit from a shared, standardized warehouse pallet detection benchmark with agreed-upon annotation guidelines, diverse warehouse environments, and consistent evaluation metrics (mAP@0.5, mAP@0.5:0.95, precision, recall, inference speed)." |
| } |
| } |
|
|
