// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 #pragma once #include #include #include #include "../geometry.h" #define MODE_GEOMETRY 0x02ull #define MODE_CHILDREN 0x00ull #define DIM_X 0x0ull #define DIM_Y 0x1ull static const size_t INVALID_IDX = -1; template struct NMS_BoundsWrapper { typedef std::unique_ptr Ptr; typedef AABB_ bds_t; size_t GeoIdx; const T *Geometry; bds_t Bounds; NMS_BoundsWrapper(size_t geoIdx, const T *geometry) : GeoIdx(geoIdx), Geometry(geometry), Bounds(geometry->Bounds()) { } }; template class NMS_NodeAllocator; template class NMS_KDTree; template class NMS_BuildCache; template class NMS_KDNode { friend class NMS_KDTree; public: typedef NMS_BoundsWrapper bds_t; typedef std::unique_ptr UPtr; typedef typename T::inner_type inner_type; typedef std::vector geo_vec_t; typedef std::unique_ptr geo_vec_ptr; void Build(geo_vec_ptr geometries, const typename bds_t::bds_t &envelope, NMS_NodeAllocator &allocator, NMS_BuildCache &buildCache); template void FindIntersections(size_t geoIdx, const typename bds_t::bds_t &bds, const Fn &fn) const; private: inline uintptr_t Dim() const { return reinterpret_cast(m_ptr) & 0x01ull; } inline uintptr_t Mode() const { return reinterpret_cast(m_ptr) & 0x02ull; } inline void Children(NMS_KDNode *&children, inner_type &splitPos) const { auto vPtr = Geometries(); splitPos = *reinterpret_cast(vPtr); children = reinterpret_cast(vPtr + sizeof(inner_type)); } inline uint8_t* Geometries() const { return reinterpret_cast(reinterpret_cast(m_ptr) & ~0x3ull); } inline void SetPtr(uint8_t *vPtr, uintptr_t mode, uintptr_t dim) { m_ptr = reinterpret_cast( reinterpret_cast(vPtr) | mode | dim ); } void AssignGeometries(geo_vec_ptr geometries, NMS_BuildCache &buildCache); uint8_t *m_ptr; }; template class NMS_NodeAllocator { public: typedef NMS_KDNode node_t; typedef typename node_t::inner_type inner_type; NMS_NodeAllocator(size_t initialGuess = 512); ~NMS_NodeAllocator(); void Get(size_t numNodes, NMS_KDNode *&outNodes, inner_type *&outSplitPos, uint8_t *&outRawPtr); private: std::vector> m_buffers; size_t m_offset; }; template class NMS_BuildCache { public: typedef typename NMS_KDNode::bds_t bds_t; typedef std::unique_ptr Ptr; typedef std::vector geo_vec_t; typedef std::unique_ptr geo_vec_ptr; NMS_BuildCache(size_t initialSize); ~NMS_BuildCache(); geo_vec_ptr Get(size_t sizeHint); bds_t** GetRawBuffer(size_t numGeos, uint8_t *&rawPtr); void Release(geo_vec_ptr buff); private: std::stack m_cache; std::vector> m_rawBuffers; size_t m_rawOffset; }; template class NMS_KDTree { typedef typename T::inner_type inner_type; typedef NMS_BoundsWrapper bds_t; typedef NMS_KDNode node_t; public: NMS_KDTree(); ~NMS_KDTree(); void Build(const std::vector &geometries); template void FindIntersections(size_t geoIdx, const Fn &fn) const; template void FindIntersections(const T &geo, const Fn &fn) const; private: bds_t *m_wrappers; NMS_NodeAllocator m_allocator; node_t m_root; typename NMS_BuildCache::Ptr m_buildCache; }; template NMS_KDTree::NMS_KDTree() : m_wrappers(nullptr) { m_root.m_ptr = nullptr; } template NMS_KDTree::~NMS_KDTree() { free(m_wrappers); } template void NMS_KDTree::Build(const std::vector &geometries) { if (geometries.empty()) { m_root.m_ptr = nullptr; return; } // Doing this so that we can perform placement-new on the array buffer, and thus // can only perform a single memory allocation for all geometries at once m_wrappers = reinterpret_cast(malloc(sizeof(bds_t) * geometries.size())); m_buildCache.reset(new NMS_BuildCache(geometries.size())); auto bdsGeos = m_buildCache->Get(geometries.size()); typename bds_t::bds_t envelope; for (size_t i = 0; i < geometries.size(); ++i) { // Placement new. Constructs the object in the place specified in the first (...) new (m_wrappers + i) bds_t(i, &geometries[i]); bdsGeos->push_back(m_wrappers + i); if (i == 0) { envelope = m_wrappers[i].Bounds; } else { envelope = envelope.Union(m_wrappers[i].Bounds); } } m_root.Build(std::move(bdsGeos), envelope, m_allocator, *m_buildCache); } template void NMS_KDNode::Build(geo_vec_ptr geometries, const typename bds_t::bds_t &envelope, NMS_NodeAllocator &allocator, NMS_BuildCache &buildCache) { static const size_t MAX_GEOMETRIES = 8; if (geometries->size() <= MAX_GEOMETRIES) { AssignGeometries(std::move(geometries), buildCache); } else { geo_vec_ptr leftGeos = buildCache.Get(geometries->size()), rightGeos = buildCache.Get(geometries->size()); inner_type szX = envelope[2] - envelope[0]; inner_type szY = envelope[3] - envelope[1]; int64_t dim = szX > szY ? 0 : 1; auto emn = envelope[dim]; auto emx = envelope[dim + 2]; auto pivotPos = (emn + emx) / 2; for (bds_t *g : *geometries) { auto mn = g->Bounds[dim]; auto mx = g->Bounds[dim + 2]; if (mn < pivotPos) { leftGeos->push_back(g); } if (mx > pivotPos) { rightGeos->push_back(g); } } if (leftGeos->size() == geometries->size() || rightGeos->size() == geometries->size()) { AssignGeometries(std::move(geometries), buildCache); buildCache.Release(std::move(leftGeos)); buildCache.Release(std::move(rightGeos)); } else { buildCache.Release(std::move(geometries)); inner_type *nodeSplitPos; uint8_t *nodeRawPtr; NMS_KDNode *children; allocator.Get(2, children, nodeSplitPos, nodeRawPtr); SetPtr(nodeRawPtr, MODE_CHILDREN, dim); *nodeSplitPos = pivotPos; typename bds_t::bds_t leftEnv{envelope}, rightEnv{envelope}; // Set the max of the left envelope to the split plane leftEnv[dim + 2] = pivotPos; // Set the min of the right envelope to the split plane rightEnv[dim] = pivotPos; children[0].Build(std::move(leftGeos), leftEnv, allocator, buildCache); children[1].Build(std::move(rightGeos), rightEnv, allocator, buildCache); } } } template void NMS_KDNode::AssignGeometries(geo_vec_ptr geometries, NMS_BuildCache &buildCache) { if (geometries->empty()) { SetPtr(nullptr, MODE_GEOMETRY, 0); } else { uint8_t *vPtr; bds_t **geoPtr = buildCache.GetRawBuffer(geometries->size(), vPtr); std::copy(geometries->begin(), geometries->end(), geoPtr); SetPtr(vPtr, MODE_GEOMETRY, 0); } buildCache.Release(std::move(geometries)); } template template void NMS_KDTree::FindIntersections(size_t geoIdx, const Fn &fn) const { if (!m_wrappers) return; auto &bds = m_wrappers[geoIdx].Bounds; m_root.FindIntersections(geoIdx, bds, fn); } template template void NMS_KDTree::FindIntersections(const T &geo, const Fn &fn) const { if (!m_wrappers) return; NMS_BoundsWrapper bdsWrapper(INVALID_IDX, &geo); m_root.FindIntersections(INVALID_IDX, bdsWrapper.Bounds, fn); } template template void NMS_KDNode::FindIntersections(size_t geoIdx, const typename bds_t::bds_t &bds, const Fn &fn) const { auto mode = Mode(); if (mode == MODE_GEOMETRY) { auto *vPtr = Geometries(); size_t numGeos = *reinterpret_cast(vPtr); bds_t **geoPtr = reinterpret_cast(vPtr + sizeof(size_t)); bds_t **endPtr = geoPtr + numGeos; for (; geoPtr != endPtr; ++geoPtr) { const bds_t *child = *geoPtr; // Don't compute this against self if (geoIdx != INVALID_IDX && child->GeoIdx <= geoIdx) continue; typename bds_t::bds_t::inner_type pctN, pctM, iou; std::tie(pctN, pctM, iou) = geometry_region_sizes(bds, child->Bounds); if (iou > 0) { fn(child->GeoIdx, pctN, pctM, iou); } } } else { auto dim = Dim(); auto mn = bds[dim]; auto mx = bds[dim + 2]; NMS_KDNode *children; inner_type splitPos; Children(children, splitPos); if (mn < splitPos) { children[0].FindIntersections(geoIdx, bds, fn); } if (mx > splitPos) { children[1].FindIntersections(geoIdx, bds, fn); } } } template NMS_NodeAllocator::NMS_NodeAllocator(size_t initialGuess) : m_offset(0) { size_t allocSize = initialGuess * (sizeof(inner_type) + 2 * sizeof(node_t)); auto ptr = reinterpret_cast(malloc(allocSize)); m_buffers.emplace_back(initialGuess, ptr); } template NMS_NodeAllocator::~NMS_NodeAllocator() { for (auto &p : m_buffers) { free(p.second); } } template void NMS_NodeAllocator::Get(size_t numNodes, node_t *&outNodes, inner_type *&outSplitPos, uint8_t *&outRawPtr) { auto &currBuff = m_buffers.back(); size_t rem = currBuff.first - m_offset; size_t reqSize = sizeof(inner_type) + sizeof(node_t) * numNodes; if (rem >= reqSize) { outRawPtr = currBuff.second + m_offset; outSplitPos = reinterpret_cast(outRawPtr); outNodes = reinterpret_cast(outRawPtr + sizeof(inner_type)); m_offset += reqSize; return; } // Rounds up to the nearest factor of 2 size_t allocSize = (std::max(currBuff.first * 2, reqSize) + 1) & ~0x01ull; auto ptr = reinterpret_cast(malloc(allocSize)); m_buffers.emplace_back(allocSize, ptr); m_offset = 0; Get(numNodes, outNodes, outSplitPos, outRawPtr); } template NMS_BuildCache::NMS_BuildCache(size_t initialSize) : m_rawOffset(0) { auto allocSize = sizeof(bds_t*) * initialSize * 2; auto raw1 = reinterpret_cast(malloc(allocSize)); m_rawBuffers.emplace_back(allocSize, raw1); } template NMS_BuildCache::~NMS_BuildCache() { for (auto &p : m_rawBuffers) { free(p.second); } } template typename NMS_BuildCache::geo_vec_ptr NMS_BuildCache::Get(size_t sizeHint) { geo_vec_ptr ret; if (! m_cache.empty()) { ret = std::move(m_cache.top()); m_cache.pop(); ret->clear(); } else { ret.reset(new std::vector); } ret->reserve(sizeHint); return ret; } template typename NMS_BuildCache::bds_t** NMS_BuildCache::GetRawBuffer(size_t numGeos, uint8_t *&rawPtr) { auto &currBuff = m_rawBuffers.back(); size_t rem = currBuff.first - m_rawOffset; size_t reqSize = sizeof(size_t) + sizeof(bds_t*) * numGeos; if (rem >= reqSize) { rawPtr = currBuff.second + m_rawOffset; m_rawOffset += reqSize; reinterpret_cast(rawPtr)[0] = numGeos; return reinterpret_cast(rawPtr + sizeof(size_t)); } size_t allocSize = (std::max(currBuff.first * 2, reqSize) + 1) & ~0x01ull; auto ptr = reinterpret_cast(malloc(allocSize)); m_rawBuffers.emplace_back(allocSize, ptr); m_rawOffset = 0; return GetRawBuffer(numGeos, rawPtr); } template void NMS_BuildCache::Release(geo_vec_ptr buff) { m_cache.push(std::move(buff)); } #undef MODE_GEOMETRY #undef MODE_CHILDREN #undef DIM_X #undef DIM_Y