(function attachBoundsEngine(root) { const ENGINE_VERSION = "1.1.0"; const BATCH_SEARCH_CAP = 10000; const STATUS_CODES = Object.freeze({ FITS: "fits", RESIDENT_NOT_FIT: "resident_not_fit", NO_SESSION_CAPACITY: "no_session_capacity", BELOW_FLOOR: "below_floor", MEETS_FLOOR: "meets_floor", EXCEEDS_MEMORY_FIT: "exceeds_memory_fit", NOT_EVALUATED: "not_evaluated", UNSUPPORTED_PROFILE: "unsupported_profile" }); const PROFILE_STATUS_CODES = Object.freeze({ OK: "ok", UNSUPPORTED_PROFILE: "unsupported_profile", RESIDENT_NOT_FIT: "resident_not_fit", NO_SESSION_CAPACITY: "no_session_capacity", NO_FLOOR: "no_floor" }); function ok(code) { return code === STATUS_CODES.FITS || code === STATUS_CODES.MEETS_FLOOR; } function status(code, detail = {}) { return { ok: ok(code), code, ...detail }; } function expectedDistinctExperts(model, emittedSequences) { const arch = model.architecture; const experts = Number(arch.routed_experts || 0); const routed = Number(arch.routed_experts_per_token || 0); if (!experts || !routed) return routed; return experts * (1 - Math.pow(1 - routed / experts, Math.max(0, emittedSequences))); } function batchWeightGB(model, batch, rho) { const adapter = model.adapter; if (adapter.kind !== "moe") { return Number(adapter.active_weight_gb || adapter.resident_weight_gb || 0); } const arch = model.architecture; const total = Number(arch.total_params_b || 0); const active = Number(arch.active_params_b || 0); const experts = Number(arch.routed_experts || 0); const routed = Number(arch.routed_experts_per_token || 0); const shared = Number(arch.shared_experts_per_token || 0); const bytesPerParam = Number(adapter.weight_bytes_per_param || 0); if (!total || !active || !experts || !bytesPerParam) return Number(adapter.active_weight_gb || 0); // Non-active parameters are the E - k routed experts a token does not // select; shared experts are always-on, so they must not appear in the // derived per-expert size denominator. Using E - k - s here overestimates // the expert size and makes W_batch saturate above the total resident // weight. Explicit adapter overrides bypass the derivation for audited // packages with nonuniform shared-expert sizes. const expertParamB = Number(adapter.expert_param_b || ((total - active) / Math.max(1, experts - routed))); const fixedParamB = Number(adapter.fixed_param_b || Math.max(0, active - (routed + shared) * expertParamB)); const sharedExpertParamB = adapter.shared_expert_param_b === undefined ? expertParamB : Number(adapter.shared_expert_param_b); const distinctExperts = expectedDistinctExperts(model, Number(batch) * Number(rho || 1)); return bytesPerParam * (fixedParamB + (expertParamB * distinctExperts) + (sharedExpertParamB * shared)); } function kvAllocGB(model, lAlloc) { const adapter = model.adapter; return Number(adapter.kv_alloc_fixed_gb || 0) + Number(adapter.kv_alloc_gb_per_1k_tokens || 0) * (Number(lAlloc) / 1000); } function kvReadGB(model, lRead) { const adapter = model.adapter; return Number(adapter.kv_read_fixed_gb || 0) + Number(adapter.kv_read_gb_per_1k_tokens || 0) * (Number(lRead) / 1000); } function calculateBatchBound({ model, hardware, batch, rho, lRead }) { const b = Math.max(1, Math.floor(Number(batch))); const safeRho = Math.max(0.1, Number(rho || 1)); const bw = Number(hardware.memory.bandwidth_gbps); const batchWeight = batchWeightGB(model, b, safeRho); const readTraffic = kvReadGB(model, lRead); const q = batchWeight / (b * safeRho) + readTraffic; const aggregate = q > 0 ? bw / q : 0; return { batch: b, batchWeight, readTraffic, q, aggregate, perSession: aggregate / b }; } function fitStatuses({ free, bMem, fixed, best, rStar }) { const residentFit = free >= 0 ? status(STATUS_CODES.FITS) : status(STATUS_CODES.RESIDENT_NOT_FIT, { free }); const sessionFit = residentFit.ok ? bMem > 0 ? status(STATUS_CODES.FITS, { sessionCount: bMem }) : status(STATUS_CODES.NO_SESSION_CAPACITY, { sessionCount: 0 }) : status(STATUS_CODES.NOT_EVALUATED); const floorFit = sessionFit.ok ? best ? status(STATUS_CODES.MEETS_FLOOR, { batch: best.batch, perSession: best.perSession }) : status(STATUS_CODES.BELOW_FLOOR, { floor: rStar }) : status(STATUS_CODES.NOT_EVALUATED); const inspectBatchFit = fixed ? sessionFit.ok ? fixed.batch <= bMem ? status(STATUS_CODES.FITS, { batch: fixed.batch, fitCap: bMem }) : status(STATUS_CODES.EXCEEDS_MEMORY_FIT, { batch: fixed.batch, fitCap: bMem }) : residentFit.ok ? status(STATUS_CODES.NO_SESSION_CAPACITY, { batch: fixed.batch, fitCap: bMem }) : status(STATUS_CODES.RESIDENT_NOT_FIT, { batch: fixed.batch }) : status(STATUS_CODES.NOT_EVALUATED); const inspectFloorFit = inspectBatchFit.ok ? fixed.perSession >= rStar ? status(STATUS_CODES.MEETS_FLOOR, { perSession: fixed.perSession, floor: rStar }) : status(STATUS_CODES.BELOW_FLOOR, { perSession: fixed.perSession, floor: rStar }) : status(STATUS_CODES.NOT_EVALUATED); return { residentFit, sessionFit, floorFit, inspectBatchFit, inspectFloorFit }; } function calculateBounds({ model, hardware, lAlloc, lRead, overhead, rho, rStar, fixedBatch }) { const capacity = Number(hardware.memory.capacity_gb); const bandwidth = Number(hardware.memory.bandwidth_gbps); const safeRho = Math.max(0.1, Number(rho || 1)); const resident = Number(model.adapter.resident_weight_gb); const alloc = kvAllocGB(model, lAlloc); const free = capacity - resident - Number(overhead || 0); const bMem = alloc > 0 ? Math.max(0, Math.floor((free / alloc) + 1e-9)) : 0; const maxSearch = Math.min(bMem, BATCH_SEARCH_CAP); let best = null; for (let b = 1; b <= maxSearch; b += 1) { const result = calculateBatchBound({ model, hardware, batch: b, rho: safeRho, lRead }); if (result.perSession < rStar) continue; if (!best || result.aggregate > best.aggregate) best = result; } const single = calculateBatchBound({ model, hardware, batch: 1, rho: safeRho, lRead }); const active = Number(model.adapter.active_weight_gb || batchWeightGB(model, 1, safeRho)); const correction = capacity > 0 ? Math.max(0, 1 - ((resident + Number(overhead || 0)) / capacity)) : 0; const memoryPower = capacity * bandwidth; const memoryPowerCeiling = alloc > 0 && active > 0 ? safeRho * memoryPower / (alloc * active) * correction : 0; const fixed = fixedBatch ? calculateBatchBound({ model, hardware, batch: fixedBatch, rho: safeRho, lRead }) : null; return { inputs: { lAlloc, lRead, overhead, rho: safeRho, rStar, fixedBatch }, capacity, bandwidth, resident, alloc, free, bMem, best, single, memoryPowerCeiling, fixed, memoryPower, status: fitStatuses({ free, bMem, fixed, best, rStar }) }; } function profiledExpectedDistinctExperts(adapter, emittedSequences) { const experts = Number(adapter.routed_experts || 0); const routed = Number(adapter.routed_experts_per_token || 0); if (!experts || !routed) return routed; return experts * (1 - Math.pow(1 - routed / experts, Math.max(0, emittedSequences))); } function profiledWeightParts(profile) { const adapter = profile.architecture.weight_adapter; const bytesPerParam = Number(profile.serving.weight_bytes_per_param); if (adapter.kind === "dense") { const resident = Number(adapter.total_params_b) * bytesPerParam; return { resident, batchWeight(batch, rho) { return resident; }, active: resident }; } if (adapter.kind === "dense_resident_swept") { const resident = Number(adapter.resident_weight_gb ?? (Number(adapter.resident_params_b) * bytesPerParam)); const swept = Number(adapter.swept_weight_gb ?? (Number(adapter.swept_params_b) * bytesPerParam)); return { resident, batchWeight(batch, rho) { return swept; }, active: swept }; } if (adapter.kind === "moe_distinct_experts_exact") { const resident = Number(adapter.resident_weight_gb); const fixed = Number(adapter.fixed_weight_gb); const expert = Number(adapter.routed_expert_weight_gb); return { resident, batchWeight(batch, rho) { const distinctExperts = profiledExpectedDistinctExperts(adapter, Number(batch) * Number(rho || 1)); return fixed + expert * distinctExperts; }, active: fixed + expert * Number(adapter.routed_experts_per_token) }; } const total = Number(adapter.total_params_b); const active = Number(adapter.active_params_b); const routed = Number(adapter.routed_experts_per_token); const experts = Number(adapter.routed_experts); const expertParamB = Number(adapter.expert_param_b || ((total - active) / Math.max(1, experts - routed))); const fixedParamB = Number(adapter.fixed_param_b || Math.max(0, active - routed * expertParamB)); return { resident: total * bytesPerParam, batchWeight(batch, rho) { const distinctExperts = profiledExpectedDistinctExperts(adapter, Number(batch) * Number(rho || 1)); return bytesPerParam * (fixedParamB + expertParamB * distinctExperts); }, active: active * bytesPerParam }; } function componentContextTokens(component, tokens) { return component.kind === "sliding_window" ? Math.min(Number(tokens), Number(component.window_tokens)) : Number(tokens); } function scalarKvLayerCount(component, kind) { const override = kind === "alloc" ? component.alloc_layers : component.read_layers; return Number(override ?? component.layers); } function scalarKvComponentGB(component, tokens, bytesPerScalar, kind) { const streams = Number(component.kv_scalar_multiplier || 2); const layers = scalarKvLayerCount(component, kind); const effectiveTokens = componentContextTokens(component, tokens); return streams * layers * Number(component.kv_heads) * Number(component.head_dim) * Number(bytesPerScalar) * effectiveTokens / 1e9; } function directKvComponentGB(component, tokens, kind) { if (component.kind === "recurrent_state") { return kind === "alloc" ? Number(component.alloc_gb_per_session || 0) : Number(component.read_gb_per_output_token || 0); } if (component.kind === "compressed_state") { const formula = kind === "alloc" ? component.alloc_formula : component.read_formula; return Number(formula.gb_per_1k_context_tokens || 0) * (Number(tokens) / 1000); } return null; } function kvAdapterGB(adapter, tokens, bytesPerScalar, kind) { if (adapter.kind === "layered_kv") { return adapter.components.reduce((sum, component) => { const direct = directKvComponentGB(component, tokens, kind); return sum + (direct ?? scalarKvComponentGB(component, tokens, bytesPerScalar, kind)); }, 0); } const direct = directKvComponentGB(adapter, tokens, kind); return direct ?? scalarKvComponentGB(adapter, tokens, bytesPerScalar, kind); } function profiledKvAllocGB(profile, lAlloc) { return kvAdapterGB( profile.architecture.kv_adapter, lAlloc, profile.serving.kv_store_bytes_per_scalar, "alloc" ); } function profiledKvReadGB(profile, lRead) { return kvAdapterGB( profile.architecture.kv_adapter, lRead, profile.serving.kv_read_bytes_per_scalar, "read" ); } function calculateProfiledBatchBound({ profile, hardware, batch, rho, lRead }) { const b = Math.max(1, Math.floor(Number(batch))); const safeRho = Number(rho || 1); const bw = Number(hardware.memory.bandwidth_gbps); const weight = profiledWeightParts(profile); const batchWeight = weight.batchWeight(b, safeRho); const readTraffic = profiledKvReadGB(profile, lRead); const q = batchWeight / (b * safeRho) + readTraffic; const aggregate = q > 0 ? bw / q : 0; return { batch: b, batchWeight, readTraffic, q, aggregate, perSession: aggregate / b }; } function unsupportedProfileResult({ model, profile, hardware, workloadSettings, reason }) { const capacity = Number(hardware.memory.capacity_gb || 0); const bandwidth = Number(hardware.memory.bandwidth_gbps || 0); const repo = profile?.repo || model?.id || ""; return { engine_version: ENGINE_VERSION, schema_version: "1.0.0", status_code: PROFILE_STATUS_CODES.UNSUPPORTED_PROFILE, profile_resolution: { repo, model_profile: profile?.id || null, model_profile_status: profile?.status || "missing" }, inputs: { hardware_id: hardware.id, workload_settings: workloadSettings }, trace: { capacity_gb: capacity, bandwidth_gbps: bandwidth, overhead_gb: Number(workloadSettings.overhead_gb || 0), free_gb: 0, w_resident_gb: 0, k_alloc_gb: 0, b_mem: 0, single_session_toks_per_s: 0, usable_batches_summary: { count: 0, min_batch: null, max_batch: null }, b_star: null, w_batch_at_b_star_gb: 0, k_read_gb: 0, q_at_b_star_gb_per_output_token: 0, aggregate_toks_per_s: 0, per_session_toks_per_s: 0, memory_power_ceiling_toks_per_s: 0 }, ceilings: {}, usable_batch: {}, warnings: [reason], inputs_legacy: {}, capacity, bandwidth, resident: 0, alloc: 0, free: 0, bMem: 0, best: null, single: { batch: 1, batchWeight: 0, readTraffic: 0, q: 0, aggregate: 0, perSession: 0 }, memoryPowerCeiling: 0, fixed: null, memoryPower: capacity * bandwidth, status: { residentFit: status(STATUS_CODES.UNSUPPORTED_PROFILE), sessionFit: status(STATUS_CODES.NOT_EVALUATED), floorFit: status(STATUS_CODES.NOT_EVALUATED), inspectBatchFit: status(STATUS_CODES.NOT_EVALUATED), inspectFloorFit: status(STATUS_CODES.NOT_EVALUATED) } }; } function profileStatusCode({ free, bMem, best }) { if (free < 0) return PROFILE_STATUS_CODES.RESIDENT_NOT_FIT; if (bMem < 1) return PROFILE_STATUS_CODES.NO_SESSION_CAPACITY; if (!best) return PROFILE_STATUS_CODES.NO_FLOOR; return PROFILE_STATUS_CODES.OK; } function usableSummary(usableBatches) { if (!usableBatches.length) { return { count: 0, min_batch: null, max_batch: null }; } return { count: usableBatches.length, min_batch: usableBatches[0], max_batch: usableBatches[usableBatches.length - 1] }; } function calculateProfiledBounds({ profile, model, hardware, workloadSettings }) { if (!profile) { return unsupportedProfileResult({ model, profile, hardware, workloadSettings, reason: "No audited model profile is available for this repo." }); } if (profile.status !== "audited") { return unsupportedProfileResult({ model, profile, hardware, workloadSettings, reason: `Model profile status is ${profile.status}; production bounds require audited.` }); } const decodePolicy = workloadSettings.decode_policy || { kind: "ordinary", rho: 1 }; if (decodePolicy.kind !== "ordinary" || Number(decodePolicy.rho) !== 1) { return unsupportedProfileResult({ model, profile, hardware, workloadSettings, reason: "Bounds Engine v1 only supports ordinary decoding with rho = 1." }); } const lAlloc = Number(workloadSettings.l_alloc_tokens); const lRead = Number(workloadSettings.l_read_tokens); const overhead = Number(workloadSettings.overhead_gb || 0); const rStar = Number(workloadSettings.min_toks_per_session || 0); const rho = 1; const capacity = Number(hardware.memory.capacity_gb); const bandwidth = Number(hardware.memory.bandwidth_gbps); const memoryPower = capacity * bandwidth; const weight = profiledWeightParts(profile); const resident = weight.resident; const alloc = profiledKvAllocGB(profile, lAlloc); const free = capacity - resident - overhead; // A profile with zero per-session allocation is not bounded by memory at // all; treat its session capacity as the search cap instead of zero. const allocUnbounded = free >= 0 && alloc <= 0; const bMem = free >= 0 ? alloc > 0 ? Math.max(0, Math.floor((free / alloc) + 1e-9)) : BATCH_SEARCH_CAP : 0; const maxSearch = Math.min(bMem, BATCH_SEARCH_CAP); const usableBatches = []; let best = null; for (let b = 1; b <= maxSearch; b += 1) { const result = calculateProfiledBatchBound({ profile, hardware, batch: b, rho, lRead }); if (result.perSession < rStar) continue; usableBatches.push(b); if (!best || result.aggregate > best.aggregate) best = result; } const warnings = []; if (allocUnbounded) { warnings.push( "Profile reports zero per-session KV/state allocation; memory does not bound concurrency and the batch search is capped at " + `${BATCH_SEARCH_CAP}.` ); } const searchTruncated = best !== null && best.batch === maxSearch && (allocUnbounded || bMem > maxSearch); if (searchTruncated) { warnings.push( `Batch search stopped at ${maxSearch} with the per-session floor still satisfied; ` + "the reported KV-aware ceiling may understate the true bound." ); } const single = calculateProfiledBatchBound({ profile, hardware, batch: 1, rho, lRead }); const correction = capacity > 0 ? Math.max(0, 1 - ((resident + overhead) / capacity)) : 0; const active = weight.batchWeight(1, rho); const memoryPowerCeiling = alloc > 0 && active > 0 ? rho * memoryPower / (alloc * active) * correction : 0; const statusCode = profileStatusCode({ free, bMem, best }); const statuses = fitStatuses({ free, bMem, fixed: null, best, rStar }); const bStar = best?.batch ?? null; const trace = { capacity_gb: capacity, bandwidth_gbps: bandwidth, overhead_gb: overhead, free_gb: free, w_resident_gb: resident, k_alloc_gb: alloc, b_mem: bMem, single_session_toks_per_s: single.aggregate, usable_batches_summary: usableSummary(usableBatches), b_star: bStar, w_batch_at_b_star_gb: best?.batchWeight ?? 0, k_read_gb: single.readTraffic, q_at_b_star_gb_per_output_token: best?.q ?? 0, aggregate_toks_per_s: best?.aggregate ?? 0, per_session_toks_per_s: best?.perSession ?? 0, memory_power_ceiling_toks_per_s: memoryPowerCeiling }; return { engine_version: ENGINE_VERSION, schema_version: "1.0.0", status_code: statusCode, profile_resolution: { repo: profile.repo, model_profile: profile.id, model_profile_status: profile.status }, inputs: { hardware_id: hardware.id, workload_settings: workloadSettings }, trace, ceilings: best ? { single_session_toks_per_s: single.aggregate, kv_aware_aggregate_toks_per_s: best.aggregate, per_session_at_b_star_toks_per_s: best.perSession, memory_power_toks_per_s: memoryPowerCeiling } : { single_session_toks_per_s: single.aggregate, memory_power_toks_per_s: memoryPowerCeiling }, usable_batch: best ? { b_star: best.batch, selection_rule: "max_aggregate_over_floor_qualified_batches" } : {}, warnings, inputs_legacy: { lAlloc, lRead, overhead, rho, rStar }, capacity, bandwidth, resident, alloc, free, bMem, best, single, memoryPowerCeiling, fixed: null, memoryPower, status: statuses }; } const api = Object.freeze({ STATUS_CODES, PROFILE_STATUS_CODES, expectedDistinctExperts, batchWeightGB, kvAllocGB, kvReadGB, calculateBatchBound, calculateBounds, profiledWeightParts, profiledKvAllocGB, profiledKvReadGB, calculateProfiledBatchBound, calculateProfiledBounds }); root.LocalFrontierBounds = api; if (typeof module !== "undefined" && module.exports) { module.exports = api; } })(typeof window !== "undefined" ? window : globalThis);