{ "as_of": "2026-07-16", "note": "Single source of truth for numbers quoted on boundarylabs.org. Peaks are dated lab measurements; only the production entry describes current serving state.", "inference": { "production": { "id": "openrouter", "model": "GPT-5.6 Luna (API via OpenRouter)", "framework": "local-proxy :8010 → OpenRouter — no local GPU since tower retirement 2026-07-20", "gen_tok_s_warm": 99, "measured": "2026-08-01, median of 5 × 600-token runs via :8010, reasoning disabled; observed range 89–106 tok/s", "context_tokens": 1050000, "context_label": "1M context", "status": "active — production (trial, started 2026-08-01)" }, "standby": [ { "id": "nemotron", "model": "Nemotron 3 Nano 30B A3B (Mamba/SSM hybrid MoE)", "peak_tok_s": 117.6, "date": "2026-05-20", "status": "standby — dated peak, not current production" }, { "id": "ornith", "model": "Ornith-1.0-35B (Q4_K_M GGUF / AEON NVFP4)", "prod_config_peak_tok_s": 124.2, "warm_tok_s": 101, "date": "2026-06-27", "status": "production trial ended 2026-07-03 — genesis restored as default" } ], "lab_peak_tok_s": { "value": 129.9, "label": "lab peak — Ornith-1.0-35B GGUF, short-context, 2026-06-25", "receipt": "https://github.com/randomchaos7800-hub/inference-research/blob/master/tower/ornith/suite-20260625-170030.json", "not": "current production throughput" } }, "consumer_blackwell_findings": { "date": "2026-07-10", "hardware": "2x RTX 5060 Ti (Blackwell SM_120, 16 GB each), TP=2 over PCIe, no P2P, no NVLink", "receipt": "https://github.com/randomchaos7800-hub/inference-research/blob/master/tower/profiling/decode-profile-2x5060ti-20260710.md", "all_reduce_tax_pct": 20.1, "all_reduce_detail": "128 all-reduces/token x 25.3us each, pure latency; NCCL routed through host RAM because consumer cards block GPU-to-GPU P2P (can_device_access_peer=False). NCCL_P2P_DISABLE is a no-op.", "pcie_decode_MBps": 250, "pcie_note": "PCIe moves ~250 MB/s during TP=2 decode — a few percent of even the Gen4 x4 slot. The no-P2P cost is per-op latency, not bandwidth.", "native_fp4": "SM_120 has native FP4 tensor cores and vLLM's cutlass FP4 kernel is compiled + SM_120-capable, but it only engages for W4A4 (4-bit activations); W4A16 quants fall back to Marlin emulation, ~1.7x slower at matched settings.", "speedup_levers": { "mtp_speculation_pct": 59, "cuda_graphs_pct": "60-75", "note": "single-stream temp 0.3 600-token measurements; both off = ~40% of the number. Distinct condition from the 97 t/s warm greedy production figure." } }, "moe_findings": { "date": "2026-07-14", "hardware": "2x RTX 5060 Ti (Blackwell SM_120, 16GB each), 32GB system RAM", "receipt": "https://github.com/randomchaos7800-hub/inference-research/blob/master/tower/moe/FINAL-REPORT-2026-07-14.md", "verdict": "MoE beats dense on speed on this hardware but loses badly on quality — it relocates the speed/quality trade, it doesn't remove it", "speed_peak_tok_s": { "value": 133.05, "model": "GPT-OSS-20B, native MXFP4, llama.cpp all-GPU" }, "best_balanced": { "model": "Nemotron-3-Nano-30B-A3B", "tok_s": 123.4, "quality_score": 2.8 }, "vllm_moe_bug": "an apparent post-load hang on vLLM 0.25.0 serving Qwen3-30B-A3B-NVFP4 was misdiagnosed twice (NCCL P2P, then auto-enabled expert-parallel) before the real cause was found by checking ps aux mid-stall: an uncapped FlashInfer JIT kernel compile storm (91 concurrent nvcc/cicc/ptxas, no precompiled kernel exists yet for this SM120+fp8_uint4 combination), which drove real host RAM/swap thrashing — not a synchronization deadlock", "custom_patch_verdict": "no case found for genesis-style custom vLLM patching on the MoE path — llama.cpp served every checkpoint cleanly once configured correctly, and beat vLLM outright for MoE (the opposite of the dense-model conclusion)" }, "mac_mini_findings": { "date": "2026-07-16", "hardware": "Mac mini M4, 16GB unified memory, MLX (not llama.cpp)", "receipt": "https://github.com/randomchaos7800-hub/model-eval/blob/master/results/domain-suite-mini-quant-hunt-20260716.md", "verdict": "Ornith-1.0-9B-4bit stays live default — smallest of four candidates tested, still beats both 27B Bonsai builds on quality", "live_default": { "model": "Ornith-1.0-9B-4bit", "quality_score": 3.0, "tok_s": 18.66 }, "not_adopted": [ { "model": "Ternary-Bonsai-27B (Prism ML, 1.71 bpw)", "quality_score": 2.93, "tok_s": 12.16 }, { "model": "Bonsai-27B (Prism ML, 1.125 bpw)", "quality_score": 2.47, "tok_s": 20.33, "note": "fastest of the four, lowest quality" } ], "failure_pattern": "both Bonsai builds fail the same two of three tested scenarios identically regardless of bit-width (rail-crossing signal/roundabout trap; Hours-of-Service clock math, no answer ever reached). The genesis reference — same Qwen3.6-27B base as Bonsai, quantized via GPTQ INT4 instead of Prism ML's ternary/binary method — clears both cleanly, pointing at the compression method rather than the shared parent model's training." }, "quality": { "domain_suite": { "date": "2026-07-10", "description": "15 domain-expert scenarios (logistics, traffic, restaurants), thinking off, judged blind", "receipt": "https://github.com/randomchaos7800-hub/model-eval/blob/master/results/domain-suite-nvidia-vs-genesis-20260710.md", "nvidia_w4a16_score": 4.13, "genesis_int4_score": 3.73, "scale": "1-5, mean over 15 scenarios", "judges": "in-session Claude (+0.40) and OpenRouter Haiku 4.5 automated (+0.20) — both agree nvidia W4A16 > genesis INT4", "note": "genesis remains production default (speed + safe floor); nvidia W4A16 is ~1/3 the speed, the consult-for-hard-ones model" }, "moe_domain_suite": { "date": "2026-07-14", "description": "same 15-scenario suite run against 6 MoE checkpoints, thinking/reasoning suppressed where the runtime allows it", "receipt": "https://github.com/randomchaos7800-hub/inference-research/blob/master/tower/moe/FINAL-REPORT-2026-07-14.md", "scores": { "qwen3.6-35b-a3b": 2.87, "nemotron-3-nano-30b-a3b": 2.8, "qwen3-30b-a3b-gptq-int4": 2.67, "gpt-oss-20b": 2.2, "qwen3-30b-a3b-q3km": 1.87 }, "scale": "1-5, mean over 15 scenarios", "comparison": "genesis 3.73, nvidia W4A16 3.87 — every MoE checkpoint tested loses to both dense models", "cross_family_failure": "two traffic-engineering scenarios (a yellow-trap signal conflict, a rail-crossing queue-spillback risk) fail identically across all 3 base-model families tested (Qwen, Nemotron/NVIDIA, OpenAI GPT-OSS), 4 quantizations, 2 runtimes — read as a capability floor at this parameter class, not a per-checkpoint defect", "note": "every MoE checkpoint beat genesis on raw tok/s (up to 133.05, GPT-OSS-20B); the fastest MoE config (Qwen3-30B-A3B Q3_K_M, 121.5 t/s) is also the worst-scoring one — speed and quality inverted, not just traded off" } }, "services_live": [ "harness", "Mike", "Hermes", "autoresearch", "chronicle" ], "experiments_logged": { "value": "215+", "as_of": "2026-06-27", "counts": "raw benchmark runs across all campaigns; milestone experiments with verdict docs number 30+", "receipts": "https://github.com/randomchaos7800-hub/inference-research" }, "longmemeval": { "mike_injection": { "acc": 0.88, "n": 25, "correct": 22, "mode": "context-window injection", "note": "25-question single-session-user subset; not the full 500-question LongMemEval" }, "paper_injection": { "acc": 0.84, "n": 25, "correct": 21, "mode": "context-window injection" }, "adam_selene": { "acc": 0.75, "mode": "LongMemEval subset; Gemma 4 26B", "multi_session": 0.54, "note": "different system and protocol than the Mike injection runs — not comparable head-to-head" } }, "paper": { "title": "Commodity Hardware for Persistent AI Companions", "doi": "10.5281/zenodo.19863336", "url": "https://doi.org/10.5281/zenodo.19863336", "peer_reviewed": false, "license": "CC BY 4.0" } }