| { |
| "platform": "macOS-27.0-arm64-arm-64bit", |
| "machine": "arm64", |
| "torch": "2.13.0", |
| "method": "one process per model and device; auto_quant memory-linear attention for all three; median over 110 benchmark items under 1,024 tokens after 10 warm-up items; one ~8k- and one ~32k-token item. The BF16 checkpoint runs in FP32 here (CPU/MPS default). Memory is the resident-set growth over the process baseline; on MPS, weights in memory = torch.mps.current_allocated_memory() after load, and the peak = the Metal driver allocation (includes the allocator cache).", |
| "runs": [ |
| { |
| "model": "bf16", |
| "device": "cpu", |
| "dtype": "torch.float32", |
| "load_seconds": 1.97, |
| "rss_after_load_gb": 0.787, |
| "mps_driver_after_load_gb": null, |
| "short_items": 110, |
| "median_ms_under_1k_tokens": 68.22, |
| "long": [ |
| { |
| "tokens": 7449, |
| "seconds": 1.65, |
| "peak_rss_gb": 1.33, |
| "peak_mps_driver_gb": null |
| }, |
| { |
| "tokens": 29233, |
| "seconds": 15.66, |
| "peak_rss_gb": 2.12, |
| "peak_mps_driver_gb": null |
| } |
| ] |
| }, |
| { |
| "model": "int8", |
| "device": "cpu", |
| "dtype": "torch.float32", |
| "load_seconds": 1.77, |
| "rss_after_load_gb": 0.37, |
| "mps_driver_after_load_gb": null, |
| "short_items": 110, |
| "median_ms_under_1k_tokens": 69.84, |
| "long": [ |
| { |
| "tokens": 7449, |
| "seconds": 1.49, |
| "peak_rss_gb": 0.97, |
| "peak_mps_driver_gb": null |
| }, |
| { |
| "tokens": 29233, |
| "seconds": 13.44, |
| "peak_rss_gb": 1.77, |
| "peak_mps_driver_gb": null |
| } |
| ] |
| }, |
| { |
| "model": "int4", |
| "device": "cpu", |
| "dtype": "torch.float32", |
| "load_seconds": 1.66, |
| "rss_after_load_gb": 0.298, |
| "mps_driver_after_load_gb": null, |
| "short_items": 110, |
| "median_ms_under_1k_tokens": 104.44, |
| "long": [ |
| { |
| "tokens": 7449, |
| "seconds": 1.51, |
| "peak_rss_gb": 0.9, |
| "peak_mps_driver_gb": null |
| }, |
| { |
| "tokens": 29233, |
| "seconds": 12.49, |
| "peak_rss_gb": 1.73, |
| "peak_mps_driver_gb": null |
| } |
| ] |
| }, |
| { |
| "model": "bf16", |
| "device": "mps", |
| "dtype": "torch.float32", |
| "load_seconds": 2.0, |
| "rss_after_load_gb": 0.233, |
| "mps_tensors_after_load_gb": 0.557, |
| "short_items": 110, |
| "median_ms_under_1k_tokens": 24.45, |
| "long": [ |
| { |
| "tokens": 7449, |
| "seconds": 0.46, |
| "peak_rss_gb": 0.77, |
| "peak_mps_driver_gb": 1.03 |
| }, |
| { |
| "tokens": 29233, |
| "seconds": 4.42, |
| "peak_rss_gb": 0.62, |
| "peak_mps_driver_gb": 2.02 |
| } |
| ] |
| }, |
| { |
| "model": "int8", |
| "device": "mps", |
| "dtype": "torch.float32", |
| "load_seconds": 1.79, |
| "rss_after_load_gb": 0.455, |
| "mps_tensors_after_load_gb": 0.14, |
| "short_items": 110, |
| "median_ms_under_1k_tokens": 29.83, |
| "long": [ |
| { |
| "tokens": 7449, |
| "seconds": 0.47, |
| "peak_rss_gb": 0.79, |
| "peak_mps_driver_gb": 1.06 |
| }, |
| { |
| "tokens": 29233, |
| "seconds": 4.09, |
| "peak_rss_gb": 0.54, |
| "peak_mps_driver_gb": 2.05 |
| } |
| ] |
| }, |
| { |
| "model": "int4", |
| "device": "mps", |
| "dtype": "torch.float32", |
| "load_seconds": 1.77, |
| "rss_after_load_gb": 0.392, |
| "mps_tensors_after_load_gb": 0.072, |
| "short_items": 110, |
| "median_ms_under_1k_tokens": 37.83, |
| "long": [ |
| { |
| "tokens": 7449, |
| "seconds": 0.47, |
| "peak_rss_gb": 0.59, |
| "peak_mps_driver_gb": 1.09 |
| }, |
| { |
| "tokens": 29233, |
| "seconds": 4.28, |
| "peak_rss_gb": 0.54, |
| "peak_mps_driver_gb": 2.07 |
| } |
| ] |
| } |
| ] |
| } |
