| { |
| "vanilla_bf16": { |
| "mode": "bf16", |
| "gpu": "NVIDIA H100 80GB HBM3", |
| "torch": "2.9.0a0+50eac811a6.nv25.09", |
| "memory_metric": "PyTorch allocated/reserved bytes; NVML process memory sampled every 5 ms", |
| "records": [ |
| { |
| "case": 0, |
| "tokens": 620, |
| "media": false, |
| "hidden_sha256": "a597b8195ef8a3bcade2f86d3768688e097b66a771b3377af2ea43cc00f6969a", |
| "logits": [ |
| [ |
| [ |
| -3.125, |
| -2.03125, |
| -0.359375, |
| -3.4375 |
| ], |
| [ |
| -1.015625, |
| -1.6171875, |
| -1.171875 |
| ], |
| [ |
| 1.640625, |
| -3.15625 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 55004298752, |
| "reserved_bytes": 55236886528, |
| "peak_allocated_bytes": 55132857344, |
| "peak_reserved_bytes": 55236886528 |
| }, |
| "median_ms": 78.40589061379433 |
| }, |
| { |
| "case": 1, |
| "tokens": 449, |
| "media": false, |
| "hidden_sha256": "85fb727a649afba0f3b8c9c81825bcb8f2ccaceaa84aa5b851d475029b719ba3", |
| "logits": [ |
| [ |
| [ |
| -4.0, |
| -1.03125, |
| -1.5859375, |
| -3.609375 |
| ], |
| [ |
| -1.4375, |
| -2.359375, |
| -1.03125 |
| ], |
| [ |
| 0.828125, |
| -2.75 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 55004296704, |
| "reserved_bytes": 55287218176, |
| "peak_allocated_bytes": 55100137472, |
| "peak_reserved_bytes": 55287218176 |
| }, |
| "median_ms": 77.41388725116849 |
| }, |
| { |
| "case": 2, |
| "tokens": 622, |
| "media": false, |
| "hidden_sha256": "1f1348abf8e3ba0c49203487244b4b3b44da05204fca2e52f23c6bbc899088c1", |
| "logits": [ |
| [ |
| [ |
| -3.921875, |
| -0.8828125, |
| -0.140625, |
| -3.78125 |
| ], |
| [ |
| -0.57421875, |
| -2.90625, |
| -1.09375 |
| ], |
| [ |
| 1.28125, |
| -3.3125 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 55004298752, |
| "reserved_bytes": 55287218176, |
| "peak_allocated_bytes": 55133167616, |
| "peak_reserved_bytes": 55287218176 |
| }, |
| "median_ms": 78.70555715635419 |
| }, |
| { |
| "case": 3, |
| "tokens": 490, |
| "media": false, |
| "hidden_sha256": "5091a522f40ff5dc7d795b6f2d01735838286a07f12cf7a0fb696f0a5e4cd292", |
| "logits": [ |
| [ |
| [ |
| -3.9375, |
| -0.96875, |
| -0.75, |
| -4.375 |
| ], |
| [ |
| -0.5625, |
| -3.5, |
| -2.53125 |
| ], |
| [ |
| 1.2109375, |
| -2.875 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 55004296704, |
| "reserved_bytes": 55335452672, |
| "peak_allocated_bytes": 55107471872, |
| "peak_reserved_bytes": 55335452672 |
| }, |
| "median_ms": 78.5877387970686 |
| }, |
| { |
| "case": 4, |
| "tokens": 448, |
| "media": false, |
| "hidden_sha256": "860c8cff7f484291cee00b1b6a10c759a4e636d2b39b022935d3e450e4360c27", |
| "logits": [ |
| [ |
| [ |
| -4.46875, |
| 0.57421875, |
| -3.265625, |
| -4.5 |
| ], |
| [ |
| 0.59375, |
| -4.3125, |
| -3.015625 |
| ], |
| [ |
| -2.109375, |
| -0.94921875 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 55004295680, |
| "reserved_bytes": 55335452672, |
| "peak_allocated_bytes": 55098472448, |
| "peak_reserved_bytes": 55335452672 |
| }, |
| "median_ms": 78.52127030491829 |
| }, |
| { |
| "case": 5, |
| "tokens": 437, |
| "media": false, |
| "hidden_sha256": "158204930cd0fcf01c6b0287f8fc9a52ee17dc1f2f947d80d89739a56cad9601", |
| "logits": [ |
| [ |
| [ |
| -3.765625, |
| -1.0703125, |
| -2.015625, |
| -3.875 |
| ], |
| [ |
| -0.69921875, |
| -2.78125, |
| -1.6484375 |
| ], |
| [ |
| 0.94921875, |
| -2.53125 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 55004295680, |
| "reserved_bytes": 55335452672, |
| "peak_allocated_bytes": 55096978432, |
| "peak_reserved_bytes": 55335452672 |
| }, |
| "median_ms": 78.34483683109283 |
| }, |
| { |
| "case": 6, |
| "tokens": 494, |
| "media": false, |
| "hidden_sha256": "764e10d942f237bbb0c977a5f505331000194a15a17b9ff3873598149961898a", |
| "logits": [ |
| [ |
| [ |
| -2.59375, |
| -2.96875, |
| -0.3125, |
| -3.5 |
| ], |
| [ |
| -3.09375, |
| -1.1328125, |
| -0.90625 |
| ], |
| [ |
| 1.6484375, |
| -3.25 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 55004296704, |
| "reserved_bytes": 55385784320, |
| "peak_allocated_bytes": 55108092928, |
| "peak_reserved_bytes": 55385784320 |
| }, |
| "median_ms": 78.5312089137733 |
| }, |
| { |
| "case": 7, |
| "tokens": 449, |
| "media": false, |
| "hidden_sha256": "cd9636133adc36071ba54ab87a7cf99211ff713a88d866a966c64b53750a8637", |
| "logits": [ |
| [ |
| [ |
| -3.9375, |
| -1.140625, |
| -2.09375, |
| -3.890625 |
| ], |
| [ |
| 0.3203125, |
| -4.03125, |
| -2.859375 |
| ], |
| [ |
| 1.34375, |
| -3.03125 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 55004296704, |
| "reserved_bytes": 55385784320, |
| "peak_allocated_bytes": 55100137472, |
| "peak_reserved_bytes": 55385784320 |
| }, |
| "median_ms": 78.0846094712615 |
| }, |
| { |
| "case": 8, |
| "tokens": 491, |
| "media": false, |
| "hidden_sha256": "8e3dd892fa0fe4e7a852f5fc094a96860954f0685cd86bd0bbad037ba5805315", |
| "logits": [ |
| [ |
| [ |
| -4.1875, |
| -2.46875, |
| -0.33984375, |
| -3.875 |
| ], |
| [ |
| -1.890625, |
| -3.0, |
| -1.265625 |
| ], |
| [ |
| 1.6328125, |
| -3.59375 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 55004296704, |
| "reserved_bytes": 55385784320, |
| "peak_allocated_bytes": 55107629056, |
| "peak_reserved_bytes": 55385784320 |
| }, |
| "median_ms": 78.83109943941236 |
| }, |
| { |
| "case": 9, |
| "tokens": 624, |
| "media": false, |
| "hidden_sha256": "1620d52e2b2963e6f7f28ee3b970ae294a01ee10d22fc4cb7fe99c46e8f2e085", |
| "logits": [ |
| [ |
| [ |
| -3.109375, |
| -0.890625, |
| -0.71875, |
| -3.765625 |
| ], |
| [ |
| -0.1484375, |
| -2.625, |
| -1.53125 |
| ], |
| [ |
| 1.359375, |
| -3.375 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 55004298752, |
| "reserved_bytes": 55385784320, |
| "peak_allocated_bytes": 55133474816, |
| "peak_reserved_bytes": 55385784320 |
| }, |
| "median_ms": 78.95864453166723 |
| }, |
| { |
| "case": 10, |
| "tokens": 454, |
| "media": false, |
| "hidden_sha256": "18e815e952a4255661599704e290b5f37c3187b8c068cc0d535566a16f7f99f9", |
| "logits": [ |
| [ |
| [ |
| -4.34375, |
| -2.484375, |
| -0.21484375, |
| -3.921875 |
| ], |
| [ |
| -2.0, |
| -2.703125, |
| -1.1640625 |
| ], |
| [ |
| 1.4453125, |
| -3.21875 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 55004296704, |
| "reserved_bytes": 55385784320, |
| "peak_allocated_bytes": 55100574208, |
| "peak_reserved_bytes": 55385784320 |
| }, |
| "median_ms": 77.87147304043174 |
| }, |
| { |
| "case": 11, |
| "tokens": 442, |
| "media": false, |
| "hidden_sha256": "e6d2eb173ad497fcff31054f9767fcbbe8604bccf1b9317b2686712d4db88e8e", |
| "logits": [ |
| [ |
| [ |
| -4.0, |
| -0.384765625, |
| -1.125, |
| -4.28125 |
| ], |
| [ |
| 0.2099609375, |
| -4.15625, |
| -3.296875 |
| ], |
| [ |
| -0.2109375, |
| -1.28125 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 55004295680, |
| "reserved_bytes": 55385784320, |
| "peak_allocated_bytes": 55097241088, |
| "peak_reserved_bytes": 55385784320 |
| }, |
| "median_ms": 78.13491160050035 |
| }, |
| { |
| "case": 12, |
| "tokens": 458, |
| "media": false, |
| "hidden_sha256": "1469570ef5926d2f8a4899966bfeeea0676cdf5eaaf53364e5b9a281cb4929f6", |
| "logits": [ |
| [ |
| [ |
| -2.125, |
| -3.796875, |
| -0.24609375, |
| -2.671875 |
| ], |
| [ |
| -3.953125, |
| -0.015625, |
| -1.75 |
| ], |
| [ |
| 1.4140625, |
| -2.703125 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 55004296704, |
| "reserved_bytes": 55385784320, |
| "peak_allocated_bytes": 55101027840, |
| "peak_reserved_bytes": 55385784320 |
| }, |
| "median_ms": 78.62287666648626 |
| }, |
| { |
| "case": 13, |
| "tokens": 620, |
| "media": false, |
| "hidden_sha256": "48880c8ff6215d9e21b7b91346a95ed21689afc1176688d73cb8f7aa141f3486", |
| "logits": [ |
| [ |
| [ |
| -3.125, |
| -1.9140625, |
| -0.88671875, |
| -3.984375 |
| ], |
| [ |
| -0.82421875, |
| -2.28125, |
| -1.3359375 |
| ], |
| [ |
| 1.7734375, |
| -3.34375 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 55004298752, |
| "reserved_bytes": 55385784320, |
| "peak_allocated_bytes": 55132857344, |
| "peak_reserved_bytes": 55385784320 |
| }, |
| "median_ms": 79.05966322869062 |
| }, |
| { |
| "case": 14, |
| "tokens": 616, |
| "media": false, |
| "hidden_sha256": "43d102db01a1c45d1148a5217938cf40a71d5e2b8ce1e11e79f640a0f3639614", |
| "logits": [ |
| [ |
| [ |
| -3.296875, |
| -0.9140625, |
| -1.1171875, |
| -3.90625 |
| ], |
| [ |
| -0.27734375, |
| -3.046875, |
| -2.046875 |
| ], |
| [ |
| 1.234375, |
| -3.578125 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 55004298752, |
| "reserved_bytes": 55385784320, |
| "peak_allocated_bytes": 55132236288, |
| "peak_reserved_bytes": 55385784320 |
| }, |
| "median_ms": 78.96459102630615 |
| }, |
| { |
| "case": 15, |
| "tokens": 493, |
| "media": false, |
| "hidden_sha256": "b24dcfc763451f39b010a276f8e6fd985a7c2bf9603816333f9dfc6427508ded", |
| "logits": [ |
| [ |
| [ |
| -2.5, |
| -2.109375, |
| -0.00390625, |
| -3.78125 |
| ], |
| [ |
| -1.5, |
| -1.6796875, |
| -1.4375 |
| ], |
| [ |
| 1.28125, |
| -3.125 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 55004296704, |
| "reserved_bytes": 55385784320, |
| "peak_allocated_bytes": 55107937280, |
| "peak_reserved_bytes": 55385784320 |
| }, |
| "median_ms": 78.66512797772884 |
| }, |
| { |
| "case": 16, |
| "tokens": 1278, |
| "media": false, |
| "hidden_sha256": "63d5d9aa787a0c2c8bd467d55405d2a5074eef3e94d726f651a692506a55aaa5", |
| "logits": [ |
| [ |
| [ |
| 1.109375, |
| 0.29296875, |
| 0.59375, |
| 0.7421875 |
| ], |
| [ |
| 0.75, |
| 0.015625, |
| 0.234375, |
| 0.5234375 |
| ], |
| [ |
| 0.5234375, |
| -0.134765625, |
| 0.138671875, |
| 0.296875 |
| ], |
| [ |
| 0.4375, |
| -0.09375, |
| 0.0, |
| 0.1640625 |
| ], |
| [ |
| 0.451171875, |
| -0.251953125, |
| -0.126953125, |
| 0.03125 |
| ], |
| [ |
| 0.3203125, |
| -0.3359375, |
| -0.1953125, |
| -0.05078125 |
| ], |
| [ |
| 0.1484375, |
| -0.296875, |
| -0.171875, |
| 0.0859375 |
| ], |
| [ |
| 0.19140625, |
| -0.22265625, |
| -0.11328125, |
| 0.32421875 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 55004308992, |
| "reserved_bytes": 55385784320, |
| "peak_allocated_bytes": 55267782144, |
| "peak_reserved_bytes": 55385784320 |
| }, |
| "median_ms": 140.98667027428746 |
| }, |
| { |
| "case": 17, |
| "tokens": 1502, |
| "media": false, |
| "hidden_sha256": "579420a72e605ed1622ca66435162bb7e7241b143240616fb5745ae5542aed09", |
| "logits": [ |
| [ |
| [ |
| 1.1640625, |
| 0.35546875, |
| 0.4453125, |
| 0.46484375 |
| ], |
| [ |
| 0.55078125, |
| -0.04296875, |
| 0.06640625, |
| 0.158203125 |
| ], |
| [ |
| 0.57421875, |
| -0.0390625, |
| 0.32421875, |
| 0.26953125 |
| ], |
| [ |
| 0.69921875, |
| 0.12890625, |
| 0.2109375, |
| 0.16015625 |
| ], |
| [ |
| 0.6640625, |
| 0.0078125, |
| 0.046875, |
| 0.07421875 |
| ], |
| [ |
| 0.55859375, |
| -0.35546875, |
| -0.09375, |
| 0.12890625 |
| ], |
| [ |
| 0.38671875, |
| -0.08984375, |
| -0.2265625, |
| 0.1015625 |
| ], |
| [ |
| 0.65234375, |
| -0.07421875, |
| 0.09765625, |
| 0.23046875 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 55004313088, |
| "reserved_bytes": 55580819456, |
| "peak_allocated_bytes": 55315161600, |
| "peak_reserved_bytes": 55580819456 |
| }, |
| "median_ms": 164.4560331478715 |
| }, |
| { |
| "case": 18, |
| "tokens": 2270, |
| "media": false, |
| "hidden_sha256": "c33c7d409cb1a0cea4cf70aaf6e7f6e8cc82afc886bf5a878bac43eb4f00c16a", |
| "logits": [ |
| [ |
| [ |
| 0.84375, |
| 0.09375, |
| 0.1640625, |
| 0.3671875 |
| ], |
| [ |
| 0.462890625, |
| -0.1484375, |
| -0.0546875, |
| 0.15625 |
| ], |
| [ |
| 0.29296875, |
| -0.328125, |
| -0.01953125, |
| 0.2109375 |
| ], |
| [ |
| 0.53125, |
| 0.05078125, |
| 0.24609375, |
| 0.4140625 |
| ], |
| [ |
| 0.25, |
| -0.3125, |
| 0.0234375, |
| 0.23046875 |
| ], |
| [ |
| 0.109375, |
| -0.47265625, |
| -0.3359375, |
| 0.05078125 |
| ], |
| [ |
| 0.03515625, |
| -0.43359375, |
| -0.22265625, |
| 0.16015625 |
| ], |
| [ |
| 0.0859375, |
| -0.5390625, |
| -0.23046875, |
| -0.1796875 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 55004325376, |
| "reserved_bytes": 56082038784, |
| "peak_allocated_bytes": 55473789440, |
| "peak_reserved_bytes": 56082038784 |
| }, |
| "median_ms": 241.0866366699338 |
| }, |
| { |
| "case": 19, |
| "tokens": 298, |
| "media": false, |
| "hidden_sha256": "e27ea5251b91abd33a98d4d116c88d118690188eb8f281e5a654583b1036788c", |
| "logits": [ |
| [ |
| [ |
| -3.4375, |
| -4.5, |
| -3.484375, |
| 0.953125 |
| ], |
| [ |
| -4.03125, |
| 0.28515625 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 55004293632, |
| "reserved_bytes": 56082038784, |
| "peak_allocated_bytes": 55067312128, |
| "peak_reserved_bytes": 56082038784 |
| }, |
| "median_ms": 77.37664878368378 |
| }, |
| { |
| "case": 20, |
| "tokens": 303, |
| "media": false, |
| "hidden_sha256": "303763a1268b7edbcc4c2fa0daae671a88f3ad2efb332825034937e7f4e8da6e", |
| "logits": [ |
| [ |
| [ |
| -4.8125, |
| -4.75, |
| 0.25, |
| -4.3125 |
| ], |
| [ |
| -0.3359375, |
| -3.234375 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 55004293632, |
| "reserved_bytes": 56082038784, |
| "peak_allocated_bytes": 55068987904, |
| "peak_reserved_bytes": 56082038784 |
| }, |
| "median_ms": 77.46736146509647 |
| }, |
| { |
| "case": 21, |
| "tokens": 304, |
| "media": false, |
| "hidden_sha256": "7067847630f4e78eead604854a28d7132ae44209f440b5667a549d569b0f3fcb", |
| "logits": [ |
| [ |
| [ |
| 0.9140625, |
| -4.4375, |
| -4.15625, |
| -3.296875 |
| ], |
| [ |
| -4.5625, |
| 0.072265625 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 55004293632, |
| "reserved_bytes": 56082038784, |
| "peak_allocated_bytes": 55069084160, |
| "peak_reserved_bytes": 56082038784 |
| }, |
| "median_ms": 77.56144786253572 |
| }, |
| { |
| "case": 22, |
| "tokens": 312, |
| "media": false, |
| "hidden_sha256": "47d4e8a47aff2f2b09ac4860a33d08cee653f449864b1379bb6237448ee43421", |
| "logits": [ |
| [ |
| [ |
| -3.6875, |
| -2.03125, |
| 1.375, |
| -3.90625 |
| ], |
| [ |
| -1.3203125, |
| -3.03125 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 55004293632, |
| "reserved_bytes": 56082038784, |
| "peak_allocated_bytes": 55070438912, |
| "peak_reserved_bytes": 56082038784 |
| }, |
| "median_ms": 77.67248200252652 |
| }, |
| { |
| "case": 23, |
| "tokens": 379, |
| "media": true, |
| "hidden_sha256": "5031e15c6e427ef014bdae66dc6f10e88053e7d826058769639790edb198ba2b", |
| "logits": [ |
| [ |
| [ |
| -3.671875, |
| -1.6171875, |
| 1.6015625, |
| -3.65625 |
| ], |
| [ |
| -1.9921875, |
| -2.5 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 55005871616, |
| "reserved_bytes": 56084135936, |
| "peak_allocated_bytes": 55087966720, |
| "peak_reserved_bytes": 56084135936 |
| }, |
| "median_ms": 89.13406962528825 |
| } |
| ], |
| "cold": { |
| "allocated_bytes": 54969685504, |
| "reserved_bytes": 54987325440, |
| "peak_allocated_bytes": 54969685504, |
| "peak_reserved_bytes": 54987325440, |
| "seconds": 13.1426798440516, |
| "nvml_process_peak_bytes": 55626956800 |
| }, |
| "repeat": 30, |
| "runtime_patched": false, |
| "runtime_sha256": { |
| "clef_int8_loader.py": "286752ba8076f222115a8e0cca9396cf6ca084d3fce2407d19aa2b3176fe7b5d", |
| "clef_int8_calibration.py": "135611910883019123eb1cb91dee7c2248e0e4e92c9563414c684d24bca794b0", |
| "clef_text_int8.py": "14f5ea9bfd5e21819cf40cf2176c87ceaa4f00d633cf658917256d4667fda84f", |
| "clef_int8_cutlass.py": "a516fa0c8007aa61814ea3e0ed251a3c0f0da626161bf2fe5ac30141732f857b", |
| "clef_text_cute.py": "ddc39f8bd543b7c6400e8c178ad408f0944fc7f1c881f4d36861ef497516689f", |
| "clef_text_math.py": "7b7f8c489fdb56b29c86a003d37ed96f440afe0fbe5001250036aad3e79ca760", |
| "clef_vision_norm.py": "398b681d0600dade5dc26f7664fa450ec39794f5615acac561c2032648947e6f" |
| }, |
| "nvml_process_peak_bytes": 56805556224, |
| "nvml_samples": 16022, |
| "batching": [ |
| { |
| "batch_size": 1, |
| "tokens_per_record": 1502, |
| "questions_per_record": 8, |
| "repeat": 10, |
| "batch_one_logits_max_abs": 0.0, |
| "batch_one_decisions_agree": true, |
| "status": "ok", |
| "latency_ms": { |
| "mean": 165.03022303804755, |
| "median": 164.39128015190363, |
| "p95": 172.891135327518, |
| "min": 161.52335330843925, |
| "max": 172.891135327518 |
| }, |
| "memory": { |
| "allocated_bytes": 55004317696, |
| "reserved_bytes": 56084135936, |
| "peak_allocated_bytes": 55315166208, |
| "peak_reserved_bytes": 56084135936 |
| }, |
| "decisions_per_second": 48.664381666763006, |
| "records_per_second": 6.083047708345376 |
| }, |
| { |
| "batch_size": 2, |
| "tokens_per_record": 1502, |
| "questions_per_record": 8, |
| "repeat": 10, |
| "batch_one_logits_max_abs": 0.30078125, |
| "batch_one_decisions_agree": true, |
| "status": "ok", |
| "latency_ms": { |
| "mean": 314.9787465110421, |
| "median": 314.8134066723287, |
| "p95": 321.4167505502701, |
| "min": 308.3176026120782, |
| "max": 321.4167505502701 |
| }, |
| "memory": { |
| "allocated_bytes": 55004369920, |
| "reserved_bytes": 56621006848, |
| "peak_allocated_bytes": 55625633792, |
| "peak_reserved_bytes": 56621006848 |
| }, |
| "decisions_per_second": 50.8237567425249, |
| "records_per_second": 6.3529695928156125 |
| }, |
| { |
| "batch_size": 4, |
| "tokens_per_record": 1502, |
| "questions_per_record": 8, |
| "repeat": 10, |
| "batch_one_logits_max_abs": 0.46875, |
| "batch_one_decisions_agree": true, |
| "status": "ok", |
| "latency_ms": { |
| "mean": 608.3244372159243, |
| "median": 607.7957293018699, |
| "p95": 611.246588639915, |
| "min": 605.1242211833596, |
| "max": 611.246588639915 |
| }, |
| "memory": { |
| "allocated_bytes": 55004441600, |
| "reserved_bytes": 57308872704, |
| "peak_allocated_bytes": 56246851584, |
| "peak_reserved_bytes": 57308872704 |
| }, |
| "decisions_per_second": 52.649267603041636, |
| "records_per_second": 6.5811584503802045 |
| }, |
| { |
| "batch_size": 8, |
| "tokens_per_record": 1502, |
| "questions_per_record": 8, |
| "repeat": 10, |
| "batch_one_logits_max_abs": 0.46875, |
| "batch_one_decisions_agree": true, |
| "status": "ok", |
| "latency_ms": { |
| "mean": 1205.6570419110358, |
| "median": 1204.5913855545223, |
| "p95": 1209.4773035496473, |
| "min": 1202.9799250885844, |
| "max": 1209.4773035496473 |
| }, |
| "memory": { |
| "allocated_bytes": 55004585984, |
| "reserved_bytes": 60299411456, |
| "peak_allocated_bytes": 57488294912, |
| "peak_reserved_bytes": 60299411456 |
| }, |
| "decisions_per_second": 53.1300495483273, |
| "records_per_second": 6.6412561935409125 |
| }, |
| { |
| "batch_size": 16, |
| "tokens_per_record": 1502, |
| "questions_per_record": 8, |
| "repeat": 10, |
| "batch_one_logits_max_abs": 0.46875, |
| "batch_one_decisions_agree": true, |
| "status": "ok", |
| "latency_ms": { |
| "mean": 2402.7484238147736, |
| "median": 2402.016118168831, |
| "p95": 2409.259708598256, |
| "min": 2395.667601376772, |
| "max": 2409.259708598256 |
| }, |
| "memory": { |
| "allocated_bytes": 55004874752, |
| "reserved_bytes": 65877835776, |
| "peak_allocated_bytes": 59975277568, |
| "peak_reserved_bytes": 65877835776 |
| }, |
| "decisions_per_second": 53.28856831218118, |
| "records_per_second": 6.661071039022647 |
| } |
| ], |
| "batching_nvml_process_peak_bytes": 66603450368, |
| "peak_allocated_bytes": 55473789440, |
| "peak_reserved_bytes": 56084135936 |
| }, |
| "bf16_cute": { |
| "mode": "bf16-cute", |
| "gpu": "NVIDIA H100 80GB HBM3", |
| "torch": "2.9.0a0+50eac811a6.nv25.09", |
| "memory_metric": "PyTorch allocated/reserved bytes; NVML process memory sampled every 5 ms", |
| "records": [ |
| { |
| "case": 0, |
| "tokens": 620, |
| "media": false, |
| "hidden_sha256": "a597b8195ef8a3bcade2f86d3768688e097b66a771b3377af2ea43cc00f6969a", |
| "logits": [ |
| [ |
| [ |
| -3.125, |
| -2.03125, |
| -0.359375, |
| -3.4375 |
| ], |
| [ |
| -1.015625, |
| -1.6171875, |
| -1.171875 |
| ], |
| [ |
| 1.640625, |
| -3.15625 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "bf16", |
| "memory": { |
| "allocated_bytes": 55004323328, |
| "reserved_bytes": 55192846336, |
| "peak_allocated_bytes": 55113447424, |
| "peak_reserved_bytes": 55192846336 |
| }, |
| "median_ms": 69.01239790022373 |
| }, |
| { |
| "case": 1, |
| "tokens": 449, |
| "media": false, |
| "hidden_sha256": "85fb727a649afba0f3b8c9c81825bcb8f2ccaceaa84aa5b851d475029b719ba3", |
| "logits": [ |
| [ |
| [ |
| -4.0, |
| -1.03125, |
| -1.5859375, |
| -3.609375 |
| ], |
| [ |
| -1.4375, |
| -2.359375, |
| -1.03125 |
| ], |
| [ |
| 0.828125, |
| -2.75 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "bf16", |
| "memory": { |
| "allocated_bytes": 55004321280, |
| "reserved_bytes": 55243177984, |
| "peak_allocated_bytes": 55086108672, |
| "peak_reserved_bytes": 55243177984 |
| }, |
| "median_ms": 67.6680295728147 |
| }, |
| { |
| "case": 2, |
| "tokens": 622, |
| "media": false, |
| "hidden_sha256": "1f1348abf8e3ba0c49203487244b4b3b44da05204fca2e52f23c6bbc899088c1", |
| "logits": [ |
| [ |
| [ |
| -3.921875, |
| -0.8828125, |
| -0.140625, |
| -3.78125 |
| ], |
| [ |
| -0.57421875, |
| -2.90625, |
| -1.09375 |
| ], |
| [ |
| 1.28125, |
| -3.3125 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "bf16", |
| "memory": { |
| "allocated_bytes": 55004323328, |
| "reserved_bytes": 55243177984, |
| "peak_allocated_bytes": 55113613312, |
| "peak_reserved_bytes": 55243177984 |
| }, |
| "median_ms": 69.76490188390017 |
| }, |
| { |
| "case": 3, |
| "tokens": 490, |
| "media": false, |
| "hidden_sha256": "5091a522f40ff5dc7d795b6f2d01735838286a07f12cf7a0fb696f0a5e4cd292", |
| "logits": [ |
| [ |
| [ |
| -3.9375, |
| -0.96875, |
| -0.75, |
| -4.375 |
| ], |
| [ |
| -0.5625, |
| -3.5, |
| -2.53125 |
| ], |
| [ |
| 1.2109375, |
| -2.875 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "bf16", |
| "memory": { |
| "allocated_bytes": 55004321280, |
| "reserved_bytes": 55291412480, |
| "peak_allocated_bytes": 55091483136, |
| "peak_reserved_bytes": 55291412480 |
| }, |
| "median_ms": 68.77257069572806 |
| }, |
| { |
| "case": 4, |
| "tokens": 448, |
| "media": false, |
| "hidden_sha256": "860c8cff7f484291cee00b1b6a10c759a4e636d2b39b022935d3e450e4360c27", |
| "logits": [ |
| [ |
| [ |
| -4.46875, |
| 0.57421875, |
| -3.265625, |
| -4.5 |
| ], |
| [ |
| 0.59375, |
| -4.3125, |
| -3.015625 |
| ], |
| [ |
| -2.109375, |
| -0.94921875 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "bf16", |
| "memory": { |
| "allocated_bytes": 55004320256, |
| "reserved_bytes": 55291412480, |
| "peak_allocated_bytes": 55084488704, |
| "peak_reserved_bytes": 55291412480 |
| }, |
| "median_ms": 68.35763994604349 |
| }, |
| { |
| "case": 5, |
| "tokens": 437, |
| "media": false, |
| "hidden_sha256": "158204930cd0fcf01c6b0287f8fc9a52ee17dc1f2f947d80d89739a56cad9601", |
| "logits": [ |
| [ |
| [ |
| -3.765625, |
| -1.0703125, |
| -2.015625, |
| -3.875 |
| ], |
| [ |
| -0.69921875, |
| -2.78125, |
| -1.6484375 |
| ], |
| [ |
| 0.94921875, |
| -2.53125 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "bf16", |
| "memory": { |
| "allocated_bytes": 55004320256, |
| "reserved_bytes": 55291412480, |
| "peak_allocated_bytes": 55081342976, |
| "peak_reserved_bytes": 55291412480 |
| }, |
| "median_ms": 68.14693473279476 |
| }, |
| { |
| "case": 6, |
| "tokens": 494, |
| "media": false, |
| "hidden_sha256": "764e10d942f237bbb0c977a5f505331000194a15a17b9ff3873598149961898a", |
| "logits": [ |
| [ |
| [ |
| -2.59375, |
| -2.96875, |
| -0.3125, |
| -3.5 |
| ], |
| [ |
| -3.09375, |
| -1.1328125, |
| -0.90625 |
| ], |
| [ |
| 1.6484375, |
| -3.25 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "bf16", |
| "memory": { |
| "allocated_bytes": 55004321280, |
| "reserved_bytes": 55341744128, |
| "peak_allocated_bytes": 55091749888, |
| "peak_reserved_bytes": 55341744128 |
| }, |
| "median_ms": 69.38431225717068 |
| }, |
| { |
| "case": 7, |
| "tokens": 449, |
| "media": false, |
| "hidden_sha256": "cd9636133adc36071ba54ab87a7cf99211ff713a88d866a966c64b53750a8637", |
| "logits": [ |
| [ |
| [ |
| -3.9375, |
| -1.140625, |
| -2.09375, |
| -3.890625 |
| ], |
| [ |
| 0.3203125, |
| -4.03125, |
| -2.859375 |
| ], |
| [ |
| 1.34375, |
| -3.03125 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "bf16", |
| "memory": { |
| "allocated_bytes": 55004321280, |
| "reserved_bytes": 55341744128, |
| "peak_allocated_bytes": 55086108672, |
| "peak_reserved_bytes": 55341744128 |
| }, |
| "median_ms": 68.46183445304632 |
| }, |
| { |
| "case": 8, |
| "tokens": 491, |
| "media": false, |
| "hidden_sha256": "8e3dd892fa0fe4e7a852f5fc094a96860954f0685cd86bd0bbad037ba5805315", |
| "logits": [ |
| [ |
| [ |
| -4.1875, |
| -2.46875, |
| -0.33984375, |
| -3.875 |
| ], |
| [ |
| -1.890625, |
| -3.0, |
| -1.265625 |
| ], |
| [ |
| 1.6328125, |
| -3.59375 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "bf16", |
| "memory": { |
| "allocated_bytes": 55004321280, |
| "reserved_bytes": 55341744128, |
| "peak_allocated_bytes": 55091551232, |
| "peak_reserved_bytes": 55341744128 |
| }, |
| "median_ms": 69.28329588845372 |
| }, |
| { |
| "case": 9, |
| "tokens": 624, |
| "media": false, |
| "hidden_sha256": "1620d52e2b2963e6f7f28ee3b970ae294a01ee10d22fc4cb7fe99c46e8f2e085", |
| "logits": [ |
| [ |
| [ |
| -3.109375, |
| -0.890625, |
| -0.71875, |
| -3.765625 |
| ], |
| [ |
| -0.1484375, |
| -2.625, |
| -1.53125 |
| ], |
| [ |
| 1.359375, |
| -3.375 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "bf16", |
| "memory": { |
| "allocated_bytes": 55004323328, |
| "reserved_bytes": 55341744128, |
| "peak_allocated_bytes": 55113777152, |
| "peak_reserved_bytes": 55341744128 |
| }, |
| "median_ms": 69.12704976275563 |
| }, |
| { |
| "case": 10, |
| "tokens": 454, |
| "media": false, |
| "hidden_sha256": "18e815e952a4255661599704e290b5f37c3187b8c068cc0d535566a16f7f99f9", |
| "logits": [ |
| [ |
| [ |
| -4.34375, |
| -2.484375, |
| -0.21484375, |
| -3.921875 |
| ], |
| [ |
| -2.0, |
| -2.703125, |
| -1.1640625 |
| ], |
| [ |
| 1.4453125, |
| -3.21875 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "bf16", |
| "memory": { |
| "allocated_bytes": 55004321280, |
| "reserved_bytes": 55341744128, |
| "peak_allocated_bytes": 55086318080, |
| "peak_reserved_bytes": 55341744128 |
| }, |
| "median_ms": 68.77609435468912 |
| }, |
| { |
| "case": 11, |
| "tokens": 442, |
| "media": false, |
| "hidden_sha256": "e6d2eb173ad497fcff31054f9767fcbbe8604bccf1b9317b2686712d4db88e8e", |
| "logits": [ |
| [ |
| [ |
| -4.0, |
| -0.384765625, |
| -1.125, |
| -4.28125 |
| ], |
| [ |
| 0.2099609375, |
| -4.15625, |
| -3.296875 |
| ], |
| [ |
| -0.2109375, |
| -1.28125 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "bf16", |
| "memory": { |
| "allocated_bytes": 55004320256, |
| "reserved_bytes": 55341744128, |
| "peak_allocated_bytes": 55082518016, |
| "peak_reserved_bytes": 55341744128 |
| }, |
| "median_ms": 68.66937736049294 |
| }, |
| { |
| "case": 12, |
| "tokens": 458, |
| "media": false, |
| "hidden_sha256": "1469570ef5926d2f8a4899966bfeeea0676cdf5eaaf53364e5b9a281cb4929f6", |
| "logits": [ |
| [ |
| [ |
| -2.125, |
| -3.796875, |
| -0.24609375, |
| -2.671875 |
| ], |
| [ |
| -3.953125, |
| -0.015625, |
| -1.75 |
| ], |
| [ |
| 1.4140625, |
| -2.703125 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "bf16", |
| "memory": { |
| "allocated_bytes": 55004321280, |
| "reserved_bytes": 55341744128, |
| "peak_allocated_bytes": 55087033856, |
| "peak_reserved_bytes": 55341744128 |
| }, |
| "median_ms": 68.95825685933232 |
| }, |
| { |
| "case": 13, |
| "tokens": 620, |
| "media": false, |
| "hidden_sha256": "48880c8ff6215d9e21b7b91346a95ed21689afc1176688d73cb8f7aa141f3486", |
| "logits": [ |
| [ |
| [ |
| -3.125, |
| -1.9140625, |
| -0.88671875, |
| -3.984375 |
| ], |
| [ |
| -0.82421875, |
| -2.28125, |
| -1.3359375 |
| ], |
| [ |
| 1.7734375, |
| -3.34375 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "bf16", |
| "memory": { |
| "allocated_bytes": 55004323328, |
| "reserved_bytes": 55341744128, |
| "peak_allocated_bytes": 55113447424, |
| "peak_reserved_bytes": 55341744128 |
| }, |
| "median_ms": 70.10784558951855 |
| }, |
| { |
| "case": 14, |
| "tokens": 616, |
| "media": false, |
| "hidden_sha256": "43d102db01a1c45d1148a5217938cf40a71d5e2b8ce1e11e79f640a0f3639614", |
| "logits": [ |
| [ |
| [ |
| -3.296875, |
| -0.9140625, |
| -1.1171875, |
| -3.90625 |
| ], |
| [ |
| -0.27734375, |
| -3.046875, |
| -2.046875 |
| ], |
| [ |
| 1.234375, |
| -3.578125 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "bf16", |
| "memory": { |
| "allocated_bytes": 55004323328, |
| "reserved_bytes": 55341744128, |
| "peak_allocated_bytes": 55113114112, |
| "peak_reserved_bytes": 55341744128 |
| }, |
| "median_ms": 68.83620982989669 |
| }, |
| { |
| "case": 15, |
| "tokens": 493, |
| "media": false, |
| "hidden_sha256": "b24dcfc763451f39b010a276f8e6fd985a7c2bf9603816333f9dfc6427508ded", |
| "logits": [ |
| [ |
| [ |
| -2.5, |
| -2.109375, |
| -0.00390625, |
| -3.78125 |
| ], |
| [ |
| -1.5, |
| -1.6796875, |
| -1.4375 |
| ], |
| [ |
| 1.28125, |
| -3.125 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "bf16", |
| "memory": { |
| "allocated_bytes": 55004321280, |
| "reserved_bytes": 55341744128, |
| "peak_allocated_bytes": 55091683328, |
| "peak_reserved_bytes": 55341744128 |
| }, |
| "median_ms": 68.89336928725243 |
| }, |
| { |
| "case": 16, |
| "tokens": 1278, |
| "media": false, |
| "hidden_sha256": "63d5d9aa787a0c2c8bd467d55405d2a5074eef3e94d726f651a692506a55aaa5", |
| "logits": [ |
| [ |
| [ |
| 1.109375, |
| 0.29296875, |
| 0.59375, |
| 0.7421875 |
| ], |
| [ |
| 0.75, |
| 0.015625, |
| 0.234375, |
| 0.5234375 |
| ], |
| [ |
| 0.5234375, |
| -0.134765625, |
| 0.138671875, |
| 0.296875 |
| ], |
| [ |
| 0.4375, |
| -0.09375, |
| 0.0, |
| 0.1640625 |
| ], |
| [ |
| 0.451171875, |
| -0.251953125, |
| -0.126953125, |
| 0.03125 |
| ], |
| [ |
| 0.3203125, |
| -0.3359375, |
| -0.1953125, |
| -0.05078125 |
| ], |
| [ |
| 0.1484375, |
| -0.296875, |
| -0.171875, |
| 0.0859375 |
| ], |
| [ |
| 0.19140625, |
| -0.22265625, |
| -0.11328125, |
| 0.32421875 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "bf16", |
| "memory": { |
| "allocated_bytes": 55004333568, |
| "reserved_bytes": 55341744128, |
| "peak_allocated_bytes": 55225634304, |
| "peak_reserved_bytes": 55341744128 |
| }, |
| "median_ms": 121.83641782030463 |
| }, |
| { |
| "case": 17, |
| "tokens": 1502, |
| "media": false, |
| "hidden_sha256": "579420a72e605ed1622ca66435162bb7e7241b143240616fb5745ae5542aed09", |
| "logits": [ |
| [ |
| [ |
| 1.1640625, |
| 0.35546875, |
| 0.4453125, |
| 0.46484375 |
| ], |
| [ |
| 0.55078125, |
| -0.04296875, |
| 0.06640625, |
| 0.158203125 |
| ], |
| [ |
| 0.57421875, |
| -0.0390625, |
| 0.32421875, |
| 0.26953125 |
| ], |
| [ |
| 0.69921875, |
| 0.12890625, |
| 0.2109375, |
| 0.16015625 |
| ], |
| [ |
| 0.6640625, |
| 0.0078125, |
| 0.046875, |
| 0.07421875 |
| ], |
| [ |
| 0.55859375, |
| -0.35546875, |
| -0.09375, |
| 0.12890625 |
| ], |
| [ |
| 0.38671875, |
| -0.08984375, |
| -0.2265625, |
| 0.1015625 |
| ], |
| [ |
| 0.65234375, |
| -0.07421875, |
| 0.09765625, |
| 0.23046875 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "bf16", |
| "memory": { |
| "allocated_bytes": 55004337664, |
| "reserved_bytes": 55499030528, |
| "peak_allocated_bytes": 55265444352, |
| "peak_reserved_bytes": 55499030528 |
| }, |
| "median_ms": 139.84269881621003 |
| }, |
| { |
| "case": 18, |
| "tokens": 2270, |
| "media": false, |
| "hidden_sha256": "c33c7d409cb1a0cea4cf70aaf6e7f6e8cc82afc886bf5a878bac43eb4f00c16a", |
| "logits": [ |
| [ |
| [ |
| 0.84375, |
| 0.09375, |
| 0.1640625, |
| 0.3671875 |
| ], |
| [ |
| 0.462890625, |
| -0.1484375, |
| -0.0546875, |
| 0.15625 |
| ], |
| [ |
| 0.29296875, |
| -0.328125, |
| -0.01953125, |
| 0.2109375 |
| ], |
| [ |
| 0.53125, |
| 0.05078125, |
| 0.24609375, |
| 0.4140625 |
| ], |
| [ |
| 0.25, |
| -0.3125, |
| 0.0234375, |
| 0.23046875 |
| ], |
| [ |
| 0.109375, |
| -0.47265625, |
| -0.3359375, |
| 0.05078125 |
| ], |
| [ |
| 0.03515625, |
| -0.43359375, |
| -0.22265625, |
| 0.16015625 |
| ], |
| [ |
| 0.0859375, |
| -0.5390625, |
| -0.23046875, |
| -0.1796875 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "bf16", |
| "memory": { |
| "allocated_bytes": 55004349952, |
| "reserved_bytes": 55882809344, |
| "peak_allocated_bytes": 55398115840, |
| "peak_reserved_bytes": 55882809344 |
| }, |
| "median_ms": 208.45949463546276 |
| }, |
| { |
| "case": 19, |
| "tokens": 298, |
| "media": false, |
| "hidden_sha256": "e27ea5251b91abd33a98d4d116c88d118690188eb8f281e5a654583b1036788c", |
| "logits": [ |
| [ |
| [ |
| -3.4375, |
| -4.5, |
| -3.484375, |
| 0.953125 |
| ], |
| [ |
| -4.03125, |
| 0.28515625 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "bf16", |
| "memory": { |
| "allocated_bytes": 55004318208, |
| "reserved_bytes": 55882809344, |
| "peak_allocated_bytes": 55056820224, |
| "peak_reserved_bytes": 55882809344 |
| }, |
| "median_ms": 67.75721022859216 |
| }, |
| { |
| "case": 20, |
| "tokens": 303, |
| "media": false, |
| "hidden_sha256": "303763a1268b7edbcc4c2fa0daae671a88f3ad2efb332825034937e7f4e8da6e", |
| "logits": [ |
| [ |
| [ |
| -4.8125, |
| -4.75, |
| 0.25, |
| -4.3125 |
| ], |
| [ |
| -0.3359375, |
| -3.234375 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "bf16", |
| "memory": { |
| "allocated_bytes": 55004318208, |
| "reserved_bytes": 55882809344, |
| "peak_allocated_bytes": 55057357312, |
| "peak_reserved_bytes": 55882809344 |
| }, |
| "median_ms": 68.8166175968945 |
| }, |
| { |
| "case": 21, |
| "tokens": 304, |
| "media": false, |
| "hidden_sha256": "7067847630f4e78eead604854a28d7132ae44209f440b5667a549d569b0f3fcb", |
| "logits": [ |
| [ |
| [ |
| 0.9140625, |
| -4.4375, |
| -4.15625, |
| -3.296875 |
| ], |
| [ |
| -4.5625, |
| 0.072265625 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "bf16", |
| "memory": { |
| "allocated_bytes": 55004318208, |
| "reserved_bytes": 55882809344, |
| "peak_allocated_bytes": 55057463808, |
| "peak_reserved_bytes": 55882809344 |
| }, |
| "median_ms": 67.84480484202504 |
| }, |
| { |
| "case": 22, |
| "tokens": 312, |
| "media": false, |
| "hidden_sha256": "47d4e8a47aff2f2b09ac4860a33d08cee653f449864b1379bb6237448ee43421", |
| "logits": [ |
| [ |
| [ |
| -3.6875, |
| -2.03125, |
| 1.375, |
| -3.90625 |
| ], |
| [ |
| -1.3203125, |
| -3.03125 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "bf16", |
| "memory": { |
| "allocated_bytes": 55004318208, |
| "reserved_bytes": 55882809344, |
| "peak_allocated_bytes": 55060323840, |
| "peak_reserved_bytes": 55882809344 |
| }, |
| "median_ms": 67.88387056440115 |
| }, |
| { |
| "case": 23, |
| "tokens": 379, |
| "media": true, |
| "hidden_sha256": "5031e15c6e427ef014bdae66dc6f10e88053e7d826058769639790edb198ba2b", |
| "logits": [ |
| [ |
| [ |
| -3.671875, |
| -1.6171875, |
| 1.6015625, |
| -3.65625 |
| ], |
| [ |
| -1.9921875, |
| -2.5 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "bf16", |
| "memory": { |
| "allocated_bytes": 55005896192, |
| "reserved_bytes": 55884906496, |
| "peak_allocated_bytes": 55074950656, |
| "peak_reserved_bytes": 55884906496 |
| }, |
| "median_ms": 82.2020024061203 |
| } |
| ], |
| "cold": { |
| "allocated_bytes": 54969710080, |
| "reserved_bytes": 54987325440, |
| "peak_allocated_bytes": 54969710592, |
| "peak_reserved_bytes": 54987325440, |
| "seconds": 12.89094135724008, |
| "nvml_process_peak_bytes": 55626956800 |
| }, |
| "repeat": 30, |
| "runtime_patched": true, |
| "runtime_sha256": { |
| "clef_int8_loader.py": "286752ba8076f222115a8e0cca9396cf6ca084d3fce2407d19aa2b3176fe7b5d", |
| "clef_int8_calibration.py": "135611910883019123eb1cb91dee7c2248e0e4e92c9563414c684d24bca794b0", |
| "clef_text_int8.py": "14f5ea9bfd5e21819cf40cf2176c87ceaa4f00d633cf658917256d4667fda84f", |
| "clef_int8_cutlass.py": "a516fa0c8007aa61814ea3e0ed251a3c0f0da626161bf2fe5ac30141732f857b", |
| "clef_text_cute.py": "ddc39f8bd543b7c6400e8c178ad408f0944fc7f1c881f4d36861ef497516689f", |
| "clef_text_math.py": "7b7f8c489fdb56b29c86a003d37ed96f440afe0fbe5001250036aad3e79ca760", |
| "clef_vision_norm.py": "398b681d0600dade5dc26f7664fa450ec39794f5615acac561c2032648947e6f" |
| }, |
| "nvml_process_peak_bytes": 56608423936, |
| "nvml_samples": 15437, |
| "batching": [ |
| { |
| "batch_size": 1, |
| "tokens_per_record": 1502, |
| "questions_per_record": 8, |
| "repeat": 10, |
| "batch_one_logits_max_abs": 0.0, |
| "batch_one_decisions_agree": true, |
| "status": "ok", |
| "latency_ms": { |
| "mean": 141.42045499756932, |
| "median": 140.2929974719882, |
| "p95": 150.26023890823126, |
| "min": 136.18104439228773, |
| "max": 150.26023890823126 |
| }, |
| "memory": { |
| "allocated_bytes": 55004342272, |
| "reserved_bytes": 55884906496, |
| "peak_allocated_bytes": 55265448960, |
| "peak_reserved_bytes": 55884906496 |
| }, |
| "decisions_per_second": 57.02351609956392, |
| "records_per_second": 7.12793951244549 |
| }, |
| { |
| "batch_size": 2, |
| "tokens_per_record": 1502, |
| "questions_per_record": 8, |
| "repeat": 10, |
| "batch_one_logits_max_abs": 0.30078125, |
| "batch_one_decisions_agree": true, |
| "status": "ok", |
| "latency_ms": { |
| "mean": 270.84807669743896, |
| "median": 268.2279543951154, |
| "p95": 284.2757198959589, |
| "min": 264.37197905033827, |
| "max": 284.2757198959589 |
| }, |
| "memory": { |
| "allocated_bytes": 55004394496, |
| "reserved_bytes": 56199479296, |
| "peak_allocated_bytes": 55526315008, |
| "peak_reserved_bytes": 56199479296 |
| }, |
| "decisions_per_second": 59.6507550306672, |
| "records_per_second": 7.4563443788334 |
| }, |
| { |
| "batch_size": 4, |
| "tokens_per_record": 1502, |
| "questions_per_record": 8, |
| "repeat": 10, |
| "batch_one_logits_max_abs": 0.46875, |
| "batch_one_decisions_agree": true, |
| "status": "ok", |
| "latency_ms": { |
| "mean": 528.4846080467105, |
| "median": 529.0066166780889, |
| "p95": 536.1536452546716, |
| "min": 518.8928348943591, |
| "max": 536.1536452546716 |
| }, |
| "memory": { |
| "allocated_bytes": 55004466176, |
| "reserved_bytes": 57302581248, |
| "peak_allocated_bytes": 56048422400, |
| "peak_reserved_bytes": 57302581248 |
| }, |
| "decisions_per_second": 60.49073677177206, |
| "records_per_second": 7.561342096471508 |
| }, |
| { |
| "batch_size": 8, |
| "tokens_per_record": 1502, |
| "questions_per_record": 8, |
| "repeat": 10, |
| "batch_one_logits_max_abs": 0.46875, |
| "batch_one_decisions_agree": true, |
| "status": "ok", |
| "latency_ms": { |
| "mean": 1052.5208386592567, |
| "median": 1052.322378847748, |
| "p95": 1054.5621123164892, |
| "min": 1049.6748918667436, |
| "max": 1054.5621123164892 |
| }, |
| "memory": { |
| "allocated_bytes": 55004610560, |
| "reserved_bytes": 59506688000, |
| "peak_allocated_bytes": 57094228992, |
| "peak_reserved_bytes": 59506688000 |
| }, |
| "decisions_per_second": 60.81786464531668, |
| "records_per_second": 7.602233080664585 |
| }, |
| { |
| "batch_size": 16, |
| "tokens_per_record": 1502, |
| "questions_per_record": 8, |
| "repeat": 10, |
| "batch_one_logits_max_abs": 0.46875, |
| "batch_one_decisions_agree": true, |
| "status": "ok", |
| "latency_ms": { |
| "mean": 2097.766940947622, |
| "median": 2097.9305459186435, |
| "p95": 2102.448247373104, |
| "min": 2094.7574246674776, |
| "max": 2102.448247373104 |
| }, |
| "memory": { |
| "allocated_bytes": 55004899328, |
| "reserved_bytes": 63902318592, |
| "peak_allocated_bytes": 59180298240, |
| "peak_reserved_bytes": 63902318592 |
| }, |
| "decisions_per_second": 61.012505990254915, |
| "records_per_second": 7.626563248781864 |
| } |
| ], |
| "batching_nvml_process_peak_bytes": 64627933184, |
| "peak_allocated_bytes": 55398115840, |
| "peak_reserved_bytes": 55884906496 |
| }, |
| "calibrated_int8_torch": { |
| "mode": "int8-torch", |
| "gpu": "NVIDIA H100 80GB HBM3", |
| "torch": "2.9.0a0+50eac811a6.nv25.09", |
| "memory_metric": "PyTorch allocated/reserved bytes; NVML process memory sampled every 5 ms", |
| "records": [ |
| { |
| "case": 0, |
| "tokens": 620, |
| "media": false, |
| "hidden_sha256": "3bd80cec0945df8323e7f97e1c446dc064516975aa15b94f5a8808715034a2a8", |
| "logits": [ |
| [ |
| [ |
| -3.140625, |
| -2.0, |
| -0.421875, |
| -3.46875 |
| ], |
| [ |
| -0.99609375, |
| -1.640625, |
| -1.21875 |
| ], |
| [ |
| 1.546875, |
| -3.21875 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 30917139968, |
| "reserved_bytes": 31234981888, |
| "peak_allocated_bytes": 31103866880, |
| "peak_reserved_bytes": 31234981888 |
| }, |
| "median_ms": 137.17662682756782 |
| }, |
| { |
| "case": 1, |
| "tokens": 449, |
| "media": false, |
| "hidden_sha256": "0e416bd18959ddda56a98b01d58fb30d7a66ee16a2dd5d34a68472dccc15dfc5", |
| "logits": [ |
| [ |
| [ |
| -3.953125, |
| -0.9609375, |
| -1.6015625, |
| -3.59375 |
| ], |
| [ |
| -1.5, |
| -2.28125, |
| -1.0 |
| ], |
| [ |
| 0.84375, |
| -2.6875 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 30917137920, |
| "reserved_bytes": 31285313536, |
| "peak_allocated_bytes": 31051187712, |
| "peak_reserved_bytes": 31285313536 |
| }, |
| "median_ms": 117.17480374500155 |
| }, |
| { |
| "case": 2, |
| "tokens": 622, |
| "media": false, |
| "hidden_sha256": "332b8329d6b9cf675783d97c37ac3bb9eb04953f6d1acdeda6fe3147ccb4fe16", |
| "logits": [ |
| [ |
| [ |
| -3.90625, |
| -0.87109375, |
| -0.142578125, |
| -3.828125 |
| ], |
| [ |
| -0.59375, |
| -2.90625, |
| -1.1015625 |
| ], |
| [ |
| 1.28125, |
| -3.28125 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 30917139968, |
| "reserved_bytes": 31285313536, |
| "peak_allocated_bytes": 31104019456, |
| "peak_reserved_bytes": 31285313536 |
| }, |
| "median_ms": 138.2606695406139 |
| }, |
| { |
| "case": 3, |
| "tokens": 490, |
| "media": false, |
| "hidden_sha256": "26362588c871286a4878b331e1d9ec29c920510e59623fea0d6d24e28fa39857", |
| "logits": [ |
| [ |
| [ |
| -3.90625, |
| -0.9609375, |
| -0.76953125, |
| -4.34375 |
| ], |
| [ |
| -0.5546875, |
| -3.453125, |
| -2.5 |
| ], |
| [ |
| 1.15625, |
| -2.90625 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 30917137920, |
| "reserved_bytes": 31333548032, |
| "peak_allocated_bytes": 31063337472, |
| "peak_reserved_bytes": 31333548032 |
| }, |
| "median_ms": 116.68091919273138 |
| }, |
| { |
| "case": 4, |
| "tokens": 448, |
| "media": false, |
| "hidden_sha256": "39bcbe99f06b8c754b1395b52aa1cf14e92ec5d38db64c7cc6b93c77200c475e", |
| "logits": [ |
| [ |
| [ |
| -4.46875, |
| 0.5703125, |
| -3.28125, |
| -4.4375 |
| ], |
| [ |
| 0.609375, |
| -4.28125, |
| -3.03125 |
| ], |
| [ |
| -2.203125, |
| -0.953125 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 30917136896, |
| "reserved_bytes": 31333548032, |
| "peak_allocated_bytes": 31048071168, |
| "peak_reserved_bytes": 31333548032 |
| }, |
| "median_ms": 109.10983849316835 |
| }, |
| { |
| "case": 5, |
| "tokens": 437, |
| "media": false, |
| "hidden_sha256": "937f7824d74e9a912f87b26807db9c82bfcf1f2e49925e1db99a3daaf797e478", |
| "logits": [ |
| [ |
| [ |
| -3.796875, |
| -1.109375, |
| -2.15625, |
| -3.90625 |
| ], |
| [ |
| -0.70703125, |
| -2.84375, |
| -1.7578125 |
| ], |
| [ |
| 0.8515625, |
| -2.703125 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 30917136896, |
| "reserved_bytes": 31333548032, |
| "peak_allocated_bytes": 31047681024, |
| "peak_reserved_bytes": 31333548032 |
| }, |
| "median_ms": 117.55172442644835 |
| }, |
| { |
| "case": 6, |
| "tokens": 494, |
| "media": false, |
| "hidden_sha256": "43d127c3d1d4a761ae05e8df2740a6f9c3e87c7e2505533dde9326a4c4f5b058", |
| "logits": [ |
| [ |
| [ |
| -2.578125, |
| -2.96875, |
| -0.484375, |
| -3.484375 |
| ], |
| [ |
| -3.125, |
| -1.109375, |
| -0.921875 |
| ], |
| [ |
| 1.6484375, |
| -3.234375 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 30917137920, |
| "reserved_bytes": 31383879680, |
| "peak_allocated_bytes": 31064884736, |
| "peak_reserved_bytes": 31383879680 |
| }, |
| "median_ms": 116.89481977373362 |
| }, |
| { |
| "case": 7, |
| "tokens": 449, |
| "media": false, |
| "hidden_sha256": "e1d3e1006ccccf6aacba450f8841f691cf92d51a40fb3be2342832e4c7798687", |
| "logits": [ |
| [ |
| [ |
| -3.90625, |
| -1.171875, |
| -1.984375, |
| -3.90625 |
| ], |
| [ |
| 0.28125, |
| -3.953125, |
| -2.796875 |
| ], |
| [ |
| 1.375, |
| -3.03125 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 30917137920, |
| "reserved_bytes": 31383879680, |
| "peak_allocated_bytes": 31051187712, |
| "peak_reserved_bytes": 31383879680 |
| }, |
| "median_ms": 117.62379622086883 |
| }, |
| { |
| "case": 8, |
| "tokens": 491, |
| "media": false, |
| "hidden_sha256": "4f5079806d430ae1d046b86d18a67af3cacdd7f62bb6ed3e7c99edecad048d31", |
| "logits": [ |
| [ |
| [ |
| -4.1875, |
| -2.46875, |
| -0.26953125, |
| -3.828125 |
| ], |
| [ |
| -1.7578125, |
| -2.90625, |
| -1.2265625 |
| ], |
| [ |
| 1.6796875, |
| -3.53125 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 30917137920, |
| "reserved_bytes": 31383879680, |
| "peak_allocated_bytes": 31063534080, |
| "peak_reserved_bytes": 31383879680 |
| }, |
| "median_ms": 117.2104193829 |
| }, |
| { |
| "case": 9, |
| "tokens": 624, |
| "media": false, |
| "hidden_sha256": "cdb5d5ec50846b16a8c87da98008b8c93e11b80570894fd3ec0ac6b28bb89cdf", |
| "logits": [ |
| [ |
| [ |
| -3.125, |
| -0.8984375, |
| -0.71875, |
| -3.765625 |
| ], |
| [ |
| -0.2109375, |
| -2.640625, |
| -1.5546875 |
| ], |
| [ |
| 1.3828125, |
| -3.359375 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 30917139968, |
| "reserved_bytes": 31383879680, |
| "peak_allocated_bytes": 31100976128, |
| "peak_reserved_bytes": 31383879680 |
| }, |
| "median_ms": 134.07096592709422 |
| }, |
| { |
| "case": 10, |
| "tokens": 454, |
| "media": false, |
| "hidden_sha256": "bc2bc8c35a51b4b082c379c097dac42759aedf388fbfbcf85f4b7a7971d0fb8b", |
| "logits": [ |
| [ |
| [ |
| -4.34375, |
| -2.59375, |
| -0.22265625, |
| -3.9375 |
| ], |
| [ |
| -2.09375, |
| -2.59375, |
| -1.1328125 |
| ], |
| [ |
| 1.4375, |
| -3.265625 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 30917137920, |
| "reserved_bytes": 31383879680, |
| "peak_allocated_bytes": 31053260288, |
| "peak_reserved_bytes": 31383879680 |
| }, |
| "median_ms": 117.72878700867295 |
| }, |
| { |
| "case": 11, |
| "tokens": 442, |
| "media": false, |
| "hidden_sha256": "caccd4f2f27ee39e8903a5e20fd24916ad6d670e66e99b8184da23725a33326b", |
| "logits": [ |
| [ |
| [ |
| -4.0, |
| -0.34765625, |
| -1.265625, |
| -4.3125 |
| ], |
| [ |
| 0.248046875, |
| -4.1875, |
| -3.3125 |
| ], |
| [ |
| -0.2578125, |
| -1.34375 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 30917136896, |
| "reserved_bytes": 31383879680, |
| "peak_allocated_bytes": 31049227264, |
| "peak_reserved_bytes": 31383879680 |
| }, |
| "median_ms": 117.89859551936388 |
| }, |
| { |
| "case": 12, |
| "tokens": 458, |
| "media": false, |
| "hidden_sha256": "412e18ef08c9375f461110ff5dd36114c13e517c6223db5df85bc20c9a045e3e", |
| "logits": [ |
| [ |
| [ |
| -2.125, |
| -3.796875, |
| -0.328125, |
| -2.671875 |
| ], |
| [ |
| -3.953125, |
| 0.001953125, |
| -1.71875 |
| ], |
| [ |
| 1.421875, |
| -2.65625 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 30917137920, |
| "reserved_bytes": 31383879680, |
| "peak_allocated_bytes": 31053707776, |
| "peak_reserved_bytes": 31383879680 |
| }, |
| "median_ms": 118.8604780472815 |
| }, |
| { |
| "case": 13, |
| "tokens": 620, |
| "media": false, |
| "hidden_sha256": "23a647bce8af2f2d6077fde8985dce1ebe78babbb63ed404ad555f0354b39555", |
| "logits": [ |
| [ |
| [ |
| -3.109375, |
| -1.8828125, |
| -0.87890625, |
| -3.953125 |
| ], |
| [ |
| -0.796875, |
| -2.28125, |
| -1.3515625 |
| ], |
| [ |
| 1.8203125, |
| -3.3125 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 30917139968, |
| "reserved_bytes": 31383879680, |
| "peak_allocated_bytes": 31103866880, |
| "peak_reserved_bytes": 31383879680 |
| }, |
| "median_ms": 137.25669821724296 |
| }, |
| { |
| "case": 14, |
| "tokens": 616, |
| "media": false, |
| "hidden_sha256": "6f9506754fcfb7e5766f7db8fdb8c73597304aa41ff38e67c41f033e26aca023", |
| "logits": [ |
| [ |
| [ |
| -3.328125, |
| -0.921875, |
| -1.140625, |
| -3.921875 |
| ], |
| [ |
| -0.26953125, |
| -2.984375, |
| -2.0625 |
| ], |
| [ |
| 1.21875, |
| -3.5625 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 30917139968, |
| "reserved_bytes": 31383879680, |
| "peak_allocated_bytes": 31096927232, |
| "peak_reserved_bytes": 31383879680 |
| }, |
| "median_ms": 133.38337745517492 |
| }, |
| { |
| "case": 15, |
| "tokens": 493, |
| "media": false, |
| "hidden_sha256": "920ca7b25e96d6dbb6bbd2585ff5931c71347c9e0a5d739298e6c9035e40e56b", |
| "logits": [ |
| [ |
| [ |
| -2.46875, |
| -2.09375, |
| 0.0078125, |
| -3.75 |
| ], |
| [ |
| -1.5546875, |
| -1.671875, |
| -1.359375 |
| ], |
| [ |
| 1.3046875, |
| -3.09375 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 30917137920, |
| "reserved_bytes": 31383879680, |
| "peak_allocated_bytes": 31064683008, |
| "peak_reserved_bytes": 31383879680 |
| }, |
| "median_ms": 117.7181527018547 |
| }, |
| { |
| "case": 16, |
| "tokens": 1278, |
| "media": false, |
| "hidden_sha256": "88bdd67d73f1462d1740dea4ebec6fdfa00e29dd76d60fe2bb3f585a0ffea188", |
| "logits": [ |
| [ |
| [ |
| 1.265625, |
| 0.40625, |
| 0.6953125, |
| 0.828125 |
| ], |
| [ |
| 0.859375, |
| 0.056640625, |
| 0.24609375, |
| 0.52734375 |
| ], |
| [ |
| 0.55859375, |
| -0.091796875, |
| 0.21875, |
| 0.314453125 |
| ], |
| [ |
| 0.51171875, |
| -0.0703125, |
| -0.0234375, |
| 0.2265625 |
| ], |
| [ |
| 0.5703125, |
| -0.150390625, |
| -0.03515625, |
| 0.0859375 |
| ], |
| [ |
| 0.4296875, |
| -0.25, |
| -0.177734375, |
| -0.015625 |
| ], |
| [ |
| 0.140625, |
| -0.197265625, |
| -0.203125, |
| 0.0390625 |
| ], |
| [ |
| 0.42578125, |
| -0.07421875, |
| -0.0234375, |
| 0.14453125 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 30917150208, |
| "reserved_bytes": 31811698688, |
| "peak_allocated_bytes": 31295658496, |
| "peak_reserved_bytes": 31811698688 |
| }, |
| "median_ms": 245.00332586467266 |
| }, |
| { |
| "case": 17, |
| "tokens": 1502, |
| "media": false, |
| "hidden_sha256": "620de62d3f3cd75f195a80ef7ce174225523fc4ea127254050b400c6b04708aa", |
| "logits": [ |
| [ |
| [ |
| 1.40625, |
| 0.5390625, |
| 0.69921875, |
| 0.7578125 |
| ], |
| [ |
| 0.7734375, |
| 0.1328125, |
| 0.25390625, |
| 0.42578125 |
| ], |
| [ |
| 0.8359375, |
| 0.1171875, |
| 0.51171875, |
| 0.453125 |
| ], |
| [ |
| 0.9453125, |
| 0.17578125, |
| 0.3515625, |
| 0.29296875 |
| ], |
| [ |
| 0.9375, |
| 0.09375, |
| 0.24609375, |
| 0.296875 |
| ], |
| [ |
| 0.7109375, |
| -0.1171875, |
| 0.078125, |
| 0.2265625 |
| ], |
| [ |
| 0.65234375, |
| 0.126953125, |
| 0.0546875, |
| 0.376953125 |
| ], |
| [ |
| 0.875, |
| 0.04296875, |
| 0.21875, |
| 0.26171875 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 30917154304, |
| "reserved_bytes": 32126271488, |
| "peak_allocated_bytes": 31361541632, |
| "peak_reserved_bytes": 32126271488 |
| }, |
| "median_ms": 278.91440875828266 |
| }, |
| { |
| "case": 18, |
| "tokens": 2270, |
| "media": false, |
| "hidden_sha256": "d6c682b59804bcd7b9bfd96f66c20a57268ce65ca24179a5da266433e0173cc2", |
| "logits": [ |
| [ |
| [ |
| 0.6953125, |
| -0.03515625, |
| 0.03515625, |
| 0.328125 |
| ], |
| [ |
| 0.3515625, |
| -0.236328125, |
| -0.10546875, |
| 0.1484375 |
| ], |
| [ |
| 0.22265625, |
| -0.34375, |
| -0.0546875, |
| 0.0625 |
| ], |
| [ |
| 0.421875, |
| -0.0390625, |
| 0.12890625, |
| 0.3671875 |
| ], |
| [ |
| 0.2734375, |
| -0.337890625, |
| -0.1015625, |
| 0.09375 |
| ], |
| [ |
| 0.078125, |
| -0.5234375, |
| -0.42578125, |
| 0.05078125 |
| ], |
| [ |
| -0.11328125, |
| -0.5078125, |
| -0.283203125, |
| 0.00390625 |
| ], |
| [ |
| -0.0234375, |
| -0.75390625, |
| -0.24609375, |
| -0.15234375 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 30917166592, |
| "reserved_bytes": 32604422144, |
| "peak_allocated_bytes": 31588405760, |
| "peak_reserved_bytes": 32604422144 |
| }, |
| "median_ms": 401.4103366062045 |
| }, |
| { |
| "case": 19, |
| "tokens": 298, |
| "media": false, |
| "hidden_sha256": "93a4b2a5b5ccc85db7ef415d90158e97efbc5cecfd28e41ee35cc11c2ac31c3a", |
| "logits": [ |
| [ |
| [ |
| -3.375, |
| -4.5, |
| -3.4375, |
| 0.984375 |
| ], |
| [ |
| -3.859375, |
| 0.1796875 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 30917134848, |
| "reserved_bytes": 32604422144, |
| "peak_allocated_bytes": 31007036928, |
| "peak_reserved_bytes": 32604422144 |
| }, |
| "median_ms": 116.23586062341928 |
| }, |
| { |
| "case": 20, |
| "tokens": 303, |
| "media": false, |
| "hidden_sha256": "c15885b277be05a2bc708b698408a398d7f631d0cb97640bd0fdbd9fa5efca32", |
| "logits": [ |
| [ |
| [ |
| -4.8125, |
| -4.71875, |
| 0.15625, |
| -4.34375 |
| ], |
| [ |
| -0.328125, |
| -3.234375 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 30917134848, |
| "reserved_bytes": 32604422144, |
| "peak_allocated_bytes": 31008309760, |
| "peak_reserved_bytes": 32604422144 |
| }, |
| "median_ms": 116.06947844848037 |
| }, |
| { |
| "case": 21, |
| "tokens": 304, |
| "media": false, |
| "hidden_sha256": "82c4876ee5204a96f0fd18c0d868680ceb321ae70f2a65d8d797cb01625c70e3", |
| "logits": [ |
| [ |
| [ |
| 0.8828125, |
| -4.40625, |
| -4.125, |
| -3.296875 |
| ], |
| [ |
| -4.53125, |
| 0.080078125 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 30917134848, |
| "reserved_bytes": 32604422144, |
| "peak_allocated_bytes": 31005399552, |
| "peak_reserved_bytes": 32604422144 |
| }, |
| "median_ms": 108.67965733632445 |
| }, |
| { |
| "case": 22, |
| "tokens": 312, |
| "media": false, |
| "hidden_sha256": "2401abb62d11d5d56ce4634631763b3c6aa6189e37fc2427129a07f2c88c4e84", |
| "logits": [ |
| [ |
| [ |
| -3.671875, |
| -1.96875, |
| 1.3828125, |
| -3.859375 |
| ], |
| [ |
| -1.2578125, |
| -3.078125 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 30917134848, |
| "reserved_bytes": 32604422144, |
| "peak_allocated_bytes": 31009006080, |
| "peak_reserved_bytes": 32604422144 |
| }, |
| "median_ms": 108.80827996879816 |
| }, |
| { |
| "case": 23, |
| "tokens": 379, |
| "media": true, |
| "hidden_sha256": "1ae8eb74ab8e4b3e777a5c62859c5334128f8d7038067b901904b886fc2a5d47", |
| "logits": [ |
| [ |
| [ |
| -3.6875, |
| -1.625, |
| 1.625, |
| -3.6875 |
| ], |
| [ |
| -2.03125, |
| -2.546875 |
| ] |
| ] |
| ], |
| "memory": { |
| "allocated_bytes": 30918712832, |
| "reserved_bytes": 32606519296, |
| "peak_allocated_bytes": 31035676672, |
| "peak_reserved_bytes": 32606519296 |
| }, |
| "median_ms": 127.35875509679317 |
| } |
| ], |
| "cold": { |
| "allocated_bytes": 30882526720, |
| "reserved_bytes": 30918311936, |
| "peak_allocated_bytes": 30882551296, |
| "peak_reserved_bytes": 30918311936, |
| "seconds": 9.994719533249736, |
| "nvml_process_peak_bytes": 31557943296 |
| }, |
| "repeat": 30, |
| "runtime_patched": false, |
| "runtime_sha256": { |
| "clef_int8_loader.py": "286752ba8076f222115a8e0cca9396cf6ca084d3fce2407d19aa2b3176fe7b5d", |
| "clef_int8_calibration.py": "135611910883019123eb1cb91dee7c2248e0e4e92c9563414c684d24bca794b0", |
| "clef_text_int8.py": "14f5ea9bfd5e21819cf40cf2176c87ceaa4f00d633cf658917256d4667fda84f", |
| "clef_int8_cutlass.py": "a516fa0c8007aa61814ea3e0ed251a3c0f0da626161bf2fe5ac30141732f857b", |
| "clef_text_cute.py": "ddc39f8bd543b7c6400e8c178ad408f0944fc7f1c881f4d36861ef497516689f", |
| "clef_text_math.py": "7b7f8c489fdb56b29c86a003d37ed96f440afe0fbe5001250036aad3e79ca760", |
| "clef_vision_norm.py": "398b681d0600dade5dc26f7664fa450ec39794f5615acac561c2032648947e6f" |
| }, |
| "bf16_loader_forbidden": true, |
| "prepacked_mlp_pairs": 0, |
| "nvml_process_peak_bytes": 33327939584, |
| "nvml_samples": 23524, |
| "batching": [ |
| { |
| "batch_size": 1, |
| "tokens_per_record": 1502, |
| "questions_per_record": 8, |
| "repeat": 10, |
| "batch_one_logits_max_abs": 0.0, |
| "batch_one_decisions_agree": true, |
| "status": "ok", |
| "latency_ms": { |
| "mean": 278.9234106428921, |
| "median": 278.92032684758306, |
| "p95": 279.09680362790823, |
| "min": 278.8346903398633, |
| "max": 279.09680362790823 |
| }, |
| "memory": { |
| "allocated_bytes": 30917158912, |
| "reserved_bytes": 32606519296, |
| "peak_allocated_bytes": 31361546240, |
| "peak_reserved_bytes": 32606519296 |
| }, |
| "decisions_per_second": 28.682025761326557, |
| "records_per_second": 3.5852532201658196 |
| }, |
| { |
| "batch_size": 2, |
| "tokens_per_record": 1502, |
| "questions_per_record": 8, |
| "repeat": 10, |
| "batch_one_logits_max_abs": 0.201171875, |
| "batch_one_decisions_agree": true, |
| "status": "ok", |
| "latency_ms": { |
| "mean": 521.7992178164423, |
| "median": 521.768159698695, |
| "p95": 522.1206778660417, |
| "min": 521.5699067339301, |
| "max": 522.1206778660417 |
| }, |
| "memory": { |
| "allocated_bytes": 30917211136, |
| "reserved_bytes": 33235664896, |
| "peak_allocated_bytes": 31805902848, |
| "peak_reserved_bytes": 33235664896 |
| }, |
| "decisions_per_second": 30.664960485974284, |
| "records_per_second": 3.8331200607467855 |
| }, |
| { |
| "batch_size": 4, |
| "tokens_per_record": 1502, |
| "questions_per_record": 8, |
| "repeat": 10, |
| "batch_one_logits_max_abs": 0.21875, |
| "batch_one_decisions_agree": true, |
| "status": "ok", |
| "latency_ms": { |
| "mean": 979.3407004326582, |
| "median": 979.3299422599375, |
| "p95": 979.7763302922249, |
| "min": 979.0457393974066, |
| "max": 979.7763302922249 |
| }, |
| "memory": { |
| "allocated_bytes": 30917282816, |
| "reserved_bytes": 35236347904, |
| "peak_allocated_bytes": 32660537856, |
| "peak_reserved_bytes": 35236347904 |
| }, |
| "decisions_per_second": 32.67540245543359, |
| "records_per_second": 4.084425306929199 |
| }, |
| { |
| "batch_size": 8, |
| "tokens_per_record": 1502, |
| "questions_per_record": 8, |
| "repeat": 10, |
| "batch_one_logits_max_abs": 0.21875, |
| "batch_one_decisions_agree": true, |
| "status": "ok", |
| "latency_ms": { |
| "mean": 1906.714035756886, |
| "median": 1906.6964145749807, |
| "p95": 1907.1267684921622, |
| "min": 1906.2150437384844, |
| "max": 1907.1267684921622 |
| }, |
| "memory": { |
| "allocated_bytes": 30917427200, |
| "reserved_bytes": 39225131008, |
| "peak_allocated_bytes": 34403283968, |
| "peak_reserved_bytes": 39225131008 |
| }, |
| "decisions_per_second": 33.56590986943569, |
| "records_per_second": 4.195738733679462 |
| }, |
| { |
| "batch_size": 16, |
| "tokens_per_record": 1502, |
| "questions_per_record": 8, |
| "repeat": 10, |
| "batch_one_logits_max_abs": 0.21875, |
| "batch_one_decisions_agree": true, |
| "status": "ok", |
| "latency_ms": { |
| "mean": 3756.9526378065348, |
| "median": 3757.018485106528, |
| "p95": 3757.3599722236395, |
| "min": 3756.4761312678456, |
| "max": 3757.3599722236395 |
| }, |
| "memory": { |
| "allocated_bytes": 30917715968, |
| "reserved_bytes": 47202697216, |
| "peak_allocated_bytes": 37889064960, |
| "peak_reserved_bytes": 47202697216 |
| }, |
| "decisions_per_second": 34.06956886355874, |
| "records_per_second": 4.258696107944843 |
| } |
| ], |
| "batching_nvml_process_peak_bytes": 47924117504, |
| "peak_allocated_bytes": 31588405760, |
| "peak_reserved_bytes": 32606519296 |
| }, |
| "calibrated_int8_cute": { |
| "mode": "direct", |
| "gpu": "NVIDIA H100 80GB HBM3", |
| "torch": "2.9.0a0+50eac811a6.nv25.09", |
| "memory_metric": "PyTorch allocated/reserved bytes; NVML process memory sampled every 5 ms", |
| "records": [ |
| { |
| "case": 0, |
| "tokens": 620, |
| "media": false, |
| "hidden_sha256": "3bd80cec0945df8323e7f97e1c446dc064516975aa15b94f5a8808715034a2a8", |
| "logits": [ |
| [ |
| [ |
| -3.140625, |
| -2.0, |
| -0.421875, |
| -3.46875 |
| ], |
| [ |
| -0.99609375, |
| -1.640625, |
| -1.21875 |
| ], |
| [ |
| 1.546875, |
| -3.21875 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "int8-torch", |
| "memory": { |
| "allocated_bytes": 30796316160, |
| "reserved_bytes": 30958157824, |
| "peak_allocated_bytes": 30905440256, |
| "peak_reserved_bytes": 30958157824 |
| }, |
| "median_ms": 82.21716340631247 |
| }, |
| { |
| "case": 1, |
| "tokens": 449, |
| "media": false, |
| "hidden_sha256": "0e416bd18959ddda56a98b01d58fb30d7a66ee16a2dd5d34a68472dccc15dfc5", |
| "logits": [ |
| [ |
| [ |
| -3.953125, |
| -0.9609375, |
| -1.6015625, |
| -3.59375 |
| ], |
| [ |
| -1.5, |
| -2.28125, |
| -1.0 |
| ], |
| [ |
| 0.84375, |
| -2.6875 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "int8-torch", |
| "memory": { |
| "allocated_bytes": 30796314112, |
| "reserved_bytes": 31008489472, |
| "peak_allocated_bytes": 30877440000, |
| "peak_reserved_bytes": 31008489472 |
| }, |
| "median_ms": 82.10805151611567 |
| }, |
| { |
| "case": 2, |
| "tokens": 622, |
| "media": false, |
| "hidden_sha256": "332b8329d6b9cf675783d97c37ac3bb9eb04953f6d1acdeda6fe3147ccb4fe16", |
| "logits": [ |
| [ |
| [ |
| -3.90625, |
| -0.87109375, |
| -0.142578125, |
| -3.828125 |
| ], |
| [ |
| -0.59375, |
| -2.90625, |
| -1.1015625 |
| ], |
| [ |
| 1.28125, |
| -3.28125 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "int8-torch", |
| "memory": { |
| "allocated_bytes": 30796316160, |
| "reserved_bytes": 31008489472, |
| "peak_allocated_bytes": 30905606144, |
| "peak_reserved_bytes": 31008489472 |
| }, |
| "median_ms": 82.51357823610306 |
| }, |
| { |
| "case": 3, |
| "tokens": 490, |
| "media": false, |
| "hidden_sha256": "26362588c871286a4878b331e1d9ec29c920510e59623fea0d6d24e28fa39857", |
| "logits": [ |
| [ |
| [ |
| -3.90625, |
| -0.9609375, |
| -0.76953125, |
| -4.34375 |
| ], |
| [ |
| -0.5546875, |
| -3.453125, |
| -2.5 |
| ], |
| [ |
| 1.15625, |
| -2.90625 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "int8-torch", |
| "memory": { |
| "allocated_bytes": 30796314112, |
| "reserved_bytes": 31056723968, |
| "peak_allocated_bytes": 30884106752, |
| "peak_reserved_bytes": 31056723968 |
| }, |
| "median_ms": 82.37049542367458 |
| }, |
| { |
| "case": 4, |
| "tokens": 448, |
| "media": false, |
| "hidden_sha256": "39bcbe99f06b8c754b1395b52aa1cf14e92ec5d38db64c7cc6b93c77200c475e", |
| "logits": [ |
| [ |
| [ |
| -4.46875, |
| 0.5703125, |
| -3.28125, |
| -4.4375 |
| ], |
| [ |
| 0.609375, |
| -4.28125, |
| -3.03125 |
| ], |
| [ |
| -2.203125, |
| -0.953125 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "int8-torch", |
| "memory": { |
| "allocated_bytes": 30796313088, |
| "reserved_bytes": 31056723968, |
| "peak_allocated_bytes": 30875826176, |
| "peak_reserved_bytes": 31056723968 |
| }, |
| "median_ms": 83.57401937246323 |
| }, |
| { |
| "case": 5, |
| "tokens": 437, |
| "media": false, |
| "hidden_sha256": "937f7824d74e9a912f87b26807db9c82bfcf1f2e49925e1db99a3daaf797e478", |
| "logits": [ |
| [ |
| [ |
| -3.796875, |
| -1.109375, |
| -2.15625, |
| -3.90625 |
| ], |
| [ |
| -0.70703125, |
| -2.84375, |
| -1.7578125 |
| ], |
| [ |
| 0.8515625, |
| -2.703125 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "int8-torch", |
| "memory": { |
| "allocated_bytes": 30796313088, |
| "reserved_bytes": 31056723968, |
| "peak_allocated_bytes": 30873589760, |
| "peak_reserved_bytes": 31056723968 |
| }, |
| "median_ms": 83.1351401284337 |
| }, |
| { |
| "case": 6, |
| "tokens": 494, |
| "media": false, |
| "hidden_sha256": "43d127c3d1d4a761ae05e8df2740a6f9c3e87c7e2505533dde9326a4c4f5b058", |
| "logits": [ |
| [ |
| [ |
| -2.578125, |
| -2.96875, |
| -0.484375, |
| -3.484375 |
| ], |
| [ |
| -3.125, |
| -1.109375, |
| -0.921875 |
| ], |
| [ |
| 1.6484375, |
| -3.234375 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "int8-torch", |
| "memory": { |
| "allocated_bytes": 30796314112, |
| "reserved_bytes": 31107055616, |
| "peak_allocated_bytes": 30884258816, |
| "peak_reserved_bytes": 31107055616 |
| }, |
| "median_ms": 83.0352883785963 |
| }, |
| { |
| "case": 7, |
| "tokens": 449, |
| "media": false, |
| "hidden_sha256": "e1d3e1006ccccf6aacba450f8841f691cf92d51a40fb3be2342832e4c7798687", |
| "logits": [ |
| [ |
| [ |
| -3.90625, |
| -1.171875, |
| -1.984375, |
| -3.90625 |
| ], |
| [ |
| 0.28125, |
| -3.953125, |
| -2.796875 |
| ], |
| [ |
| 1.375, |
| -3.03125 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "int8-torch", |
| "memory": { |
| "allocated_bytes": 30796314112, |
| "reserved_bytes": 31107055616, |
| "peak_allocated_bytes": 30877440000, |
| "peak_reserved_bytes": 31107055616 |
| }, |
| "median_ms": 82.84110855311155 |
| }, |
| { |
| "case": 8, |
| "tokens": 491, |
| "media": false, |
| "hidden_sha256": "4f5079806d430ae1d046b86d18a67af3cacdd7f62bb6ed3e7c99edecad048d31", |
| "logits": [ |
| [ |
| [ |
| -4.1875, |
| -2.46875, |
| -0.26953125, |
| -3.828125 |
| ], |
| [ |
| -1.7578125, |
| -2.90625, |
| -1.2265625 |
| ], |
| [ |
| 1.6796875, |
| -3.53125 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "int8-torch", |
| "memory": { |
| "allocated_bytes": 30796314112, |
| "reserved_bytes": 31107055616, |
| "peak_allocated_bytes": 30884146176, |
| "peak_reserved_bytes": 31107055616 |
| }, |
| "median_ms": 82.49550918117166 |
| }, |
| { |
| "case": 9, |
| "tokens": 624, |
| "media": false, |
| "hidden_sha256": "cdb5d5ec50846b16a8c87da98008b8c93e11b80570894fd3ec0ac6b28bb89cdf", |
| "logits": [ |
| [ |
| [ |
| -3.125, |
| -0.8984375, |
| -0.71875, |
| -3.765625 |
| ], |
| [ |
| -0.2109375, |
| -2.640625, |
| -1.5546875 |
| ], |
| [ |
| 1.3828125, |
| -3.359375 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "int8-torch", |
| "memory": { |
| "allocated_bytes": 30796316160, |
| "reserved_bytes": 31107055616, |
| "peak_allocated_bytes": 30905769984, |
| "peak_reserved_bytes": 31107055616 |
| }, |
| "median_ms": 83.52960739284754 |
| }, |
| { |
| "case": 10, |
| "tokens": 454, |
| "media": false, |
| "hidden_sha256": "bc2bc8c35a51b4b082c379c097dac42759aedf388fbfbcf85f4b7a7971d0fb8b", |
| "logits": [ |
| [ |
| [ |
| -4.34375, |
| -2.59375, |
| -0.22265625, |
| -3.9375 |
| ], |
| [ |
| -2.09375, |
| -2.59375, |
| -1.1328125 |
| ], |
| [ |
| 1.4375, |
| -3.265625 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "int8-torch", |
| "memory": { |
| "allocated_bytes": 30796314112, |
| "reserved_bytes": 31107055616, |
| "peak_allocated_bytes": 30877618688, |
| "peak_reserved_bytes": 31107055616 |
| }, |
| "median_ms": 83.26610410586 |
| }, |
| { |
| "case": 11, |
| "tokens": 442, |
| "media": false, |
| "hidden_sha256": "caccd4f2f27ee39e8903a5e20fd24916ad6d670e66e99b8184da23725a33326b", |
| "logits": [ |
| [ |
| [ |
| -4.0, |
| -0.34765625, |
| -1.265625, |
| -4.3125 |
| ], |
| [ |
| 0.248046875, |
| -4.1875, |
| -3.3125 |
| ], |
| [ |
| -0.2578125, |
| -1.34375 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "int8-torch", |
| "memory": { |
| "allocated_bytes": 30796313088, |
| "reserved_bytes": 31107055616, |
| "peak_allocated_bytes": 30873892352, |
| "peak_reserved_bytes": 31107055616 |
| }, |
| "median_ms": 82.95116154477 |
| }, |
| { |
| "case": 12, |
| "tokens": 458, |
| "media": false, |
| "hidden_sha256": "412e18ef08c9375f461110ff5dd36114c13e517c6223db5df85bc20c9a045e3e", |
| "logits": [ |
| [ |
| [ |
| -2.125, |
| -3.796875, |
| -0.328125, |
| -2.671875 |
| ], |
| [ |
| -3.953125, |
| 0.001953125, |
| -1.71875 |
| ], |
| [ |
| 1.421875, |
| -2.65625 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "int8-torch", |
| "memory": { |
| "allocated_bytes": 30796314112, |
| "reserved_bytes": 31107055616, |
| "peak_allocated_bytes": 30878805504, |
| "peak_reserved_bytes": 31107055616 |
| }, |
| "median_ms": 83.5030721500516 |
| }, |
| { |
| "case": 13, |
| "tokens": 620, |
| "media": false, |
| "hidden_sha256": "23a647bce8af2f2d6077fde8985dce1ebe78babbb63ed404ad555f0354b39555", |
| "logits": [ |
| [ |
| [ |
| -3.109375, |
| -1.8828125, |
| -0.87890625, |
| -3.953125 |
| ], |
| [ |
| -0.796875, |
| -2.28125, |
| -1.3515625 |
| ], |
| [ |
| 1.8203125, |
| -3.3125 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "int8-torch", |
| "memory": { |
| "allocated_bytes": 30796316160, |
| "reserved_bytes": 31107055616, |
| "peak_allocated_bytes": 30905440256, |
| "peak_reserved_bytes": 31107055616 |
| }, |
| "median_ms": 82.6035849750042 |
| }, |
| { |
| "case": 14, |
| "tokens": 616, |
| "media": false, |
| "hidden_sha256": "6f9506754fcfb7e5766f7db8fdb8c73597304aa41ff38e67c41f033e26aca023", |
| "logits": [ |
| [ |
| [ |
| -3.328125, |
| -0.921875, |
| -1.140625, |
| -3.921875 |
| ], |
| [ |
| -0.26953125, |
| -2.984375, |
| -2.0625 |
| ], |
| [ |
| 1.21875, |
| -3.5625 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "int8-torch", |
| "memory": { |
| "allocated_bytes": 30796316160, |
| "reserved_bytes": 31107055616, |
| "peak_allocated_bytes": 30905106944, |
| "peak_reserved_bytes": 31107055616 |
| }, |
| "median_ms": 83.08295579627156 |
| }, |
| { |
| "case": 15, |
| "tokens": 493, |
| "media": false, |
| "hidden_sha256": "920ca7b25e96d6dbb6bbd2585ff5931c71347c9e0a5d739298e6c9035e40e56b", |
| "logits": [ |
| [ |
| [ |
| -2.46875, |
| -2.09375, |
| 0.0078125, |
| -3.75 |
| ], |
| [ |
| -1.5546875, |
| -1.671875, |
| -1.359375 |
| ], |
| [ |
| 1.3046875, |
| -3.09375 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "int8-torch", |
| "memory": { |
| "allocated_bytes": 30796314112, |
| "reserved_bytes": 31107055616, |
| "peak_allocated_bytes": 30884220928, |
| "peak_reserved_bytes": 31107055616 |
| }, |
| "median_ms": 83.13107490539551 |
| }, |
| { |
| "case": 16, |
| "tokens": 1278, |
| "media": false, |
| "hidden_sha256": "88bdd67d73f1462d1740dea4ebec6fdfa00e29dd76d60fe2bb3f585a0ffea188", |
| "logits": [ |
| [ |
| [ |
| 1.265625, |
| 0.40625, |
| 0.6953125, |
| 0.828125 |
| ], |
| [ |
| 0.859375, |
| 0.056640625, |
| 0.24609375, |
| 0.52734375 |
| ], |
| [ |
| 0.55859375, |
| -0.091796875, |
| 0.21875, |
| 0.314453125 |
| ], |
| [ |
| 0.51171875, |
| -0.0703125, |
| -0.0234375, |
| 0.2265625 |
| ], |
| [ |
| 0.5703125, |
| -0.150390625, |
| -0.03515625, |
| 0.0859375 |
| ], |
| [ |
| 0.4296875, |
| -0.25, |
| -0.177734375, |
| -0.015625 |
| ], |
| [ |
| 0.140625, |
| -0.197265625, |
| -0.203125, |
| 0.0390625 |
| ], |
| [ |
| 0.42578125, |
| -0.07421875, |
| -0.0234375, |
| 0.14453125 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "int8-torch", |
| "memory": { |
| "allocated_bytes": 30796326400, |
| "reserved_bytes": 31107055616, |
| "peak_allocated_bytes": 31017659904, |
| "peak_reserved_bytes": 31107055616 |
| }, |
| "median_ms": 105.7324018329382 |
| }, |
| { |
| "case": 17, |
| "tokens": 1502, |
| "media": false, |
| "hidden_sha256": "620de62d3f3cd75f195a80ef7ce174225523fc4ea127254050b400c6b04708aa", |
| "logits": [ |
| [ |
| [ |
| 1.40625, |
| 0.5390625, |
| 0.69921875, |
| 0.7578125 |
| ], |
| [ |
| 0.7734375, |
| 0.1328125, |
| 0.25390625, |
| 0.42578125 |
| ], |
| [ |
| 0.8359375, |
| 0.1171875, |
| 0.51171875, |
| 0.453125 |
| ], |
| [ |
| 0.9453125, |
| 0.17578125, |
| 0.3515625, |
| 0.29296875 |
| ], |
| [ |
| 0.9375, |
| 0.09375, |
| 0.24609375, |
| 0.296875 |
| ], |
| [ |
| 0.7109375, |
| -0.1171875, |
| 0.078125, |
| 0.2265625 |
| ], |
| [ |
| 0.65234375, |
| 0.126953125, |
| 0.0546875, |
| 0.376953125 |
| ], |
| [ |
| 0.875, |
| 0.04296875, |
| 0.21875, |
| 0.26171875 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "int8-torch", |
| "memory": { |
| "allocated_bytes": 30796330496, |
| "reserved_bytes": 31178358784, |
| "peak_allocated_bytes": 31057437184, |
| "peak_reserved_bytes": 31178358784 |
| }, |
| "median_ms": 119.79983653873205 |
| }, |
| { |
| "case": 18, |
| "tokens": 2270, |
| "media": false, |
| "hidden_sha256": "d6c682b59804bcd7b9bfd96f66c20a57268ce65ca24179a5da266433e0173cc2", |
| "logits": [ |
| [ |
| [ |
| 0.6953125, |
| -0.03515625, |
| 0.03515625, |
| 0.328125 |
| ], |
| [ |
| 0.3515625, |
| -0.236328125, |
| -0.10546875, |
| 0.1484375 |
| ], |
| [ |
| 0.22265625, |
| -0.34375, |
| -0.0546875, |
| 0.0625 |
| ], |
| [ |
| 0.421875, |
| -0.0390625, |
| 0.12890625, |
| 0.3671875 |
| ], |
| [ |
| 0.2734375, |
| -0.337890625, |
| -0.1015625, |
| 0.09375 |
| ], |
| [ |
| 0.078125, |
| -0.5234375, |
| -0.42578125, |
| 0.05078125 |
| ], |
| [ |
| -0.11328125, |
| -0.5078125, |
| -0.283203125, |
| 0.00390625 |
| ], |
| [ |
| -0.0234375, |
| -0.75390625, |
| -0.24609375, |
| -0.15234375 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "int8-torch", |
| "memory": { |
| "allocated_bytes": 30796342784, |
| "reserved_bytes": 31480348672, |
| "peak_allocated_bytes": 31190387200, |
| "peak_reserved_bytes": 31480348672 |
| }, |
| "median_ms": 170.62044935300946 |
| }, |
| { |
| "case": 19, |
| "tokens": 298, |
| "media": false, |
| "hidden_sha256": "93a4b2a5b5ccc85db7ef415d90158e97efbc5cecfd28e41ee35cc11c2ac31c3a", |
| "logits": [ |
| [ |
| [ |
| -3.375, |
| -4.5, |
| -3.4375, |
| 0.984375 |
| ], |
| [ |
| -3.859375, |
| 0.1796875 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "int8-torch", |
| "memory": { |
| "allocated_bytes": 30796311040, |
| "reserved_bytes": 31480348672, |
| "peak_allocated_bytes": 30850066432, |
| "peak_reserved_bytes": 31480348672 |
| }, |
| "median_ms": 83.5722703486681 |
| }, |
| { |
| "case": 20, |
| "tokens": 303, |
| "media": false, |
| "hidden_sha256": "c15885b277be05a2bc708b698408a398d7f631d0cb97640bd0fdbd9fa5efca32", |
| "logits": [ |
| [ |
| [ |
| -4.8125, |
| -4.71875, |
| 0.15625, |
| -4.34375 |
| ], |
| [ |
| -0.328125, |
| -3.234375 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "int8-torch", |
| "memory": { |
| "allocated_bytes": 30796311040, |
| "reserved_bytes": 31480348672, |
| "peak_allocated_bytes": 30850537984, |
| "peak_reserved_bytes": 31480348672 |
| }, |
| "median_ms": 82.87730859592557 |
| }, |
| { |
| "case": 21, |
| "tokens": 304, |
| "media": false, |
| "hidden_sha256": "82c4876ee5204a96f0fd18c0d868680ceb321ae70f2a65d8d797cb01625c70e3", |
| "logits": [ |
| [ |
| [ |
| 0.8828125, |
| -4.40625, |
| -4.125, |
| -3.296875 |
| ], |
| [ |
| -4.53125, |
| 0.080078125 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "int8-torch", |
| "memory": { |
| "allocated_bytes": 30796311040, |
| "reserved_bytes": 31480348672, |
| "peak_allocated_bytes": 30850636288, |
| "peak_reserved_bytes": 31480348672 |
| }, |
| "median_ms": 83.31082761287689 |
| }, |
| { |
| "case": 22, |
| "tokens": 312, |
| "media": false, |
| "hidden_sha256": "2401abb62d11d5d56ce4634631763b3c6aa6189e37fc2427129a07f2c88c4e84", |
| "logits": [ |
| [ |
| [ |
| -3.671875, |
| -1.96875, |
| 1.3828125, |
| -3.859375 |
| ], |
| [ |
| -1.2578125, |
| -3.078125 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "int8-torch", |
| "memory": { |
| "allocated_bytes": 30796311040, |
| "reserved_bytes": 31480348672, |
| "peak_allocated_bytes": 30852791808, |
| "peak_reserved_bytes": 31480348672 |
| }, |
| "median_ms": 82.5475174933672 |
| }, |
| { |
| "case": 23, |
| "tokens": 379, |
| "media": true, |
| "hidden_sha256": "1ae8eb74ab8e4b3e777a5c62859c5334128f8d7038067b901904b886fc2a5d47", |
| "logits": [ |
| [ |
| [ |
| -3.6875, |
| -1.625, |
| 1.625, |
| -3.6875 |
| ], |
| [ |
| -2.03125, |
| -2.546875 |
| ] |
| ] |
| ], |
| "overlay_exact": true, |
| "reference_exact": true, |
| "reference_mode": "int8-torch", |
| "memory": { |
| "allocated_bytes": 30797889024, |
| "reserved_bytes": 31480348672, |
| "peak_allocated_bytes": 30868823552, |
| "peak_reserved_bytes": 31480348672 |
| }, |
| "median_ms": 92.6055503077805 |
| } |
| ], |
| "cold": { |
| "allocated_bytes": 30761702912, |
| "reserved_bytes": 30798774272, |
| "peak_allocated_bytes": 30761728000, |
| "peak_reserved_bytes": 30798774272, |
| "seconds": 11.18691000342369, |
| "nvml_process_peak_bytes": 31440502784 |
| }, |
| "repeat": 30, |
| "runtime_patched": true, |
| "runtime_sha256": { |
| "clef_int8_loader.py": "286752ba8076f222115a8e0cca9396cf6ca084d3fce2407d19aa2b3176fe7b5d", |
| "clef_int8_calibration.py": "135611910883019123eb1cb91dee7c2248e0e4e92c9563414c684d24bca794b0", |
| "clef_text_int8.py": "14f5ea9bfd5e21819cf40cf2176c87ceaa4f00d633cf658917256d4667fda84f", |
| "clef_int8_cutlass.py": "a516fa0c8007aa61814ea3e0ed251a3c0f0da626161bf2fe5ac30141732f857b", |
| "clef_text_cute.py": "ddc39f8bd543b7c6400e8c178ad408f0944fc7f1c881f4d36861ef497516689f", |
| "clef_text_math.py": "7b7f8c489fdb56b29c86a003d37ed96f440afe0fbe5001250036aad3e79ca760", |
| "clef_vision_norm.py": "398b681d0600dade5dc26f7664fa450ec39794f5615acac561c2032648947e6f" |
| }, |
| "bf16_loader_forbidden": true, |
| "prepacked_mlp_pairs": 64, |
| "nvml_process_peak_bytes": 32203866112, |
| "nvml_samples": 16696, |
| "batching": [ |
| { |
| "batch_size": 1, |
| "tokens_per_record": 1502, |
| "questions_per_record": 8, |
| "repeat": 10, |
| "batch_one_logits_max_abs": 0.0, |
| "batch_one_decisions_agree": true, |
| "status": "ok", |
| "latency_ms": { |
| "mean": 119.60493177175522, |
| "median": 119.60197286680341, |
| "p95": 119.72382944077253, |
| "min": 119.49383094906807, |
| "max": 119.72382944077253 |
| }, |
| "memory": { |
| "allocated_bytes": 30796335104, |
| "reserved_bytes": 31480348672, |
| "peak_allocated_bytes": 31057441792, |
| "peak_reserved_bytes": 31480348672 |
| }, |
| "decisions_per_second": 66.88852874449925, |
| "records_per_second": 8.361066093062407 |
| }, |
| { |
| "batch_size": 2, |
| "tokens_per_record": 1502, |
| "questions_per_record": 8, |
| "repeat": 10, |
| "batch_one_logits_max_abs": 0.201171875, |
| "batch_one_decisions_agree": true, |
| "status": "ok", |
| "latency_ms": { |
| "mean": 223.49175326526165, |
| "median": 223.23126904666424, |
| "p95": 224.78003799915314, |
| "min": 222.98206482082605, |
| "max": 224.78003799915314 |
| }, |
| "memory": { |
| "allocated_bytes": 30796387328, |
| "reserved_bytes": 31723618304, |
| "peak_allocated_bytes": 31319557120, |
| "peak_reserved_bytes": 31723618304 |
| }, |
| "decisions_per_second": 71.6745466185356, |
| "records_per_second": 8.95931832731695 |
| }, |
| { |
| "batch_size": 4, |
| "tokens_per_record": 1502, |
| "questions_per_record": 8, |
| "repeat": 10, |
| "batch_one_logits_max_abs": 0.21875, |
| "batch_one_decisions_agree": true, |
| "status": "ok", |
| "latency_ms": { |
| "mean": 426.28292785957456, |
| "median": 426.41479056328535, |
| "p95": 426.95363610982895, |
| "min": 425.36329943686724, |
| "max": 426.95363610982895 |
| }, |
| "memory": { |
| "allocated_bytes": 30796459008, |
| "reserved_bytes": 32633782272, |
| "peak_allocated_bytes": 31841909248, |
| "peak_reserved_bytes": 32633782272 |
| }, |
| "decisions_per_second": 75.04430124885829, |
| "records_per_second": 9.380537656107286 |
| }, |
| { |
| "batch_size": 8, |
| "tokens_per_record": 1502, |
| "questions_per_record": 8, |
| "repeat": 10, |
| "batch_one_logits_max_abs": 0.21875, |
| "batch_one_decisions_agree": true, |
| "status": "ok", |
| "latency_ms": { |
| "mean": 835.4205219075084, |
| "median": 835.1544700562954, |
| "p95": 838.116173632443, |
| "min": 833.198837004602, |
| "max": 838.116173632443 |
| }, |
| "memory": { |
| "allocated_bytes": 30796603392, |
| "reserved_bytes": 34296823808, |
| "peak_allocated_bytes": 32886256640, |
| "peak_reserved_bytes": 34296823808 |
| }, |
| "decisions_per_second": 76.63253002248308, |
| "records_per_second": 9.579066252810385 |
| }, |
| { |
| "batch_size": 16, |
| "tokens_per_record": 1502, |
| "questions_per_record": 8, |
| "repeat": 10, |
| "batch_one_logits_max_abs": 0.21875, |
| "batch_one_decisions_agree": true, |
| "status": "ok", |
| "latency_ms": { |
| "mean": 1663.1453910842538, |
| "median": 1663.0147453397512, |
| "p95": 1668.0818516761065, |
| "min": 1659.6905048936605, |
| "max": 1668.0818516761065 |
| }, |
| "memory": { |
| "allocated_bytes": 30796892160, |
| "reserved_bytes": 37610323968, |
| "peak_allocated_bytes": 34972632064, |
| "peak_reserved_bytes": 37610323968 |
| }, |
| "decisions_per_second": 76.9686500728229, |
| "records_per_second": 9.621081259102862 |
| } |
| ], |
| "batching_nvml_process_peak_bytes": 38335938560, |
| "peak_allocated_bytes": 31190387200, |
| "peak_reserved_bytes": 31480348672 |
| } |
| } |
|
|
