{
  "vanilla_bf16": {
    "mode": "bf16",
    "gpu": "NVIDIA H100 80GB HBM3",
    "torch": "2.9.0a0+50eac811a6.nv25.09",
    "memory_metric": "PyTorch allocated/reserved bytes; NVML process memory sampled every 5 ms",
    "records": [
      {
        "case": 0,
        "tokens": 620,
        "media": false,
        "hidden_sha256": "a597b8195ef8a3bcade2f86d3768688e097b66a771b3377af2ea43cc00f6969a",
        "logits": [
          [
            [
              -3.125,
              -2.03125,
              -0.359375,
              -3.4375
            ],
            [
              -1.015625,
              -1.6171875,
              -1.171875
            ],
            [
              1.640625,
              -3.15625
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 55004298752,
          "reserved_bytes": 55236886528,
          "peak_allocated_bytes": 55132857344,
          "peak_reserved_bytes": 55236886528
        },
        "median_ms": 78.40589061379433
      },
      {
        "case": 1,
        "tokens": 449,
        "media": false,
        "hidden_sha256": "85fb727a649afba0f3b8c9c81825bcb8f2ccaceaa84aa5b851d475029b719ba3",
        "logits": [
          [
            [
              -4.0,
              -1.03125,
              -1.5859375,
              -3.609375
            ],
            [
              -1.4375,
              -2.359375,
              -1.03125
            ],
            [
              0.828125,
              -2.75
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 55004296704,
          "reserved_bytes": 55287218176,
          "peak_allocated_bytes": 55100137472,
          "peak_reserved_bytes": 55287218176
        },
        "median_ms": 77.41388725116849
      },
      {
        "case": 2,
        "tokens": 622,
        "media": false,
        "hidden_sha256": "1f1348abf8e3ba0c49203487244b4b3b44da05204fca2e52f23c6bbc899088c1",
        "logits": [
          [
            [
              -3.921875,
              -0.8828125,
              -0.140625,
              -3.78125
            ],
            [
              -0.57421875,
              -2.90625,
              -1.09375
            ],
            [
              1.28125,
              -3.3125
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 55004298752,
          "reserved_bytes": 55287218176,
          "peak_allocated_bytes": 55133167616,
          "peak_reserved_bytes": 55287218176
        },
        "median_ms": 78.70555715635419
      },
      {
        "case": 3,
        "tokens": 490,
        "media": false,
        "hidden_sha256": "5091a522f40ff5dc7d795b6f2d01735838286a07f12cf7a0fb696f0a5e4cd292",
        "logits": [
          [
            [
              -3.9375,
              -0.96875,
              -0.75,
              -4.375
            ],
            [
              -0.5625,
              -3.5,
              -2.53125
            ],
            [
              1.2109375,
              -2.875
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 55004296704,
          "reserved_bytes": 55335452672,
          "peak_allocated_bytes": 55107471872,
          "peak_reserved_bytes": 55335452672
        },
        "median_ms": 78.5877387970686
      },
      {
        "case": 4,
        "tokens": 448,
        "media": false,
        "hidden_sha256": "860c8cff7f484291cee00b1b6a10c759a4e636d2b39b022935d3e450e4360c27",
        "logits": [
          [
            [
              -4.46875,
              0.57421875,
              -3.265625,
              -4.5
            ],
            [
              0.59375,
              -4.3125,
              -3.015625
            ],
            [
              -2.109375,
              -0.94921875
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 55004295680,
          "reserved_bytes": 55335452672,
          "peak_allocated_bytes": 55098472448,
          "peak_reserved_bytes": 55335452672
        },
        "median_ms": 78.52127030491829
      },
      {
        "case": 5,
        "tokens": 437,
        "media": false,
        "hidden_sha256": "158204930cd0fcf01c6b0287f8fc9a52ee17dc1f2f947d80d89739a56cad9601",
        "logits": [
          [
            [
              -3.765625,
              -1.0703125,
              -2.015625,
              -3.875
            ],
            [
              -0.69921875,
              -2.78125,
              -1.6484375
            ],
            [
              0.94921875,
              -2.53125
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 55004295680,
          "reserved_bytes": 55335452672,
          "peak_allocated_bytes": 55096978432,
          "peak_reserved_bytes": 55335452672
        },
        "median_ms": 78.34483683109283
      },
      {
        "case": 6,
        "tokens": 494,
        "media": false,
        "hidden_sha256": "764e10d942f237bbb0c977a5f505331000194a15a17b9ff3873598149961898a",
        "logits": [
          [
            [
              -2.59375,
              -2.96875,
              -0.3125,
              -3.5
            ],
            [
              -3.09375,
              -1.1328125,
              -0.90625
            ],
            [
              1.6484375,
              -3.25
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 55004296704,
          "reserved_bytes": 55385784320,
          "peak_allocated_bytes": 55108092928,
          "peak_reserved_bytes": 55385784320
        },
        "median_ms": 78.5312089137733
      },
      {
        "case": 7,
        "tokens": 449,
        "media": false,
        "hidden_sha256": "cd9636133adc36071ba54ab87a7cf99211ff713a88d866a966c64b53750a8637",
        "logits": [
          [
            [
              -3.9375,
              -1.140625,
              -2.09375,
              -3.890625
            ],
            [
              0.3203125,
              -4.03125,
              -2.859375
            ],
            [
              1.34375,
              -3.03125
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 55004296704,
          "reserved_bytes": 55385784320,
          "peak_allocated_bytes": 55100137472,
          "peak_reserved_bytes": 55385784320
        },
        "median_ms": 78.0846094712615
      },
      {
        "case": 8,
        "tokens": 491,
        "media": false,
        "hidden_sha256": "8e3dd892fa0fe4e7a852f5fc094a96860954f0685cd86bd0bbad037ba5805315",
        "logits": [
          [
            [
              -4.1875,
              -2.46875,
              -0.33984375,
              -3.875
            ],
            [
              -1.890625,
              -3.0,
              -1.265625
            ],
            [
              1.6328125,
              -3.59375
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 55004296704,
          "reserved_bytes": 55385784320,
          "peak_allocated_bytes": 55107629056,
          "peak_reserved_bytes": 55385784320
        },
        "median_ms": 78.83109943941236
      },
      {
        "case": 9,
        "tokens": 624,
        "media": false,
        "hidden_sha256": "1620d52e2b2963e6f7f28ee3b970ae294a01ee10d22fc4cb7fe99c46e8f2e085",
        "logits": [
          [
            [
              -3.109375,
              -0.890625,
              -0.71875,
              -3.765625
            ],
            [
              -0.1484375,
              -2.625,
              -1.53125
            ],
            [
              1.359375,
              -3.375
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 55004298752,
          "reserved_bytes": 55385784320,
          "peak_allocated_bytes": 55133474816,
          "peak_reserved_bytes": 55385784320
        },
        "median_ms": 78.95864453166723
      },
      {
        "case": 10,
        "tokens": 454,
        "media": false,
        "hidden_sha256": "18e815e952a4255661599704e290b5f37c3187b8c068cc0d535566a16f7f99f9",
        "logits": [
          [
            [
              -4.34375,
              -2.484375,
              -0.21484375,
              -3.921875
            ],
            [
              -2.0,
              -2.703125,
              -1.1640625
            ],
            [
              1.4453125,
              -3.21875
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 55004296704,
          "reserved_bytes": 55385784320,
          "peak_allocated_bytes": 55100574208,
          "peak_reserved_bytes": 55385784320
        },
        "median_ms": 77.87147304043174
      },
      {
        "case": 11,
        "tokens": 442,
        "media": false,
        "hidden_sha256": "e6d2eb173ad497fcff31054f9767fcbbe8604bccf1b9317b2686712d4db88e8e",
        "logits": [
          [
            [
              -4.0,
              -0.384765625,
              -1.125,
              -4.28125
            ],
            [
              0.2099609375,
              -4.15625,
              -3.296875
            ],
            [
              -0.2109375,
              -1.28125
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 55004295680,
          "reserved_bytes": 55385784320,
          "peak_allocated_bytes": 55097241088,
          "peak_reserved_bytes": 55385784320
        },
        "median_ms": 78.13491160050035
      },
      {
        "case": 12,
        "tokens": 458,
        "media": false,
        "hidden_sha256": "1469570ef5926d2f8a4899966bfeeea0676cdf5eaaf53364e5b9a281cb4929f6",
        "logits": [
          [
            [
              -2.125,
              -3.796875,
              -0.24609375,
              -2.671875
            ],
            [
              -3.953125,
              -0.015625,
              -1.75
            ],
            [
              1.4140625,
              -2.703125
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 55004296704,
          "reserved_bytes": 55385784320,
          "peak_allocated_bytes": 55101027840,
          "peak_reserved_bytes": 55385784320
        },
        "median_ms": 78.62287666648626
      },
      {
        "case": 13,
        "tokens": 620,
        "media": false,
        "hidden_sha256": "48880c8ff6215d9e21b7b91346a95ed21689afc1176688d73cb8f7aa141f3486",
        "logits": [
          [
            [
              -3.125,
              -1.9140625,
              -0.88671875,
              -3.984375
            ],
            [
              -0.82421875,
              -2.28125,
              -1.3359375
            ],
            [
              1.7734375,
              -3.34375
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 55004298752,
          "reserved_bytes": 55385784320,
          "peak_allocated_bytes": 55132857344,
          "peak_reserved_bytes": 55385784320
        },
        "median_ms": 79.05966322869062
      },
      {
        "case": 14,
        "tokens": 616,
        "media": false,
        "hidden_sha256": "43d102db01a1c45d1148a5217938cf40a71d5e2b8ce1e11e79f640a0f3639614",
        "logits": [
          [
            [
              -3.296875,
              -0.9140625,
              -1.1171875,
              -3.90625
            ],
            [
              -0.27734375,
              -3.046875,
              -2.046875
            ],
            [
              1.234375,
              -3.578125
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 55004298752,
          "reserved_bytes": 55385784320,
          "peak_allocated_bytes": 55132236288,
          "peak_reserved_bytes": 55385784320
        },
        "median_ms": 78.96459102630615
      },
      {
        "case": 15,
        "tokens": 493,
        "media": false,
        "hidden_sha256": "b24dcfc763451f39b010a276f8e6fd985a7c2bf9603816333f9dfc6427508ded",
        "logits": [
          [
            [
              -2.5,
              -2.109375,
              -0.00390625,
              -3.78125
            ],
            [
              -1.5,
              -1.6796875,
              -1.4375
            ],
            [
              1.28125,
              -3.125
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 55004296704,
          "reserved_bytes": 55385784320,
          "peak_allocated_bytes": 55107937280,
          "peak_reserved_bytes": 55385784320
        },
        "median_ms": 78.66512797772884
      },
      {
        "case": 16,
        "tokens": 1278,
        "media": false,
        "hidden_sha256": "63d5d9aa787a0c2c8bd467d55405d2a5074eef3e94d726f651a692506a55aaa5",
        "logits": [
          [
            [
              1.109375,
              0.29296875,
              0.59375,
              0.7421875
            ],
            [
              0.75,
              0.015625,
              0.234375,
              0.5234375
            ],
            [
              0.5234375,
              -0.134765625,
              0.138671875,
              0.296875
            ],
            [
              0.4375,
              -0.09375,
              0.0,
              0.1640625
            ],
            [
              0.451171875,
              -0.251953125,
              -0.126953125,
              0.03125
            ],
            [
              0.3203125,
              -0.3359375,
              -0.1953125,
              -0.05078125
            ],
            [
              0.1484375,
              -0.296875,
              -0.171875,
              0.0859375
            ],
            [
              0.19140625,
              -0.22265625,
              -0.11328125,
              0.32421875
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 55004308992,
          "reserved_bytes": 55385784320,
          "peak_allocated_bytes": 55267782144,
          "peak_reserved_bytes": 55385784320
        },
        "median_ms": 140.98667027428746
      },
      {
        "case": 17,
        "tokens": 1502,
        "media": false,
        "hidden_sha256": "579420a72e605ed1622ca66435162bb7e7241b143240616fb5745ae5542aed09",
        "logits": [
          [
            [
              1.1640625,
              0.35546875,
              0.4453125,
              0.46484375
            ],
            [
              0.55078125,
              -0.04296875,
              0.06640625,
              0.158203125
            ],
            [
              0.57421875,
              -0.0390625,
              0.32421875,
              0.26953125
            ],
            [
              0.69921875,
              0.12890625,
              0.2109375,
              0.16015625
            ],
            [
              0.6640625,
              0.0078125,
              0.046875,
              0.07421875
            ],
            [
              0.55859375,
              -0.35546875,
              -0.09375,
              0.12890625
            ],
            [
              0.38671875,
              -0.08984375,
              -0.2265625,
              0.1015625
            ],
            [
              0.65234375,
              -0.07421875,
              0.09765625,
              0.23046875
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 55004313088,
          "reserved_bytes": 55580819456,
          "peak_allocated_bytes": 55315161600,
          "peak_reserved_bytes": 55580819456
        },
        "median_ms": 164.4560331478715
      },
      {
        "case": 18,
        "tokens": 2270,
        "media": false,
        "hidden_sha256": "c33c7d409cb1a0cea4cf70aaf6e7f6e8cc82afc886bf5a878bac43eb4f00c16a",
        "logits": [
          [
            [
              0.84375,
              0.09375,
              0.1640625,
              0.3671875
            ],
            [
              0.462890625,
              -0.1484375,
              -0.0546875,
              0.15625
            ],
            [
              0.29296875,
              -0.328125,
              -0.01953125,
              0.2109375
            ],
            [
              0.53125,
              0.05078125,
              0.24609375,
              0.4140625
            ],
            [
              0.25,
              -0.3125,
              0.0234375,
              0.23046875
            ],
            [
              0.109375,
              -0.47265625,
              -0.3359375,
              0.05078125
            ],
            [
              0.03515625,
              -0.43359375,
              -0.22265625,
              0.16015625
            ],
            [
              0.0859375,
              -0.5390625,
              -0.23046875,
              -0.1796875
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 55004325376,
          "reserved_bytes": 56082038784,
          "peak_allocated_bytes": 55473789440,
          "peak_reserved_bytes": 56082038784
        },
        "median_ms": 241.0866366699338
      },
      {
        "case": 19,
        "tokens": 298,
        "media": false,
        "hidden_sha256": "e27ea5251b91abd33a98d4d116c88d118690188eb8f281e5a654583b1036788c",
        "logits": [
          [
            [
              -3.4375,
              -4.5,
              -3.484375,
              0.953125
            ],
            [
              -4.03125,
              0.28515625
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 55004293632,
          "reserved_bytes": 56082038784,
          "peak_allocated_bytes": 55067312128,
          "peak_reserved_bytes": 56082038784
        },
        "median_ms": 77.37664878368378
      },
      {
        "case": 20,
        "tokens": 303,
        "media": false,
        "hidden_sha256": "303763a1268b7edbcc4c2fa0daae671a88f3ad2efb332825034937e7f4e8da6e",
        "logits": [
          [
            [
              -4.8125,
              -4.75,
              0.25,
              -4.3125
            ],
            [
              -0.3359375,
              -3.234375
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 55004293632,
          "reserved_bytes": 56082038784,
          "peak_allocated_bytes": 55068987904,
          "peak_reserved_bytes": 56082038784
        },
        "median_ms": 77.46736146509647
      },
      {
        "case": 21,
        "tokens": 304,
        "media": false,
        "hidden_sha256": "7067847630f4e78eead604854a28d7132ae44209f440b5667a549d569b0f3fcb",
        "logits": [
          [
            [
              0.9140625,
              -4.4375,
              -4.15625,
              -3.296875
            ],
            [
              -4.5625,
              0.072265625
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 55004293632,
          "reserved_bytes": 56082038784,
          "peak_allocated_bytes": 55069084160,
          "peak_reserved_bytes": 56082038784
        },
        "median_ms": 77.56144786253572
      },
      {
        "case": 22,
        "tokens": 312,
        "media": false,
        "hidden_sha256": "47d4e8a47aff2f2b09ac4860a33d08cee653f449864b1379bb6237448ee43421",
        "logits": [
          [
            [
              -3.6875,
              -2.03125,
              1.375,
              -3.90625
            ],
            [
              -1.3203125,
              -3.03125
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 55004293632,
          "reserved_bytes": 56082038784,
          "peak_allocated_bytes": 55070438912,
          "peak_reserved_bytes": 56082038784
        },
        "median_ms": 77.67248200252652
      },
      {
        "case": 23,
        "tokens": 379,
        "media": true,
        "hidden_sha256": "5031e15c6e427ef014bdae66dc6f10e88053e7d826058769639790edb198ba2b",
        "logits": [
          [
            [
              -3.671875,
              -1.6171875,
              1.6015625,
              -3.65625
            ],
            [
              -1.9921875,
              -2.5
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 55005871616,
          "reserved_bytes": 56084135936,
          "peak_allocated_bytes": 55087966720,
          "peak_reserved_bytes": 56084135936
        },
        "median_ms": 89.13406962528825
      }
    ],
    "cold": {
      "allocated_bytes": 54969685504,
      "reserved_bytes": 54987325440,
      "peak_allocated_bytes": 54969685504,
      "peak_reserved_bytes": 54987325440,
      "seconds": 13.1426798440516,
      "nvml_process_peak_bytes": 55626956800
    },
    "repeat": 30,
    "runtime_patched": false,
    "runtime_sha256": {
      "clef_int8_loader.py": "286752ba8076f222115a8e0cca9396cf6ca084d3fce2407d19aa2b3176fe7b5d",
      "clef_int8_calibration.py": "135611910883019123eb1cb91dee7c2248e0e4e92c9563414c684d24bca794b0",
      "clef_text_int8.py": "14f5ea9bfd5e21819cf40cf2176c87ceaa4f00d633cf658917256d4667fda84f",
      "clef_int8_cutlass.py": "a516fa0c8007aa61814ea3e0ed251a3c0f0da626161bf2fe5ac30141732f857b",
      "clef_text_cute.py": "ddc39f8bd543b7c6400e8c178ad408f0944fc7f1c881f4d36861ef497516689f",
      "clef_text_math.py": "7b7f8c489fdb56b29c86a003d37ed96f440afe0fbe5001250036aad3e79ca760",
      "clef_vision_norm.py": "398b681d0600dade5dc26f7664fa450ec39794f5615acac561c2032648947e6f"
    },
    "nvml_process_peak_bytes": 56805556224,
    "nvml_samples": 16022,
    "batching": [
      {
        "batch_size": 1,
        "tokens_per_record": 1502,
        "questions_per_record": 8,
        "repeat": 10,
        "batch_one_logits_max_abs": 0.0,
        "batch_one_decisions_agree": true,
        "status": "ok",
        "latency_ms": {
          "mean": 165.03022303804755,
          "median": 164.39128015190363,
          "p95": 172.891135327518,
          "min": 161.52335330843925,
          "max": 172.891135327518
        },
        "memory": {
          "allocated_bytes": 55004317696,
          "reserved_bytes": 56084135936,
          "peak_allocated_bytes": 55315166208,
          "peak_reserved_bytes": 56084135936
        },
        "decisions_per_second": 48.664381666763006,
        "records_per_second": 6.083047708345376
      },
      {
        "batch_size": 2,
        "tokens_per_record": 1502,
        "questions_per_record": 8,
        "repeat": 10,
        "batch_one_logits_max_abs": 0.30078125,
        "batch_one_decisions_agree": true,
        "status": "ok",
        "latency_ms": {
          "mean": 314.9787465110421,
          "median": 314.8134066723287,
          "p95": 321.4167505502701,
          "min": 308.3176026120782,
          "max": 321.4167505502701
        },
        "memory": {
          "allocated_bytes": 55004369920,
          "reserved_bytes": 56621006848,
          "peak_allocated_bytes": 55625633792,
          "peak_reserved_bytes": 56621006848
        },
        "decisions_per_second": 50.8237567425249,
        "records_per_second": 6.3529695928156125
      },
      {
        "batch_size": 4,
        "tokens_per_record": 1502,
        "questions_per_record": 8,
        "repeat": 10,
        "batch_one_logits_max_abs": 0.46875,
        "batch_one_decisions_agree": true,
        "status": "ok",
        "latency_ms": {
          "mean": 608.3244372159243,
          "median": 607.7957293018699,
          "p95": 611.246588639915,
          "min": 605.1242211833596,
          "max": 611.246588639915
        },
        "memory": {
          "allocated_bytes": 55004441600,
          "reserved_bytes": 57308872704,
          "peak_allocated_bytes": 56246851584,
          "peak_reserved_bytes": 57308872704
        },
        "decisions_per_second": 52.649267603041636,
        "records_per_second": 6.5811584503802045
      },
      {
        "batch_size": 8,
        "tokens_per_record": 1502,
        "questions_per_record": 8,
        "repeat": 10,
        "batch_one_logits_max_abs": 0.46875,
        "batch_one_decisions_agree": true,
        "status": "ok",
        "latency_ms": {
          "mean": 1205.6570419110358,
          "median": 1204.5913855545223,
          "p95": 1209.4773035496473,
          "min": 1202.9799250885844,
          "max": 1209.4773035496473
        },
        "memory": {
          "allocated_bytes": 55004585984,
          "reserved_bytes": 60299411456,
          "peak_allocated_bytes": 57488294912,
          "peak_reserved_bytes": 60299411456
        },
        "decisions_per_second": 53.1300495483273,
        "records_per_second": 6.6412561935409125
      },
      {
        "batch_size": 16,
        "tokens_per_record": 1502,
        "questions_per_record": 8,
        "repeat": 10,
        "batch_one_logits_max_abs": 0.46875,
        "batch_one_decisions_agree": true,
        "status": "ok",
        "latency_ms": {
          "mean": 2402.7484238147736,
          "median": 2402.016118168831,
          "p95": 2409.259708598256,
          "min": 2395.667601376772,
          "max": 2409.259708598256
        },
        "memory": {
          "allocated_bytes": 55004874752,
          "reserved_bytes": 65877835776,
          "peak_allocated_bytes": 59975277568,
          "peak_reserved_bytes": 65877835776
        },
        "decisions_per_second": 53.28856831218118,
        "records_per_second": 6.661071039022647
      }
    ],
    "batching_nvml_process_peak_bytes": 66603450368,
    "peak_allocated_bytes": 55473789440,
    "peak_reserved_bytes": 56084135936
  },
  "bf16_cute": {
    "mode": "bf16-cute",
    "gpu": "NVIDIA H100 80GB HBM3",
    "torch": "2.9.0a0+50eac811a6.nv25.09",
    "memory_metric": "PyTorch allocated/reserved bytes; NVML process memory sampled every 5 ms",
    "records": [
      {
        "case": 0,
        "tokens": 620,
        "media": false,
        "hidden_sha256": "a597b8195ef8a3bcade2f86d3768688e097b66a771b3377af2ea43cc00f6969a",
        "logits": [
          [
            [
              -3.125,
              -2.03125,
              -0.359375,
              -3.4375
            ],
            [
              -1.015625,
              -1.6171875,
              -1.171875
            ],
            [
              1.640625,
              -3.15625
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "bf16",
        "memory": {
          "allocated_bytes": 55004323328,
          "reserved_bytes": 55192846336,
          "peak_allocated_bytes": 55113447424,
          "peak_reserved_bytes": 55192846336
        },
        "median_ms": 69.01239790022373
      },
      {
        "case": 1,
        "tokens": 449,
        "media": false,
        "hidden_sha256": "85fb727a649afba0f3b8c9c81825bcb8f2ccaceaa84aa5b851d475029b719ba3",
        "logits": [
          [
            [
              -4.0,
              -1.03125,
              -1.5859375,
              -3.609375
            ],
            [
              -1.4375,
              -2.359375,
              -1.03125
            ],
            [
              0.828125,
              -2.75
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "bf16",
        "memory": {
          "allocated_bytes": 55004321280,
          "reserved_bytes": 55243177984,
          "peak_allocated_bytes": 55086108672,
          "peak_reserved_bytes": 55243177984
        },
        "median_ms": 67.6680295728147
      },
      {
        "case": 2,
        "tokens": 622,
        "media": false,
        "hidden_sha256": "1f1348abf8e3ba0c49203487244b4b3b44da05204fca2e52f23c6bbc899088c1",
        "logits": [
          [
            [
              -3.921875,
              -0.8828125,
              -0.140625,
              -3.78125
            ],
            [
              -0.57421875,
              -2.90625,
              -1.09375
            ],
            [
              1.28125,
              -3.3125
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "bf16",
        "memory": {
          "allocated_bytes": 55004323328,
          "reserved_bytes": 55243177984,
          "peak_allocated_bytes": 55113613312,
          "peak_reserved_bytes": 55243177984
        },
        "median_ms": 69.76490188390017
      },
      {
        "case": 3,
        "tokens": 490,
        "media": false,
        "hidden_sha256": "5091a522f40ff5dc7d795b6f2d01735838286a07f12cf7a0fb696f0a5e4cd292",
        "logits": [
          [
            [
              -3.9375,
              -0.96875,
              -0.75,
              -4.375
            ],
            [
              -0.5625,
              -3.5,
              -2.53125
            ],
            [
              1.2109375,
              -2.875
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "bf16",
        "memory": {
          "allocated_bytes": 55004321280,
          "reserved_bytes": 55291412480,
          "peak_allocated_bytes": 55091483136,
          "peak_reserved_bytes": 55291412480
        },
        "median_ms": 68.77257069572806
      },
      {
        "case": 4,
        "tokens": 448,
        "media": false,
        "hidden_sha256": "860c8cff7f484291cee00b1b6a10c759a4e636d2b39b022935d3e450e4360c27",
        "logits": [
          [
            [
              -4.46875,
              0.57421875,
              -3.265625,
              -4.5
            ],
            [
              0.59375,
              -4.3125,
              -3.015625
            ],
            [
              -2.109375,
              -0.94921875
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "bf16",
        "memory": {
          "allocated_bytes": 55004320256,
          "reserved_bytes": 55291412480,
          "peak_allocated_bytes": 55084488704,
          "peak_reserved_bytes": 55291412480
        },
        "median_ms": 68.35763994604349
      },
      {
        "case": 5,
        "tokens": 437,
        "media": false,
        "hidden_sha256": "158204930cd0fcf01c6b0287f8fc9a52ee17dc1f2f947d80d89739a56cad9601",
        "logits": [
          [
            [
              -3.765625,
              -1.0703125,
              -2.015625,
              -3.875
            ],
            [
              -0.69921875,
              -2.78125,
              -1.6484375
            ],
            [
              0.94921875,
              -2.53125
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "bf16",
        "memory": {
          "allocated_bytes": 55004320256,
          "reserved_bytes": 55291412480,
          "peak_allocated_bytes": 55081342976,
          "peak_reserved_bytes": 55291412480
        },
        "median_ms": 68.14693473279476
      },
      {
        "case": 6,
        "tokens": 494,
        "media": false,
        "hidden_sha256": "764e10d942f237bbb0c977a5f505331000194a15a17b9ff3873598149961898a",
        "logits": [
          [
            [
              -2.59375,
              -2.96875,
              -0.3125,
              -3.5
            ],
            [
              -3.09375,
              -1.1328125,
              -0.90625
            ],
            [
              1.6484375,
              -3.25
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "bf16",
        "memory": {
          "allocated_bytes": 55004321280,
          "reserved_bytes": 55341744128,
          "peak_allocated_bytes": 55091749888,
          "peak_reserved_bytes": 55341744128
        },
        "median_ms": 69.38431225717068
      },
      {
        "case": 7,
        "tokens": 449,
        "media": false,
        "hidden_sha256": "cd9636133adc36071ba54ab87a7cf99211ff713a88d866a966c64b53750a8637",
        "logits": [
          [
            [
              -3.9375,
              -1.140625,
              -2.09375,
              -3.890625
            ],
            [
              0.3203125,
              -4.03125,
              -2.859375
            ],
            [
              1.34375,
              -3.03125
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "bf16",
        "memory": {
          "allocated_bytes": 55004321280,
          "reserved_bytes": 55341744128,
          "peak_allocated_bytes": 55086108672,
          "peak_reserved_bytes": 55341744128
        },
        "median_ms": 68.46183445304632
      },
      {
        "case": 8,
        "tokens": 491,
        "media": false,
        "hidden_sha256": "8e3dd892fa0fe4e7a852f5fc094a96860954f0685cd86bd0bbad037ba5805315",
        "logits": [
          [
            [
              -4.1875,
              -2.46875,
              -0.33984375,
              -3.875
            ],
            [
              -1.890625,
              -3.0,
              -1.265625
            ],
            [
              1.6328125,
              -3.59375
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "bf16",
        "memory": {
          "allocated_bytes": 55004321280,
          "reserved_bytes": 55341744128,
          "peak_allocated_bytes": 55091551232,
          "peak_reserved_bytes": 55341744128
        },
        "median_ms": 69.28329588845372
      },
      {
        "case": 9,
        "tokens": 624,
        "media": false,
        "hidden_sha256": "1620d52e2b2963e6f7f28ee3b970ae294a01ee10d22fc4cb7fe99c46e8f2e085",
        "logits": [
          [
            [
              -3.109375,
              -0.890625,
              -0.71875,
              -3.765625
            ],
            [
              -0.1484375,
              -2.625,
              -1.53125
            ],
            [
              1.359375,
              -3.375
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "bf16",
        "memory": {
          "allocated_bytes": 55004323328,
          "reserved_bytes": 55341744128,
          "peak_allocated_bytes": 55113777152,
          "peak_reserved_bytes": 55341744128
        },
        "median_ms": 69.12704976275563
      },
      {
        "case": 10,
        "tokens": 454,
        "media": false,
        "hidden_sha256": "18e815e952a4255661599704e290b5f37c3187b8c068cc0d535566a16f7f99f9",
        "logits": [
          [
            [
              -4.34375,
              -2.484375,
              -0.21484375,
              -3.921875
            ],
            [
              -2.0,
              -2.703125,
              -1.1640625
            ],
            [
              1.4453125,
              -3.21875
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "bf16",
        "memory": {
          "allocated_bytes": 55004321280,
          "reserved_bytes": 55341744128,
          "peak_allocated_bytes": 55086318080,
          "peak_reserved_bytes": 55341744128
        },
        "median_ms": 68.77609435468912
      },
      {
        "case": 11,
        "tokens": 442,
        "media": false,
        "hidden_sha256": "e6d2eb173ad497fcff31054f9767fcbbe8604bccf1b9317b2686712d4db88e8e",
        "logits": [
          [
            [
              -4.0,
              -0.384765625,
              -1.125,
              -4.28125
            ],
            [
              0.2099609375,
              -4.15625,
              -3.296875
            ],
            [
              -0.2109375,
              -1.28125
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "bf16",
        "memory": {
          "allocated_bytes": 55004320256,
          "reserved_bytes": 55341744128,
          "peak_allocated_bytes": 55082518016,
          "peak_reserved_bytes": 55341744128
        },
        "median_ms": 68.66937736049294
      },
      {
        "case": 12,
        "tokens": 458,
        "media": false,
        "hidden_sha256": "1469570ef5926d2f8a4899966bfeeea0676cdf5eaaf53364e5b9a281cb4929f6",
        "logits": [
          [
            [
              -2.125,
              -3.796875,
              -0.24609375,
              -2.671875
            ],
            [
              -3.953125,
              -0.015625,
              -1.75
            ],
            [
              1.4140625,
              -2.703125
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "bf16",
        "memory": {
          "allocated_bytes": 55004321280,
          "reserved_bytes": 55341744128,
          "peak_allocated_bytes": 55087033856,
          "peak_reserved_bytes": 55341744128
        },
        "median_ms": 68.95825685933232
      },
      {
        "case": 13,
        "tokens": 620,
        "media": false,
        "hidden_sha256": "48880c8ff6215d9e21b7b91346a95ed21689afc1176688d73cb8f7aa141f3486",
        "logits": [
          [
            [
              -3.125,
              -1.9140625,
              -0.88671875,
              -3.984375
            ],
            [
              -0.82421875,
              -2.28125,
              -1.3359375
            ],
            [
              1.7734375,
              -3.34375
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "bf16",
        "memory": {
          "allocated_bytes": 55004323328,
          "reserved_bytes": 55341744128,
          "peak_allocated_bytes": 55113447424,
          "peak_reserved_bytes": 55341744128
        },
        "median_ms": 70.10784558951855
      },
      {
        "case": 14,
        "tokens": 616,
        "media": false,
        "hidden_sha256": "43d102db01a1c45d1148a5217938cf40a71d5e2b8ce1e11e79f640a0f3639614",
        "logits": [
          [
            [
              -3.296875,
              -0.9140625,
              -1.1171875,
              -3.90625
            ],
            [
              -0.27734375,
              -3.046875,
              -2.046875
            ],
            [
              1.234375,
              -3.578125
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "bf16",
        "memory": {
          "allocated_bytes": 55004323328,
          "reserved_bytes": 55341744128,
          "peak_allocated_bytes": 55113114112,
          "peak_reserved_bytes": 55341744128
        },
        "median_ms": 68.83620982989669
      },
      {
        "case": 15,
        "tokens": 493,
        "media": false,
        "hidden_sha256": "b24dcfc763451f39b010a276f8e6fd985a7c2bf9603816333f9dfc6427508ded",
        "logits": [
          [
            [
              -2.5,
              -2.109375,
              -0.00390625,
              -3.78125
            ],
            [
              -1.5,
              -1.6796875,
              -1.4375
            ],
            [
              1.28125,
              -3.125
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "bf16",
        "memory": {
          "allocated_bytes": 55004321280,
          "reserved_bytes": 55341744128,
          "peak_allocated_bytes": 55091683328,
          "peak_reserved_bytes": 55341744128
        },
        "median_ms": 68.89336928725243
      },
      {
        "case": 16,
        "tokens": 1278,
        "media": false,
        "hidden_sha256": "63d5d9aa787a0c2c8bd467d55405d2a5074eef3e94d726f651a692506a55aaa5",
        "logits": [
          [
            [
              1.109375,
              0.29296875,
              0.59375,
              0.7421875
            ],
            [
              0.75,
              0.015625,
              0.234375,
              0.5234375
            ],
            [
              0.5234375,
              -0.134765625,
              0.138671875,
              0.296875
            ],
            [
              0.4375,
              -0.09375,
              0.0,
              0.1640625
            ],
            [
              0.451171875,
              -0.251953125,
              -0.126953125,
              0.03125
            ],
            [
              0.3203125,
              -0.3359375,
              -0.1953125,
              -0.05078125
            ],
            [
              0.1484375,
              -0.296875,
              -0.171875,
              0.0859375
            ],
            [
              0.19140625,
              -0.22265625,
              -0.11328125,
              0.32421875
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "bf16",
        "memory": {
          "allocated_bytes": 55004333568,
          "reserved_bytes": 55341744128,
          "peak_allocated_bytes": 55225634304,
          "peak_reserved_bytes": 55341744128
        },
        "median_ms": 121.83641782030463
      },
      {
        "case": 17,
        "tokens": 1502,
        "media": false,
        "hidden_sha256": "579420a72e605ed1622ca66435162bb7e7241b143240616fb5745ae5542aed09",
        "logits": [
          [
            [
              1.1640625,
              0.35546875,
              0.4453125,
              0.46484375
            ],
            [
              0.55078125,
              -0.04296875,
              0.06640625,
              0.158203125
            ],
            [
              0.57421875,
              -0.0390625,
              0.32421875,
              0.26953125
            ],
            [
              0.69921875,
              0.12890625,
              0.2109375,
              0.16015625
            ],
            [
              0.6640625,
              0.0078125,
              0.046875,
              0.07421875
            ],
            [
              0.55859375,
              -0.35546875,
              -0.09375,
              0.12890625
            ],
            [
              0.38671875,
              -0.08984375,
              -0.2265625,
              0.1015625
            ],
            [
              0.65234375,
              -0.07421875,
              0.09765625,
              0.23046875
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "bf16",
        "memory": {
          "allocated_bytes": 55004337664,
          "reserved_bytes": 55499030528,
          "peak_allocated_bytes": 55265444352,
          "peak_reserved_bytes": 55499030528
        },
        "median_ms": 139.84269881621003
      },
      {
        "case": 18,
        "tokens": 2270,
        "media": false,
        "hidden_sha256": "c33c7d409cb1a0cea4cf70aaf6e7f6e8cc82afc886bf5a878bac43eb4f00c16a",
        "logits": [
          [
            [
              0.84375,
              0.09375,
              0.1640625,
              0.3671875
            ],
            [
              0.462890625,
              -0.1484375,
              -0.0546875,
              0.15625
            ],
            [
              0.29296875,
              -0.328125,
              -0.01953125,
              0.2109375
            ],
            [
              0.53125,
              0.05078125,
              0.24609375,
              0.4140625
            ],
            [
              0.25,
              -0.3125,
              0.0234375,
              0.23046875
            ],
            [
              0.109375,
              -0.47265625,
              -0.3359375,
              0.05078125
            ],
            [
              0.03515625,
              -0.43359375,
              -0.22265625,
              0.16015625
            ],
            [
              0.0859375,
              -0.5390625,
              -0.23046875,
              -0.1796875
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "bf16",
        "memory": {
          "allocated_bytes": 55004349952,
          "reserved_bytes": 55882809344,
          "peak_allocated_bytes": 55398115840,
          "peak_reserved_bytes": 55882809344
        },
        "median_ms": 208.45949463546276
      },
      {
        "case": 19,
        "tokens": 298,
        "media": false,
        "hidden_sha256": "e27ea5251b91abd33a98d4d116c88d118690188eb8f281e5a654583b1036788c",
        "logits": [
          [
            [
              -3.4375,
              -4.5,
              -3.484375,
              0.953125
            ],
            [
              -4.03125,
              0.28515625
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "bf16",
        "memory": {
          "allocated_bytes": 55004318208,
          "reserved_bytes": 55882809344,
          "peak_allocated_bytes": 55056820224,
          "peak_reserved_bytes": 55882809344
        },
        "median_ms": 67.75721022859216
      },
      {
        "case": 20,
        "tokens": 303,
        "media": false,
        "hidden_sha256": "303763a1268b7edbcc4c2fa0daae671a88f3ad2efb332825034937e7f4e8da6e",
        "logits": [
          [
            [
              -4.8125,
              -4.75,
              0.25,
              -4.3125
            ],
            [
              -0.3359375,
              -3.234375
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "bf16",
        "memory": {
          "allocated_bytes": 55004318208,
          "reserved_bytes": 55882809344,
          "peak_allocated_bytes": 55057357312,
          "peak_reserved_bytes": 55882809344
        },
        "median_ms": 68.8166175968945
      },
      {
        "case": 21,
        "tokens": 304,
        "media": false,
        "hidden_sha256": "7067847630f4e78eead604854a28d7132ae44209f440b5667a549d569b0f3fcb",
        "logits": [
          [
            [
              0.9140625,
              -4.4375,
              -4.15625,
              -3.296875
            ],
            [
              -4.5625,
              0.072265625
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "bf16",
        "memory": {
          "allocated_bytes": 55004318208,
          "reserved_bytes": 55882809344,
          "peak_allocated_bytes": 55057463808,
          "peak_reserved_bytes": 55882809344
        },
        "median_ms": 67.84480484202504
      },
      {
        "case": 22,
        "tokens": 312,
        "media": false,
        "hidden_sha256": "47d4e8a47aff2f2b09ac4860a33d08cee653f449864b1379bb6237448ee43421",
        "logits": [
          [
            [
              -3.6875,
              -2.03125,
              1.375,
              -3.90625
            ],
            [
              -1.3203125,
              -3.03125
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "bf16",
        "memory": {
          "allocated_bytes": 55004318208,
          "reserved_bytes": 55882809344,
          "peak_allocated_bytes": 55060323840,
          "peak_reserved_bytes": 55882809344
        },
        "median_ms": 67.88387056440115
      },
      {
        "case": 23,
        "tokens": 379,
        "media": true,
        "hidden_sha256": "5031e15c6e427ef014bdae66dc6f10e88053e7d826058769639790edb198ba2b",
        "logits": [
          [
            [
              -3.671875,
              -1.6171875,
              1.6015625,
              -3.65625
            ],
            [
              -1.9921875,
              -2.5
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "bf16",
        "memory": {
          "allocated_bytes": 55005896192,
          "reserved_bytes": 55884906496,
          "peak_allocated_bytes": 55074950656,
          "peak_reserved_bytes": 55884906496
        },
        "median_ms": 82.2020024061203
      }
    ],
    "cold": {
      "allocated_bytes": 54969710080,
      "reserved_bytes": 54987325440,
      "peak_allocated_bytes": 54969710592,
      "peak_reserved_bytes": 54987325440,
      "seconds": 12.89094135724008,
      "nvml_process_peak_bytes": 55626956800
    },
    "repeat": 30,
    "runtime_patched": true,
    "runtime_sha256": {
      "clef_int8_loader.py": "286752ba8076f222115a8e0cca9396cf6ca084d3fce2407d19aa2b3176fe7b5d",
      "clef_int8_calibration.py": "135611910883019123eb1cb91dee7c2248e0e4e92c9563414c684d24bca794b0",
      "clef_text_int8.py": "14f5ea9bfd5e21819cf40cf2176c87ceaa4f00d633cf658917256d4667fda84f",
      "clef_int8_cutlass.py": "a516fa0c8007aa61814ea3e0ed251a3c0f0da626161bf2fe5ac30141732f857b",
      "clef_text_cute.py": "ddc39f8bd543b7c6400e8c178ad408f0944fc7f1c881f4d36861ef497516689f",
      "clef_text_math.py": "7b7f8c489fdb56b29c86a003d37ed96f440afe0fbe5001250036aad3e79ca760",
      "clef_vision_norm.py": "398b681d0600dade5dc26f7664fa450ec39794f5615acac561c2032648947e6f"
    },
    "nvml_process_peak_bytes": 56608423936,
    "nvml_samples": 15437,
    "batching": [
      {
        "batch_size": 1,
        "tokens_per_record": 1502,
        "questions_per_record": 8,
        "repeat": 10,
        "batch_one_logits_max_abs": 0.0,
        "batch_one_decisions_agree": true,
        "status": "ok",
        "latency_ms": {
          "mean": 141.42045499756932,
          "median": 140.2929974719882,
          "p95": 150.26023890823126,
          "min": 136.18104439228773,
          "max": 150.26023890823126
        },
        "memory": {
          "allocated_bytes": 55004342272,
          "reserved_bytes": 55884906496,
          "peak_allocated_bytes": 55265448960,
          "peak_reserved_bytes": 55884906496
        },
        "decisions_per_second": 57.02351609956392,
        "records_per_second": 7.12793951244549
      },
      {
        "batch_size": 2,
        "tokens_per_record": 1502,
        "questions_per_record": 8,
        "repeat": 10,
        "batch_one_logits_max_abs": 0.30078125,
        "batch_one_decisions_agree": true,
        "status": "ok",
        "latency_ms": {
          "mean": 270.84807669743896,
          "median": 268.2279543951154,
          "p95": 284.2757198959589,
          "min": 264.37197905033827,
          "max": 284.2757198959589
        },
        "memory": {
          "allocated_bytes": 55004394496,
          "reserved_bytes": 56199479296,
          "peak_allocated_bytes": 55526315008,
          "peak_reserved_bytes": 56199479296
        },
        "decisions_per_second": 59.6507550306672,
        "records_per_second": 7.4563443788334
      },
      {
        "batch_size": 4,
        "tokens_per_record": 1502,
        "questions_per_record": 8,
        "repeat": 10,
        "batch_one_logits_max_abs": 0.46875,
        "batch_one_decisions_agree": true,
        "status": "ok",
        "latency_ms": {
          "mean": 528.4846080467105,
          "median": 529.0066166780889,
          "p95": 536.1536452546716,
          "min": 518.8928348943591,
          "max": 536.1536452546716
        },
        "memory": {
          "allocated_bytes": 55004466176,
          "reserved_bytes": 57302581248,
          "peak_allocated_bytes": 56048422400,
          "peak_reserved_bytes": 57302581248
        },
        "decisions_per_second": 60.49073677177206,
        "records_per_second": 7.561342096471508
      },
      {
        "batch_size": 8,
        "tokens_per_record": 1502,
        "questions_per_record": 8,
        "repeat": 10,
        "batch_one_logits_max_abs": 0.46875,
        "batch_one_decisions_agree": true,
        "status": "ok",
        "latency_ms": {
          "mean": 1052.5208386592567,
          "median": 1052.322378847748,
          "p95": 1054.5621123164892,
          "min": 1049.6748918667436,
          "max": 1054.5621123164892
        },
        "memory": {
          "allocated_bytes": 55004610560,
          "reserved_bytes": 59506688000,
          "peak_allocated_bytes": 57094228992,
          "peak_reserved_bytes": 59506688000
        },
        "decisions_per_second": 60.81786464531668,
        "records_per_second": 7.602233080664585
      },
      {
        "batch_size": 16,
        "tokens_per_record": 1502,
        "questions_per_record": 8,
        "repeat": 10,
        "batch_one_logits_max_abs": 0.46875,
        "batch_one_decisions_agree": true,
        "status": "ok",
        "latency_ms": {
          "mean": 2097.766940947622,
          "median": 2097.9305459186435,
          "p95": 2102.448247373104,
          "min": 2094.7574246674776,
          "max": 2102.448247373104
        },
        "memory": {
          "allocated_bytes": 55004899328,
          "reserved_bytes": 63902318592,
          "peak_allocated_bytes": 59180298240,
          "peak_reserved_bytes": 63902318592
        },
        "decisions_per_second": 61.012505990254915,
        "records_per_second": 7.626563248781864
      }
    ],
    "batching_nvml_process_peak_bytes": 64627933184,
    "peak_allocated_bytes": 55398115840,
    "peak_reserved_bytes": 55884906496
  },
  "calibrated_int8_torch": {
    "mode": "int8-torch",
    "gpu": "NVIDIA H100 80GB HBM3",
    "torch": "2.9.0a0+50eac811a6.nv25.09",
    "memory_metric": "PyTorch allocated/reserved bytes; NVML process memory sampled every 5 ms",
    "records": [
      {
        "case": 0,
        "tokens": 620,
        "media": false,
        "hidden_sha256": "3bd80cec0945df8323e7f97e1c446dc064516975aa15b94f5a8808715034a2a8",
        "logits": [
          [
            [
              -3.140625,
              -2.0,
              -0.421875,
              -3.46875
            ],
            [
              -0.99609375,
              -1.640625,
              -1.21875
            ],
            [
              1.546875,
              -3.21875
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 30917139968,
          "reserved_bytes": 31234981888,
          "peak_allocated_bytes": 31103866880,
          "peak_reserved_bytes": 31234981888
        },
        "median_ms": 137.17662682756782
      },
      {
        "case": 1,
        "tokens": 449,
        "media": false,
        "hidden_sha256": "0e416bd18959ddda56a98b01d58fb30d7a66ee16a2dd5d34a68472dccc15dfc5",
        "logits": [
          [
            [
              -3.953125,
              -0.9609375,
              -1.6015625,
              -3.59375
            ],
            [
              -1.5,
              -2.28125,
              -1.0
            ],
            [
              0.84375,
              -2.6875
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 30917137920,
          "reserved_bytes": 31285313536,
          "peak_allocated_bytes": 31051187712,
          "peak_reserved_bytes": 31285313536
        },
        "median_ms": 117.17480374500155
      },
      {
        "case": 2,
        "tokens": 622,
        "media": false,
        "hidden_sha256": "332b8329d6b9cf675783d97c37ac3bb9eb04953f6d1acdeda6fe3147ccb4fe16",
        "logits": [
          [
            [
              -3.90625,
              -0.87109375,
              -0.142578125,
              -3.828125
            ],
            [
              -0.59375,
              -2.90625,
              -1.1015625
            ],
            [
              1.28125,
              -3.28125
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 30917139968,
          "reserved_bytes": 31285313536,
          "peak_allocated_bytes": 31104019456,
          "peak_reserved_bytes": 31285313536
        },
        "median_ms": 138.2606695406139
      },
      {
        "case": 3,
        "tokens": 490,
        "media": false,
        "hidden_sha256": "26362588c871286a4878b331e1d9ec29c920510e59623fea0d6d24e28fa39857",
        "logits": [
          [
            [
              -3.90625,
              -0.9609375,
              -0.76953125,
              -4.34375
            ],
            [
              -0.5546875,
              -3.453125,
              -2.5
            ],
            [
              1.15625,
              -2.90625
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 30917137920,
          "reserved_bytes": 31333548032,
          "peak_allocated_bytes": 31063337472,
          "peak_reserved_bytes": 31333548032
        },
        "median_ms": 116.68091919273138
      },
      {
        "case": 4,
        "tokens": 448,
        "media": false,
        "hidden_sha256": "39bcbe99f06b8c754b1395b52aa1cf14e92ec5d38db64c7cc6b93c77200c475e",
        "logits": [
          [
            [
              -4.46875,
              0.5703125,
              -3.28125,
              -4.4375
            ],
            [
              0.609375,
              -4.28125,
              -3.03125
            ],
            [
              -2.203125,
              -0.953125
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 30917136896,
          "reserved_bytes": 31333548032,
          "peak_allocated_bytes": 31048071168,
          "peak_reserved_bytes": 31333548032
        },
        "median_ms": 109.10983849316835
      },
      {
        "case": 5,
        "tokens": 437,
        "media": false,
        "hidden_sha256": "937f7824d74e9a912f87b26807db9c82bfcf1f2e49925e1db99a3daaf797e478",
        "logits": [
          [
            [
              -3.796875,
              -1.109375,
              -2.15625,
              -3.90625
            ],
            [
              -0.70703125,
              -2.84375,
              -1.7578125
            ],
            [
              0.8515625,
              -2.703125
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 30917136896,
          "reserved_bytes": 31333548032,
          "peak_allocated_bytes": 31047681024,
          "peak_reserved_bytes": 31333548032
        },
        "median_ms": 117.55172442644835
      },
      {
        "case": 6,
        "tokens": 494,
        "media": false,
        "hidden_sha256": "43d127c3d1d4a761ae05e8df2740a6f9c3e87c7e2505533dde9326a4c4f5b058",
        "logits": [
          [
            [
              -2.578125,
              -2.96875,
              -0.484375,
              -3.484375
            ],
            [
              -3.125,
              -1.109375,
              -0.921875
            ],
            [
              1.6484375,
              -3.234375
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 30917137920,
          "reserved_bytes": 31383879680,
          "peak_allocated_bytes": 31064884736,
          "peak_reserved_bytes": 31383879680
        },
        "median_ms": 116.89481977373362
      },
      {
        "case": 7,
        "tokens": 449,
        "media": false,
        "hidden_sha256": "e1d3e1006ccccf6aacba450f8841f691cf92d51a40fb3be2342832e4c7798687",
        "logits": [
          [
            [
              -3.90625,
              -1.171875,
              -1.984375,
              -3.90625
            ],
            [
              0.28125,
              -3.953125,
              -2.796875
            ],
            [
              1.375,
              -3.03125
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 30917137920,
          "reserved_bytes": 31383879680,
          "peak_allocated_bytes": 31051187712,
          "peak_reserved_bytes": 31383879680
        },
        "median_ms": 117.62379622086883
      },
      {
        "case": 8,
        "tokens": 491,
        "media": false,
        "hidden_sha256": "4f5079806d430ae1d046b86d18a67af3cacdd7f62bb6ed3e7c99edecad048d31",
        "logits": [
          [
            [
              -4.1875,
              -2.46875,
              -0.26953125,
              -3.828125
            ],
            [
              -1.7578125,
              -2.90625,
              -1.2265625
            ],
            [
              1.6796875,
              -3.53125
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 30917137920,
          "reserved_bytes": 31383879680,
          "peak_allocated_bytes": 31063534080,
          "peak_reserved_bytes": 31383879680
        },
        "median_ms": 117.2104193829
      },
      {
        "case": 9,
        "tokens": 624,
        "media": false,
        "hidden_sha256": "cdb5d5ec50846b16a8c87da98008b8c93e11b80570894fd3ec0ac6b28bb89cdf",
        "logits": [
          [
            [
              -3.125,
              -0.8984375,
              -0.71875,
              -3.765625
            ],
            [
              -0.2109375,
              -2.640625,
              -1.5546875
            ],
            [
              1.3828125,
              -3.359375
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 30917139968,
          "reserved_bytes": 31383879680,
          "peak_allocated_bytes": 31100976128,
          "peak_reserved_bytes": 31383879680
        },
        "median_ms": 134.07096592709422
      },
      {
        "case": 10,
        "tokens": 454,
        "media": false,
        "hidden_sha256": "bc2bc8c35a51b4b082c379c097dac42759aedf388fbfbcf85f4b7a7971d0fb8b",
        "logits": [
          [
            [
              -4.34375,
              -2.59375,
              -0.22265625,
              -3.9375
            ],
            [
              -2.09375,
              -2.59375,
              -1.1328125
            ],
            [
              1.4375,
              -3.265625
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 30917137920,
          "reserved_bytes": 31383879680,
          "peak_allocated_bytes": 31053260288,
          "peak_reserved_bytes": 31383879680
        },
        "median_ms": 117.72878700867295
      },
      {
        "case": 11,
        "tokens": 442,
        "media": false,
        "hidden_sha256": "caccd4f2f27ee39e8903a5e20fd24916ad6d670e66e99b8184da23725a33326b",
        "logits": [
          [
            [
              -4.0,
              -0.34765625,
              -1.265625,
              -4.3125
            ],
            [
              0.248046875,
              -4.1875,
              -3.3125
            ],
            [
              -0.2578125,
              -1.34375
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 30917136896,
          "reserved_bytes": 31383879680,
          "peak_allocated_bytes": 31049227264,
          "peak_reserved_bytes": 31383879680
        },
        "median_ms": 117.89859551936388
      },
      {
        "case": 12,
        "tokens": 458,
        "media": false,
        "hidden_sha256": "412e18ef08c9375f461110ff5dd36114c13e517c6223db5df85bc20c9a045e3e",
        "logits": [
          [
            [
              -2.125,
              -3.796875,
              -0.328125,
              -2.671875
            ],
            [
              -3.953125,
              0.001953125,
              -1.71875
            ],
            [
              1.421875,
              -2.65625
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 30917137920,
          "reserved_bytes": 31383879680,
          "peak_allocated_bytes": 31053707776,
          "peak_reserved_bytes": 31383879680
        },
        "median_ms": 118.8604780472815
      },
      {
        "case": 13,
        "tokens": 620,
        "media": false,
        "hidden_sha256": "23a647bce8af2f2d6077fde8985dce1ebe78babbb63ed404ad555f0354b39555",
        "logits": [
          [
            [
              -3.109375,
              -1.8828125,
              -0.87890625,
              -3.953125
            ],
            [
              -0.796875,
              -2.28125,
              -1.3515625
            ],
            [
              1.8203125,
              -3.3125
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 30917139968,
          "reserved_bytes": 31383879680,
          "peak_allocated_bytes": 31103866880,
          "peak_reserved_bytes": 31383879680
        },
        "median_ms": 137.25669821724296
      },
      {
        "case": 14,
        "tokens": 616,
        "media": false,
        "hidden_sha256": "6f9506754fcfb7e5766f7db8fdb8c73597304aa41ff38e67c41f033e26aca023",
        "logits": [
          [
            [
              -3.328125,
              -0.921875,
              -1.140625,
              -3.921875
            ],
            [
              -0.26953125,
              -2.984375,
              -2.0625
            ],
            [
              1.21875,
              -3.5625
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 30917139968,
          "reserved_bytes": 31383879680,
          "peak_allocated_bytes": 31096927232,
          "peak_reserved_bytes": 31383879680
        },
        "median_ms": 133.38337745517492
      },
      {
        "case": 15,
        "tokens": 493,
        "media": false,
        "hidden_sha256": "920ca7b25e96d6dbb6bbd2585ff5931c71347c9e0a5d739298e6c9035e40e56b",
        "logits": [
          [
            [
              -2.46875,
              -2.09375,
              0.0078125,
              -3.75
            ],
            [
              -1.5546875,
              -1.671875,
              -1.359375
            ],
            [
              1.3046875,
              -3.09375
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 30917137920,
          "reserved_bytes": 31383879680,
          "peak_allocated_bytes": 31064683008,
          "peak_reserved_bytes": 31383879680
        },
        "median_ms": 117.7181527018547
      },
      {
        "case": 16,
        "tokens": 1278,
        "media": false,
        "hidden_sha256": "88bdd67d73f1462d1740dea4ebec6fdfa00e29dd76d60fe2bb3f585a0ffea188",
        "logits": [
          [
            [
              1.265625,
              0.40625,
              0.6953125,
              0.828125
            ],
            [
              0.859375,
              0.056640625,
              0.24609375,
              0.52734375
            ],
            [
              0.55859375,
              -0.091796875,
              0.21875,
              0.314453125
            ],
            [
              0.51171875,
              -0.0703125,
              -0.0234375,
              0.2265625
            ],
            [
              0.5703125,
              -0.150390625,
              -0.03515625,
              0.0859375
            ],
            [
              0.4296875,
              -0.25,
              -0.177734375,
              -0.015625
            ],
            [
              0.140625,
              -0.197265625,
              -0.203125,
              0.0390625
            ],
            [
              0.42578125,
              -0.07421875,
              -0.0234375,
              0.14453125
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 30917150208,
          "reserved_bytes": 31811698688,
          "peak_allocated_bytes": 31295658496,
          "peak_reserved_bytes": 31811698688
        },
        "median_ms": 245.00332586467266
      },
      {
        "case": 17,
        "tokens": 1502,
        "media": false,
        "hidden_sha256": "620de62d3f3cd75f195a80ef7ce174225523fc4ea127254050b400c6b04708aa",
        "logits": [
          [
            [
              1.40625,
              0.5390625,
              0.69921875,
              0.7578125
            ],
            [
              0.7734375,
              0.1328125,
              0.25390625,
              0.42578125
            ],
            [
              0.8359375,
              0.1171875,
              0.51171875,
              0.453125
            ],
            [
              0.9453125,
              0.17578125,
              0.3515625,
              0.29296875
            ],
            [
              0.9375,
              0.09375,
              0.24609375,
              0.296875
            ],
            [
              0.7109375,
              -0.1171875,
              0.078125,
              0.2265625
            ],
            [
              0.65234375,
              0.126953125,
              0.0546875,
              0.376953125
            ],
            [
              0.875,
              0.04296875,
              0.21875,
              0.26171875
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 30917154304,
          "reserved_bytes": 32126271488,
          "peak_allocated_bytes": 31361541632,
          "peak_reserved_bytes": 32126271488
        },
        "median_ms": 278.91440875828266
      },
      {
        "case": 18,
        "tokens": 2270,
        "media": false,
        "hidden_sha256": "d6c682b59804bcd7b9bfd96f66c20a57268ce65ca24179a5da266433e0173cc2",
        "logits": [
          [
            [
              0.6953125,
              -0.03515625,
              0.03515625,
              0.328125
            ],
            [
              0.3515625,
              -0.236328125,
              -0.10546875,
              0.1484375
            ],
            [
              0.22265625,
              -0.34375,
              -0.0546875,
              0.0625
            ],
            [
              0.421875,
              -0.0390625,
              0.12890625,
              0.3671875
            ],
            [
              0.2734375,
              -0.337890625,
              -0.1015625,
              0.09375
            ],
            [
              0.078125,
              -0.5234375,
              -0.42578125,
              0.05078125
            ],
            [
              -0.11328125,
              -0.5078125,
              -0.283203125,
              0.00390625
            ],
            [
              -0.0234375,
              -0.75390625,
              -0.24609375,
              -0.15234375
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 30917166592,
          "reserved_bytes": 32604422144,
          "peak_allocated_bytes": 31588405760,
          "peak_reserved_bytes": 32604422144
        },
        "median_ms": 401.4103366062045
      },
      {
        "case": 19,
        "tokens": 298,
        "media": false,
        "hidden_sha256": "93a4b2a5b5ccc85db7ef415d90158e97efbc5cecfd28e41ee35cc11c2ac31c3a",
        "logits": [
          [
            [
              -3.375,
              -4.5,
              -3.4375,
              0.984375
            ],
            [
              -3.859375,
              0.1796875
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 30917134848,
          "reserved_bytes": 32604422144,
          "peak_allocated_bytes": 31007036928,
          "peak_reserved_bytes": 32604422144
        },
        "median_ms": 116.23586062341928
      },
      {
        "case": 20,
        "tokens": 303,
        "media": false,
        "hidden_sha256": "c15885b277be05a2bc708b698408a398d7f631d0cb97640bd0fdbd9fa5efca32",
        "logits": [
          [
            [
              -4.8125,
              -4.71875,
              0.15625,
              -4.34375
            ],
            [
              -0.328125,
              -3.234375
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 30917134848,
          "reserved_bytes": 32604422144,
          "peak_allocated_bytes": 31008309760,
          "peak_reserved_bytes": 32604422144
        },
        "median_ms": 116.06947844848037
      },
      {
        "case": 21,
        "tokens": 304,
        "media": false,
        "hidden_sha256": "82c4876ee5204a96f0fd18c0d868680ceb321ae70f2a65d8d797cb01625c70e3",
        "logits": [
          [
            [
              0.8828125,
              -4.40625,
              -4.125,
              -3.296875
            ],
            [
              -4.53125,
              0.080078125
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 30917134848,
          "reserved_bytes": 32604422144,
          "peak_allocated_bytes": 31005399552,
          "peak_reserved_bytes": 32604422144
        },
        "median_ms": 108.67965733632445
      },
      {
        "case": 22,
        "tokens": 312,
        "media": false,
        "hidden_sha256": "2401abb62d11d5d56ce4634631763b3c6aa6189e37fc2427129a07f2c88c4e84",
        "logits": [
          [
            [
              -3.671875,
              -1.96875,
              1.3828125,
              -3.859375
            ],
            [
              -1.2578125,
              -3.078125
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 30917134848,
          "reserved_bytes": 32604422144,
          "peak_allocated_bytes": 31009006080,
          "peak_reserved_bytes": 32604422144
        },
        "median_ms": 108.80827996879816
      },
      {
        "case": 23,
        "tokens": 379,
        "media": true,
        "hidden_sha256": "1ae8eb74ab8e4b3e777a5c62859c5334128f8d7038067b901904b886fc2a5d47",
        "logits": [
          [
            [
              -3.6875,
              -1.625,
              1.625,
              -3.6875
            ],
            [
              -2.03125,
              -2.546875
            ]
          ]
        ],
        "memory": {
          "allocated_bytes": 30918712832,
          "reserved_bytes": 32606519296,
          "peak_allocated_bytes": 31035676672,
          "peak_reserved_bytes": 32606519296
        },
        "median_ms": 127.35875509679317
      }
    ],
    "cold": {
      "allocated_bytes": 30882526720,
      "reserved_bytes": 30918311936,
      "peak_allocated_bytes": 30882551296,
      "peak_reserved_bytes": 30918311936,
      "seconds": 9.994719533249736,
      "nvml_process_peak_bytes": 31557943296
    },
    "repeat": 30,
    "runtime_patched": false,
    "runtime_sha256": {
      "clef_int8_loader.py": "286752ba8076f222115a8e0cca9396cf6ca084d3fce2407d19aa2b3176fe7b5d",
      "clef_int8_calibration.py": "135611910883019123eb1cb91dee7c2248e0e4e92c9563414c684d24bca794b0",
      "clef_text_int8.py": "14f5ea9bfd5e21819cf40cf2176c87ceaa4f00d633cf658917256d4667fda84f",
      "clef_int8_cutlass.py": "a516fa0c8007aa61814ea3e0ed251a3c0f0da626161bf2fe5ac30141732f857b",
      "clef_text_cute.py": "ddc39f8bd543b7c6400e8c178ad408f0944fc7f1c881f4d36861ef497516689f",
      "clef_text_math.py": "7b7f8c489fdb56b29c86a003d37ed96f440afe0fbe5001250036aad3e79ca760",
      "clef_vision_norm.py": "398b681d0600dade5dc26f7664fa450ec39794f5615acac561c2032648947e6f"
    },
    "bf16_loader_forbidden": true,
    "prepacked_mlp_pairs": 0,
    "nvml_process_peak_bytes": 33327939584,
    "nvml_samples": 23524,
    "batching": [
      {
        "batch_size": 1,
        "tokens_per_record": 1502,
        "questions_per_record": 8,
        "repeat": 10,
        "batch_one_logits_max_abs": 0.0,
        "batch_one_decisions_agree": true,
        "status": "ok",
        "latency_ms": {
          "mean": 278.9234106428921,
          "median": 278.92032684758306,
          "p95": 279.09680362790823,
          "min": 278.8346903398633,
          "max": 279.09680362790823
        },
        "memory": {
          "allocated_bytes": 30917158912,
          "reserved_bytes": 32606519296,
          "peak_allocated_bytes": 31361546240,
          "peak_reserved_bytes": 32606519296
        },
        "decisions_per_second": 28.682025761326557,
        "records_per_second": 3.5852532201658196
      },
      {
        "batch_size": 2,
        "tokens_per_record": 1502,
        "questions_per_record": 8,
        "repeat": 10,
        "batch_one_logits_max_abs": 0.201171875,
        "batch_one_decisions_agree": true,
        "status": "ok",
        "latency_ms": {
          "mean": 521.7992178164423,
          "median": 521.768159698695,
          "p95": 522.1206778660417,
          "min": 521.5699067339301,
          "max": 522.1206778660417
        },
        "memory": {
          "allocated_bytes": 30917211136,
          "reserved_bytes": 33235664896,
          "peak_allocated_bytes": 31805902848,
          "peak_reserved_bytes": 33235664896
        },
        "decisions_per_second": 30.664960485974284,
        "records_per_second": 3.8331200607467855
      },
      {
        "batch_size": 4,
        "tokens_per_record": 1502,
        "questions_per_record": 8,
        "repeat": 10,
        "batch_one_logits_max_abs": 0.21875,
        "batch_one_decisions_agree": true,
        "status": "ok",
        "latency_ms": {
          "mean": 979.3407004326582,
          "median": 979.3299422599375,
          "p95": 979.7763302922249,
          "min": 979.0457393974066,
          "max": 979.7763302922249
        },
        "memory": {
          "allocated_bytes": 30917282816,
          "reserved_bytes": 35236347904,
          "peak_allocated_bytes": 32660537856,
          "peak_reserved_bytes": 35236347904
        },
        "decisions_per_second": 32.67540245543359,
        "records_per_second": 4.084425306929199
      },
      {
        "batch_size": 8,
        "tokens_per_record": 1502,
        "questions_per_record": 8,
        "repeat": 10,
        "batch_one_logits_max_abs": 0.21875,
        "batch_one_decisions_agree": true,
        "status": "ok",
        "latency_ms": {
          "mean": 1906.714035756886,
          "median": 1906.6964145749807,
          "p95": 1907.1267684921622,
          "min": 1906.2150437384844,
          "max": 1907.1267684921622
        },
        "memory": {
          "allocated_bytes": 30917427200,
          "reserved_bytes": 39225131008,
          "peak_allocated_bytes": 34403283968,
          "peak_reserved_bytes": 39225131008
        },
        "decisions_per_second": 33.56590986943569,
        "records_per_second": 4.195738733679462
      },
      {
        "batch_size": 16,
        "tokens_per_record": 1502,
        "questions_per_record": 8,
        "repeat": 10,
        "batch_one_logits_max_abs": 0.21875,
        "batch_one_decisions_agree": true,
        "status": "ok",
        "latency_ms": {
          "mean": 3756.9526378065348,
          "median": 3757.018485106528,
          "p95": 3757.3599722236395,
          "min": 3756.4761312678456,
          "max": 3757.3599722236395
        },
        "memory": {
          "allocated_bytes": 30917715968,
          "reserved_bytes": 47202697216,
          "peak_allocated_bytes": 37889064960,
          "peak_reserved_bytes": 47202697216
        },
        "decisions_per_second": 34.06956886355874,
        "records_per_second": 4.258696107944843
      }
    ],
    "batching_nvml_process_peak_bytes": 47924117504,
    "peak_allocated_bytes": 31588405760,
    "peak_reserved_bytes": 32606519296
  },
  "calibrated_int8_cute": {
    "mode": "direct",
    "gpu": "NVIDIA H100 80GB HBM3",
    "torch": "2.9.0a0+50eac811a6.nv25.09",
    "memory_metric": "PyTorch allocated/reserved bytes; NVML process memory sampled every 5 ms",
    "records": [
      {
        "case": 0,
        "tokens": 620,
        "media": false,
        "hidden_sha256": "3bd80cec0945df8323e7f97e1c446dc064516975aa15b94f5a8808715034a2a8",
        "logits": [
          [
            [
              -3.140625,
              -2.0,
              -0.421875,
              -3.46875
            ],
            [
              -0.99609375,
              -1.640625,
              -1.21875
            ],
            [
              1.546875,
              -3.21875
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "int8-torch",
        "memory": {
          "allocated_bytes": 30796316160,
          "reserved_bytes": 30958157824,
          "peak_allocated_bytes": 30905440256,
          "peak_reserved_bytes": 30958157824
        },
        "median_ms": 82.21716340631247
      },
      {
        "case": 1,
        "tokens": 449,
        "media": false,
        "hidden_sha256": "0e416bd18959ddda56a98b01d58fb30d7a66ee16a2dd5d34a68472dccc15dfc5",
        "logits": [
          [
            [
              -3.953125,
              -0.9609375,
              -1.6015625,
              -3.59375
            ],
            [
              -1.5,
              -2.28125,
              -1.0
            ],
            [
              0.84375,
              -2.6875
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "int8-torch",
        "memory": {
          "allocated_bytes": 30796314112,
          "reserved_bytes": 31008489472,
          "peak_allocated_bytes": 30877440000,
          "peak_reserved_bytes": 31008489472
        },
        "median_ms": 82.10805151611567
      },
      {
        "case": 2,
        "tokens": 622,
        "media": false,
        "hidden_sha256": "332b8329d6b9cf675783d97c37ac3bb9eb04953f6d1acdeda6fe3147ccb4fe16",
        "logits": [
          [
            [
              -3.90625,
              -0.87109375,
              -0.142578125,
              -3.828125
            ],
            [
              -0.59375,
              -2.90625,
              -1.1015625
            ],
            [
              1.28125,
              -3.28125
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "int8-torch",
        "memory": {
          "allocated_bytes": 30796316160,
          "reserved_bytes": 31008489472,
          "peak_allocated_bytes": 30905606144,
          "peak_reserved_bytes": 31008489472
        },
        "median_ms": 82.51357823610306
      },
      {
        "case": 3,
        "tokens": 490,
        "media": false,
        "hidden_sha256": "26362588c871286a4878b331e1d9ec29c920510e59623fea0d6d24e28fa39857",
        "logits": [
          [
            [
              -3.90625,
              -0.9609375,
              -0.76953125,
              -4.34375
            ],
            [
              -0.5546875,
              -3.453125,
              -2.5
            ],
            [
              1.15625,
              -2.90625
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "int8-torch",
        "memory": {
          "allocated_bytes": 30796314112,
          "reserved_bytes": 31056723968,
          "peak_allocated_bytes": 30884106752,
          "peak_reserved_bytes": 31056723968
        },
        "median_ms": 82.37049542367458
      },
      {
        "case": 4,
        "tokens": 448,
        "media": false,
        "hidden_sha256": "39bcbe99f06b8c754b1395b52aa1cf14e92ec5d38db64c7cc6b93c77200c475e",
        "logits": [
          [
            [
              -4.46875,
              0.5703125,
              -3.28125,
              -4.4375
            ],
            [
              0.609375,
              -4.28125,
              -3.03125
            ],
            [
              -2.203125,
              -0.953125
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "int8-torch",
        "memory": {
          "allocated_bytes": 30796313088,
          "reserved_bytes": 31056723968,
          "peak_allocated_bytes": 30875826176,
          "peak_reserved_bytes": 31056723968
        },
        "median_ms": 83.57401937246323
      },
      {
        "case": 5,
        "tokens": 437,
        "media": false,
        "hidden_sha256": "937f7824d74e9a912f87b26807db9c82bfcf1f2e49925e1db99a3daaf797e478",
        "logits": [
          [
            [
              -3.796875,
              -1.109375,
              -2.15625,
              -3.90625
            ],
            [
              -0.70703125,
              -2.84375,
              -1.7578125
            ],
            [
              0.8515625,
              -2.703125
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "int8-torch",
        "memory": {
          "allocated_bytes": 30796313088,
          "reserved_bytes": 31056723968,
          "peak_allocated_bytes": 30873589760,
          "peak_reserved_bytes": 31056723968
        },
        "median_ms": 83.1351401284337
      },
      {
        "case": 6,
        "tokens": 494,
        "media": false,
        "hidden_sha256": "43d127c3d1d4a761ae05e8df2740a6f9c3e87c7e2505533dde9326a4c4f5b058",
        "logits": [
          [
            [
              -2.578125,
              -2.96875,
              -0.484375,
              -3.484375
            ],
            [
              -3.125,
              -1.109375,
              -0.921875
            ],
            [
              1.6484375,
              -3.234375
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "int8-torch",
        "memory": {
          "allocated_bytes": 30796314112,
          "reserved_bytes": 31107055616,
          "peak_allocated_bytes": 30884258816,
          "peak_reserved_bytes": 31107055616
        },
        "median_ms": 83.0352883785963
      },
      {
        "case": 7,
        "tokens": 449,
        "media": false,
        "hidden_sha256": "e1d3e1006ccccf6aacba450f8841f691cf92d51a40fb3be2342832e4c7798687",
        "logits": [
          [
            [
              -3.90625,
              -1.171875,
              -1.984375,
              -3.90625
            ],
            [
              0.28125,
              -3.953125,
              -2.796875
            ],
            [
              1.375,
              -3.03125
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "int8-torch",
        "memory": {
          "allocated_bytes": 30796314112,
          "reserved_bytes": 31107055616,
          "peak_allocated_bytes": 30877440000,
          "peak_reserved_bytes": 31107055616
        },
        "median_ms": 82.84110855311155
      },
      {
        "case": 8,
        "tokens": 491,
        "media": false,
        "hidden_sha256": "4f5079806d430ae1d046b86d18a67af3cacdd7f62bb6ed3e7c99edecad048d31",
        "logits": [
          [
            [
              -4.1875,
              -2.46875,
              -0.26953125,
              -3.828125
            ],
            [
              -1.7578125,
              -2.90625,
              -1.2265625
            ],
            [
              1.6796875,
              -3.53125
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "int8-torch",
        "memory": {
          "allocated_bytes": 30796314112,
          "reserved_bytes": 31107055616,
          "peak_allocated_bytes": 30884146176,
          "peak_reserved_bytes": 31107055616
        },
        "median_ms": 82.49550918117166
      },
      {
        "case": 9,
        "tokens": 624,
        "media": false,
        "hidden_sha256": "cdb5d5ec50846b16a8c87da98008b8c93e11b80570894fd3ec0ac6b28bb89cdf",
        "logits": [
          [
            [
              -3.125,
              -0.8984375,
              -0.71875,
              -3.765625
            ],
            [
              -0.2109375,
              -2.640625,
              -1.5546875
            ],
            [
              1.3828125,
              -3.359375
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "int8-torch",
        "memory": {
          "allocated_bytes": 30796316160,
          "reserved_bytes": 31107055616,
          "peak_allocated_bytes": 30905769984,
          "peak_reserved_bytes": 31107055616
        },
        "median_ms": 83.52960739284754
      },
      {
        "case": 10,
        "tokens": 454,
        "media": false,
        "hidden_sha256": "bc2bc8c35a51b4b082c379c097dac42759aedf388fbfbcf85f4b7a7971d0fb8b",
        "logits": [
          [
            [
              -4.34375,
              -2.59375,
              -0.22265625,
              -3.9375
            ],
            [
              -2.09375,
              -2.59375,
              -1.1328125
            ],
            [
              1.4375,
              -3.265625
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "int8-torch",
        "memory": {
          "allocated_bytes": 30796314112,
          "reserved_bytes": 31107055616,
          "peak_allocated_bytes": 30877618688,
          "peak_reserved_bytes": 31107055616
        },
        "median_ms": 83.26610410586
      },
      {
        "case": 11,
        "tokens": 442,
        "media": false,
        "hidden_sha256": "caccd4f2f27ee39e8903a5e20fd24916ad6d670e66e99b8184da23725a33326b",
        "logits": [
          [
            [
              -4.0,
              -0.34765625,
              -1.265625,
              -4.3125
            ],
            [
              0.248046875,
              -4.1875,
              -3.3125
            ],
            [
              -0.2578125,
              -1.34375
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "int8-torch",
        "memory": {
          "allocated_bytes": 30796313088,
          "reserved_bytes": 31107055616,
          "peak_allocated_bytes": 30873892352,
          "peak_reserved_bytes": 31107055616
        },
        "median_ms": 82.95116154477
      },
      {
        "case": 12,
        "tokens": 458,
        "media": false,
        "hidden_sha256": "412e18ef08c9375f461110ff5dd36114c13e517c6223db5df85bc20c9a045e3e",
        "logits": [
          [
            [
              -2.125,
              -3.796875,
              -0.328125,
              -2.671875
            ],
            [
              -3.953125,
              0.001953125,
              -1.71875
            ],
            [
              1.421875,
              -2.65625
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "int8-torch",
        "memory": {
          "allocated_bytes": 30796314112,
          "reserved_bytes": 31107055616,
          "peak_allocated_bytes": 30878805504,
          "peak_reserved_bytes": 31107055616
        },
        "median_ms": 83.5030721500516
      },
      {
        "case": 13,
        "tokens": 620,
        "media": false,
        "hidden_sha256": "23a647bce8af2f2d6077fde8985dce1ebe78babbb63ed404ad555f0354b39555",
        "logits": [
          [
            [
              -3.109375,
              -1.8828125,
              -0.87890625,
              -3.953125
            ],
            [
              -0.796875,
              -2.28125,
              -1.3515625
            ],
            [
              1.8203125,
              -3.3125
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "int8-torch",
        "memory": {
          "allocated_bytes": 30796316160,
          "reserved_bytes": 31107055616,
          "peak_allocated_bytes": 30905440256,
          "peak_reserved_bytes": 31107055616
        },
        "median_ms": 82.6035849750042
      },
      {
        "case": 14,
        "tokens": 616,
        "media": false,
        "hidden_sha256": "6f9506754fcfb7e5766f7db8fdb8c73597304aa41ff38e67c41f033e26aca023",
        "logits": [
          [
            [
              -3.328125,
              -0.921875,
              -1.140625,
              -3.921875
            ],
            [
              -0.26953125,
              -2.984375,
              -2.0625
            ],
            [
              1.21875,
              -3.5625
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "int8-torch",
        "memory": {
          "allocated_bytes": 30796316160,
          "reserved_bytes": 31107055616,
          "peak_allocated_bytes": 30905106944,
          "peak_reserved_bytes": 31107055616
        },
        "median_ms": 83.08295579627156
      },
      {
        "case": 15,
        "tokens": 493,
        "media": false,
        "hidden_sha256": "920ca7b25e96d6dbb6bbd2585ff5931c71347c9e0a5d739298e6c9035e40e56b",
        "logits": [
          [
            [
              -2.46875,
              -2.09375,
              0.0078125,
              -3.75
            ],
            [
              -1.5546875,
              -1.671875,
              -1.359375
            ],
            [
              1.3046875,
              -3.09375
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "int8-torch",
        "memory": {
          "allocated_bytes": 30796314112,
          "reserved_bytes": 31107055616,
          "peak_allocated_bytes": 30884220928,
          "peak_reserved_bytes": 31107055616
        },
        "median_ms": 83.13107490539551
      },
      {
        "case": 16,
        "tokens": 1278,
        "media": false,
        "hidden_sha256": "88bdd67d73f1462d1740dea4ebec6fdfa00e29dd76d60fe2bb3f585a0ffea188",
        "logits": [
          [
            [
              1.265625,
              0.40625,
              0.6953125,
              0.828125
            ],
            [
              0.859375,
              0.056640625,
              0.24609375,
              0.52734375
            ],
            [
              0.55859375,
              -0.091796875,
              0.21875,
              0.314453125
            ],
            [
              0.51171875,
              -0.0703125,
              -0.0234375,
              0.2265625
            ],
            [
              0.5703125,
              -0.150390625,
              -0.03515625,
              0.0859375
            ],
            [
              0.4296875,
              -0.25,
              -0.177734375,
              -0.015625
            ],
            [
              0.140625,
              -0.197265625,
              -0.203125,
              0.0390625
            ],
            [
              0.42578125,
              -0.07421875,
              -0.0234375,
              0.14453125
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "int8-torch",
        "memory": {
          "allocated_bytes": 30796326400,
          "reserved_bytes": 31107055616,
          "peak_allocated_bytes": 31017659904,
          "peak_reserved_bytes": 31107055616
        },
        "median_ms": 105.7324018329382
      },
      {
        "case": 17,
        "tokens": 1502,
        "media": false,
        "hidden_sha256": "620de62d3f3cd75f195a80ef7ce174225523fc4ea127254050b400c6b04708aa",
        "logits": [
          [
            [
              1.40625,
              0.5390625,
              0.69921875,
              0.7578125
            ],
            [
              0.7734375,
              0.1328125,
              0.25390625,
              0.42578125
            ],
            [
              0.8359375,
              0.1171875,
              0.51171875,
              0.453125
            ],
            [
              0.9453125,
              0.17578125,
              0.3515625,
              0.29296875
            ],
            [
              0.9375,
              0.09375,
              0.24609375,
              0.296875
            ],
            [
              0.7109375,
              -0.1171875,
              0.078125,
              0.2265625
            ],
            [
              0.65234375,
              0.126953125,
              0.0546875,
              0.376953125
            ],
            [
              0.875,
              0.04296875,
              0.21875,
              0.26171875
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "int8-torch",
        "memory": {
          "allocated_bytes": 30796330496,
          "reserved_bytes": 31178358784,
          "peak_allocated_bytes": 31057437184,
          "peak_reserved_bytes": 31178358784
        },
        "median_ms": 119.79983653873205
      },
      {
        "case": 18,
        "tokens": 2270,
        "media": false,
        "hidden_sha256": "d6c682b59804bcd7b9bfd96f66c20a57268ce65ca24179a5da266433e0173cc2",
        "logits": [
          [
            [
              0.6953125,
              -0.03515625,
              0.03515625,
              0.328125
            ],
            [
              0.3515625,
              -0.236328125,
              -0.10546875,
              0.1484375
            ],
            [
              0.22265625,
              -0.34375,
              -0.0546875,
              0.0625
            ],
            [
              0.421875,
              -0.0390625,
              0.12890625,
              0.3671875
            ],
            [
              0.2734375,
              -0.337890625,
              -0.1015625,
              0.09375
            ],
            [
              0.078125,
              -0.5234375,
              -0.42578125,
              0.05078125
            ],
            [
              -0.11328125,
              -0.5078125,
              -0.283203125,
              0.00390625
            ],
            [
              -0.0234375,
              -0.75390625,
              -0.24609375,
              -0.15234375
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "int8-torch",
        "memory": {
          "allocated_bytes": 30796342784,
          "reserved_bytes": 31480348672,
          "peak_allocated_bytes": 31190387200,
          "peak_reserved_bytes": 31480348672
        },
        "median_ms": 170.62044935300946
      },
      {
        "case": 19,
        "tokens": 298,
        "media": false,
        "hidden_sha256": "93a4b2a5b5ccc85db7ef415d90158e97efbc5cecfd28e41ee35cc11c2ac31c3a",
        "logits": [
          [
            [
              -3.375,
              -4.5,
              -3.4375,
              0.984375
            ],
            [
              -3.859375,
              0.1796875
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "int8-torch",
        "memory": {
          "allocated_bytes": 30796311040,
          "reserved_bytes": 31480348672,
          "peak_allocated_bytes": 30850066432,
          "peak_reserved_bytes": 31480348672
        },
        "median_ms": 83.5722703486681
      },
      {
        "case": 20,
        "tokens": 303,
        "media": false,
        "hidden_sha256": "c15885b277be05a2bc708b698408a398d7f631d0cb97640bd0fdbd9fa5efca32",
        "logits": [
          [
            [
              -4.8125,
              -4.71875,
              0.15625,
              -4.34375
            ],
            [
              -0.328125,
              -3.234375
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "int8-torch",
        "memory": {
          "allocated_bytes": 30796311040,
          "reserved_bytes": 31480348672,
          "peak_allocated_bytes": 30850537984,
          "peak_reserved_bytes": 31480348672
        },
        "median_ms": 82.87730859592557
      },
      {
        "case": 21,
        "tokens": 304,
        "media": false,
        "hidden_sha256": "82c4876ee5204a96f0fd18c0d868680ceb321ae70f2a65d8d797cb01625c70e3",
        "logits": [
          [
            [
              0.8828125,
              -4.40625,
              -4.125,
              -3.296875
            ],
            [
              -4.53125,
              0.080078125
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "int8-torch",
        "memory": {
          "allocated_bytes": 30796311040,
          "reserved_bytes": 31480348672,
          "peak_allocated_bytes": 30850636288,
          "peak_reserved_bytes": 31480348672
        },
        "median_ms": 83.31082761287689
      },
      {
        "case": 22,
        "tokens": 312,
        "media": false,
        "hidden_sha256": "2401abb62d11d5d56ce4634631763b3c6aa6189e37fc2427129a07f2c88c4e84",
        "logits": [
          [
            [
              -3.671875,
              -1.96875,
              1.3828125,
              -3.859375
            ],
            [
              -1.2578125,
              -3.078125
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "int8-torch",
        "memory": {
          "allocated_bytes": 30796311040,
          "reserved_bytes": 31480348672,
          "peak_allocated_bytes": 30852791808,
          "peak_reserved_bytes": 31480348672
        },
        "median_ms": 82.5475174933672
      },
      {
        "case": 23,
        "tokens": 379,
        "media": true,
        "hidden_sha256": "1ae8eb74ab8e4b3e777a5c62859c5334128f8d7038067b901904b886fc2a5d47",
        "logits": [
          [
            [
              -3.6875,
              -1.625,
              1.625,
              -3.6875
            ],
            [
              -2.03125,
              -2.546875
            ]
          ]
        ],
        "overlay_exact": true,
        "reference_exact": true,
        "reference_mode": "int8-torch",
        "memory": {
          "allocated_bytes": 30797889024,
          "reserved_bytes": 31480348672,
          "peak_allocated_bytes": 30868823552,
          "peak_reserved_bytes": 31480348672
        },
        "median_ms": 92.6055503077805
      }
    ],
    "cold": {
      "allocated_bytes": 30761702912,
      "reserved_bytes": 30798774272,
      "peak_allocated_bytes": 30761728000,
      "peak_reserved_bytes": 30798774272,
      "seconds": 11.18691000342369,
      "nvml_process_peak_bytes": 31440502784
    },
    "repeat": 30,
    "runtime_patched": true,
    "runtime_sha256": {
      "clef_int8_loader.py": "286752ba8076f222115a8e0cca9396cf6ca084d3fce2407d19aa2b3176fe7b5d",
      "clef_int8_calibration.py": "135611910883019123eb1cb91dee7c2248e0e4e92c9563414c684d24bca794b0",
      "clef_text_int8.py": "14f5ea9bfd5e21819cf40cf2176c87ceaa4f00d633cf658917256d4667fda84f",
      "clef_int8_cutlass.py": "a516fa0c8007aa61814ea3e0ed251a3c0f0da626161bf2fe5ac30141732f857b",
      "clef_text_cute.py": "ddc39f8bd543b7c6400e8c178ad408f0944fc7f1c881f4d36861ef497516689f",
      "clef_text_math.py": "7b7f8c489fdb56b29c86a003d37ed96f440afe0fbe5001250036aad3e79ca760",
      "clef_vision_norm.py": "398b681d0600dade5dc26f7664fa450ec39794f5615acac561c2032648947e6f"
    },
    "bf16_loader_forbidden": true,
    "prepacked_mlp_pairs": 64,
    "nvml_process_peak_bytes": 32203866112,
    "nvml_samples": 16696,
    "batching": [
      {
        "batch_size": 1,
        "tokens_per_record": 1502,
        "questions_per_record": 8,
        "repeat": 10,
        "batch_one_logits_max_abs": 0.0,
        "batch_one_decisions_agree": true,
        "status": "ok",
        "latency_ms": {
          "mean": 119.60493177175522,
          "median": 119.60197286680341,
          "p95": 119.72382944077253,
          "min": 119.49383094906807,
          "max": 119.72382944077253
        },
        "memory": {
          "allocated_bytes": 30796335104,
          "reserved_bytes": 31480348672,
          "peak_allocated_bytes": 31057441792,
          "peak_reserved_bytes": 31480348672
        },
        "decisions_per_second": 66.88852874449925,
        "records_per_second": 8.361066093062407
      },
      {
        "batch_size": 2,
        "tokens_per_record": 1502,
        "questions_per_record": 8,
        "repeat": 10,
        "batch_one_logits_max_abs": 0.201171875,
        "batch_one_decisions_agree": true,
        "status": "ok",
        "latency_ms": {
          "mean": 223.49175326526165,
          "median": 223.23126904666424,
          "p95": 224.78003799915314,
          "min": 222.98206482082605,
          "max": 224.78003799915314
        },
        "memory": {
          "allocated_bytes": 30796387328,
          "reserved_bytes": 31723618304,
          "peak_allocated_bytes": 31319557120,
          "peak_reserved_bytes": 31723618304
        },
        "decisions_per_second": 71.6745466185356,
        "records_per_second": 8.95931832731695
      },
      {
        "batch_size": 4,
        "tokens_per_record": 1502,
        "questions_per_record": 8,
        "repeat": 10,
        "batch_one_logits_max_abs": 0.21875,
        "batch_one_decisions_agree": true,
        "status": "ok",
        "latency_ms": {
          "mean": 426.28292785957456,
          "median": 426.41479056328535,
          "p95": 426.95363610982895,
          "min": 425.36329943686724,
          "max": 426.95363610982895
        },
        "memory": {
          "allocated_bytes": 30796459008,
          "reserved_bytes": 32633782272,
          "peak_allocated_bytes": 31841909248,
          "peak_reserved_bytes": 32633782272
        },
        "decisions_per_second": 75.04430124885829,
        "records_per_second": 9.380537656107286
      },
      {
        "batch_size": 8,
        "tokens_per_record": 1502,
        "questions_per_record": 8,
        "repeat": 10,
        "batch_one_logits_max_abs": 0.21875,
        "batch_one_decisions_agree": true,
        "status": "ok",
        "latency_ms": {
          "mean": 835.4205219075084,
          "median": 835.1544700562954,
          "p95": 838.116173632443,
          "min": 833.198837004602,
          "max": 838.116173632443
        },
        "memory": {
          "allocated_bytes": 30796603392,
          "reserved_bytes": 34296823808,
          "peak_allocated_bytes": 32886256640,
          "peak_reserved_bytes": 34296823808
        },
        "decisions_per_second": 76.63253002248308,
        "records_per_second": 9.579066252810385
      },
      {
        "batch_size": 16,
        "tokens_per_record": 1502,
        "questions_per_record": 8,
        "repeat": 10,
        "batch_one_logits_max_abs": 0.21875,
        "batch_one_decisions_agree": true,
        "status": "ok",
        "latency_ms": {
          "mean": 1663.1453910842538,
          "median": 1663.0147453397512,
          "p95": 1668.0818516761065,
          "min": 1659.6905048936605,
          "max": 1668.0818516761065
        },
        "memory": {
          "allocated_bytes": 30796892160,
          "reserved_bytes": 37610323968,
          "peak_allocated_bytes": 34972632064,
          "peak_reserved_bytes": 37610323968
        },
        "decisions_per_second": 76.9686500728229,
        "records_per_second": 9.621081259102862
      }
    ],
    "batching_nvml_process_peak_bytes": 38335938560,
    "peak_allocated_bytes": 31190387200,
    "peak_reserved_bytes": 31480348672
  }
}