{ "lanes": { "lola": { "pass": 144, "fail": 20 }, "claude": { "pass": 159, "fail": 5 }, "grok": { "pass": 135, "fail": 29 }, "deep": { "pass": 137, "fail": 27 } }, "problems": [ { "task_id": "HumanEval/0", "lanes": { "lola": false, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/1", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/2", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/3", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/4", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/5", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/6", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/7", "lanes": { "lola": false, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/8", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/9", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/10", "lanes": { "lola": true, "claude": true, "grok": true, "deep": false } }, { "task_id": "HumanEval/11", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/12", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/13", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/14", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/15", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/16", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/17", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/18", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/19", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/20", "lanes": { "lola": false, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/21", "lanes": { "lola": false, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/22", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/23", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/24", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/25", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/26", "lanes": { "lola": true, "claude": true, "grok": false, "deep": false } }, { "task_id": "HumanEval/27", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/28", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/29", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/30", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/31", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/32", "lanes": { "lola": false, "claude": true, "grok": false, "deep": false } }, { "task_id": "HumanEval/33", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/34", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/35", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/36", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/37", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/38", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/39", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/40", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/41", "lanes": { "lola": false, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/42", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/43", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/44", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/45", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/46", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/47", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/48", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/49", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/50", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/51", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/52", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/53", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/54", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/55", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/56", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/57", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/58", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/59", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/60", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/61", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/62", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/63", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/64", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/65", "lanes": { "lola": true, "claude": true, "grok": false, "deep": false } }, { "task_id": "HumanEval/66", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/67", "lanes": { "lola": true, "claude": true, "grok": false, "deep": true } }, { "task_id": "HumanEval/68", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/69", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/70", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/71", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/72", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/73", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/74", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/75", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/76", "lanes": { "lola": true, "claude": false, "grok": false, "deep": true } }, { "task_id": "HumanEval/77", "lanes": { "lola": false, "claude": true, "grok": true, "deep": false } }, { "task_id": "HumanEval/78", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/79", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/80", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/81", "lanes": { "lola": true, "claude": true, "grok": true, "deep": false } }, { "task_id": "HumanEval/82", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/83", "lanes": { "lola": true, "claude": false, "grok": false, "deep": true } }, { "task_id": "HumanEval/84", "lanes": { "lola": true, "claude": true, "grok": true, "deep": false } }, { "task_id": "HumanEval/85", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/86", "lanes": { "lola": true, "claude": true, "grok": false, "deep": true } }, { "task_id": "HumanEval/87", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/88", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/89", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/90", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/91", "lanes": { "lola": false, "claude": true, "grok": false, "deep": false } }, { "task_id": "HumanEval/92", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/93", "lanes": { "lola": true, "claude": false, "grok": false, "deep": false } }, { "task_id": "HumanEval/94", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/95", "lanes": { "lola": true, "claude": true, "grok": false, "deep": true } }, { "task_id": "HumanEval/96", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/97", "lanes": { "lola": false, "claude": true, "grok": true, "deep": false } }, { "task_id": "HumanEval/98", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/99", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/100", "lanes": { "lola": true, "claude": true, "grok": false, "deep": true } }, { "task_id": "HumanEval/101", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/102", "lanes": { "lola": true, "claude": true, "grok": false, "deep": true } }, { "task_id": "HumanEval/103", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/104", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/105", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/106", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/107", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/108", "lanes": { "lola": false, "claude": true, "grok": false, "deep": false } }, { "task_id": "HumanEval/109", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/110", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/111", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/112", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/113", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/114", "lanes": { "lola": true, "claude": true, "grok": true, "deep": false } }, { "task_id": "HumanEval/115", "lanes": { "lola": false, "claude": true, "grok": false, "deep": false } }, { "task_id": "HumanEval/116", "lanes": { "lola": true, "claude": true, "grok": true, "deep": false } }, { "task_id": "HumanEval/117", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/118", "lanes": { "lola": true, "claude": true, "grok": false, "deep": true } }, { "task_id": "HumanEval/119", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/120", "lanes": { "lola": false, "claude": true, "grok": false, "deep": true } }, { "task_id": "HumanEval/121", "lanes": { "lola": true, "claude": true, "grok": false, "deep": true } }, { "task_id": "HumanEval/122", "lanes": { "lola": true, "claude": true, "grok": true, "deep": false } }, { "task_id": "HumanEval/123", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/124", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/125", "lanes": { "lola": true, "claude": true, "grok": false, "deep": true } }, { "task_id": "HumanEval/126", "lanes": { "lola": true, "claude": true, "grok": false, "deep": true } }, { "task_id": "HumanEval/127", "lanes": { "lola": true, "claude": true, "grok": true, "deep": false } }, { "task_id": "HumanEval/128", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/129", "lanes": { "lola": true, "claude": true, "grok": false, "deep": true } }, { "task_id": "HumanEval/130", "lanes": { "lola": false, "claude": false, "grok": false, "deep": false } }, { "task_id": "HumanEval/131", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/132", "lanes": { "lola": false, "claude": false, "grok": false, "deep": false } }, { "task_id": "HumanEval/133", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/134", "lanes": { "lola": false, "claude": true, "grok": false, "deep": false } }, { "task_id": "HumanEval/135", "lanes": { "lola": true, "claude": true, "grok": false, "deep": true } }, { "task_id": "HumanEval/136", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/137", "lanes": { "lola": true, "claude": true, "grok": true, "deep": false } }, { "task_id": "HumanEval/138", "lanes": { "lola": true, "claude": true, "grok": false, "deep": true } }, { "task_id": "HumanEval/139", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/140", "lanes": { "lola": false, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/141", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/142", "lanes": { "lola": true, "claude": true, "grok": false, "deep": true } }, { "task_id": "HumanEval/143", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/144", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/145", "lanes": { "lola": false, "claude": true, "grok": false, "deep": false } }, { "task_id": "HumanEval/146", "lanes": { "lola": true, "claude": true, "grok": true, "deep": false } }, { "task_id": "HumanEval/147", "lanes": { "lola": true, "claude": true, "grok": true, "deep": false } }, { "task_id": "HumanEval/148", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/149", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/150", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/151", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/152", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/153", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/154", "lanes": { "lola": true, "claude": true, "grok": false, "deep": false } }, { "task_id": "HumanEval/155", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/156", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/157", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/158", "lanes": { "lola": false, "claude": true, "grok": true, "deep": false } }, { "task_id": "HumanEval/159", "lanes": { "lola": false, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/160", "lanes": { "lola": true, "claude": true, "grok": true, "deep": false } }, { "task_id": "HumanEval/161", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/162", "lanes": { "lola": true, "claude": true, "grok": true, "deep": true } }, { "task_id": "HumanEval/163", "lanes": { "lola": false, "claude": true, "grok": false, "deep": false } } ] }