Add Details tab

app.py

@@ -19,6 +19,8 @@ EXCLUDED_KEYS =  {
 #     "alias",
 # }
 
+DETAILS_DATASET_ID = "datasets/open-llm-leaderboard/{model_name_sanitized}-details"
+DETAILS_FILENAME = "samples_{subtask}_*.json"
 
 TASKS = {
     "leaderboard_arc_challenge": ("ARC", "leaderboard_arc_challenge"),
@@ -29,6 +31,57 @@ TASKS = {
     "leaderboard_mmlu_pro": ("MMLU-Pro", "leaderboard_mmlu_pro"),
     "leaderboard_musr": ("MuSR", "leaderboard_musr"),
 }
+SUBTASKS = {
+    "leaderboard_arc_challenge": ["leaderboard_arc_challenge"],
+    "leaderboard_bbh": [
+        "leaderboard_bbh_boolean_expressions",
+        "leaderboard_bbh_causal_judgement",
+        "leaderboard_bbh_date_understanding",
+        "leaderboard_bbh_disambiguation_qa",
+        "leaderboard_bbh_formal_fallacies",
+        "leaderboard_bbh_geometric_shapes",
+        "leaderboard_bbh_hyperbaton",
+        "leaderboard_bbh_logical_deduction_five_objects",
+        "leaderboard_bbh_logical_deduction_seven_objects",
+        "leaderboard_bbh_logical_deduction_three_objects",
+        "leaderboard_bbh_movie_recommendation",
+        "leaderboard_bbh_navigate",
+        "leaderboard_bbh_object_counting",
+        "leaderboard_bbh_penguins_in_a_table",
+        "leaderboard_bbh_reasoning_about_colored_objects",
+        "leaderboard_bbh_ruin_names",
+        "leaderboard_bbh_salient_translation_error_detection",
+        "leaderboard_bbh_snarks", "leaderboard_bbh_sports_understanding",
+        "leaderboard_bbh_temporal_sequences",
+        "leaderboard_bbh_tracking_shuffled_objects_five_objects",
+        "leaderboard_bbh_tracking_shuffled_objects_seven_objects",
+        "leaderboard_bbh_tracking_shuffled_objects_three_objects",
+        "leaderboard_bbh_web_of_lies",
+    ],
+    "leaderboard_gpqa": [
+        "leaderboard_gpqa_extended",
+        "leaderboard_gpqa_diamond",
+        "leaderboard_gpqa_main",
+    ],
+    "leaderboard_ifeval": ["leaderboard_ifeval"],
+    # "leaderboard_math_hard": [
+    "leaderboard_math": [
+        "leaderboard_math_algebra_hard",
+        "leaderboard_math_counting_and_prob_hard",
+        "leaderboard_math_geometry_hard",
+        "leaderboard_math_intermediate_algebra_hard",
+        "leaderboard_math_num_theory_hard",
+        "leaderboard_math_prealgebra_hard",
+        "leaderboard_math_precalculus_hard",
+    ],
+    "leaderboard_mmlu_pro": ["leaderboard_mmlu_pro"],
+    "leaderboard_musr": [
+        "leaderboard_musr_murder_mysteries",
+        "leaderboard_musr_object_placements",
+        "leaderboard_musr_team_allocation",
+    ],
+}
+
 
 fs = HfFileSystem()
 
@@ -103,6 +156,49 @@ def update_tasks(task):
         )
 
 
+def update_subtasks(task):
+    return gr.Radio(
+        SUBTASKS.get(task),
+        info="Evaluation subtasks to be displayed",
+    )
+
+
+def load_details_dataframe(model_id, subtask):
+    if not model_id or not subtask:
+        return
+    model_name_sanitized = model_id.replace("/", "__")
+    paths = fs.glob(
+        f"{DETAILS_DATASET_ID}/**/{DETAILS_FILENAME}".format(
+            model_name_sanitized=model_name_sanitized, subtask=subtask
+        )
+    )
+    if not paths:
+        return
+    path = max(paths)
+    with fs.open(path, "r") as f:
+        data = [json.loads(line) for line in f]
+    df = pd.json_normalize(data)
+    # df = df.rename_axis("Parameters", axis="columns")
+    df["model_name"] = model_id  # Keep model_name
+    return df
+    # return df.set_index(pd.Index([model_id])).reset_index()
+
+
+def display_details(df_1, df_2, sample_idx):
+    s_1 = df_1.iloc[sample_idx]
+    s_2 = df_2.iloc[sample_idx]
+    # Pop model_name and add it to the column name
+    s_1 = s_1.rename(s_1.pop("model_name"))
+    s_2 = s_2.rename(s_2.pop("model_name"))
+    df = pd.concat([s_1, s_2], axis="columns")#.rename_axis("Parameters").reset_index()
+    return (
+        df.style
+        .format(na_rep="")
+        # .hide(axis="index")
+        .to_html()
+    )
+
+
 # if __name__ == "__main__":
 latest_result_path_per_model = filter_latest_result_path_per_model(fetch_result_paths())
 
@@ -135,6 +231,18 @@ with gr.Blocks(fill_height=True) as demo:
             results = gr.HTML()
         with gr.Tab("Configs"):
             configs = gr.HTML()
+        with gr.Tab("Details"):
+            subtask = gr.Radio(
+                SUBTASKS.get(task.value),
+                label="Subtasks",
+                info="Evaluation subtasks to be displayed (choose one of the Tasks above)",
+            )
+            sample_idx = gr.Number(value=0, label="Sample Index", info="Index of the sample to be displayed", minimum=0)
+            load_details_btn = gr.Button("Load Details")
+            details = gr.HTML()
+            details_dataframe_1 = gr.Dataframe(visible=False)
+            details_dataframe_2 = gr.Dataframe(visible=False)
+            details_dataframe = gr.DataFrame(visible=False)
 
     load_btn_1.click(
         fn=load_result_dataframe,
@@ -166,6 +274,29 @@ with gr.Blocks(fill_height=True) as demo:
         fn=display_results,
         inputs=[dataframe_1, dataframe_2, task],
         outputs=[results, configs],
+    ).then(
+        fn=update_subtasks,
+        inputs=task,
+        outputs=subtask,
+    )
+
+    load_details_btn.click(
+        fn=load_details_dataframe,
+        inputs=[model_id_1, subtask],
+        outputs=details_dataframe_1,
+    ).then(
+        fn=load_details_dataframe,
+        inputs=[model_id_2, subtask],
+        outputs=details_dataframe_2,
+    ).then(
+        fn=display_details,
+        inputs=[details_dataframe_1, details_dataframe_2, sample_idx],
+        outputs=details,
+    )
+    sample_idx.change(
+        fn=display_details,
+        inputs=[details_dataframe_1, details_dataframe_2, sample_idx],
+        outputs=details,
     )
 
 demo.launch()