dynamic averages

app.py

@@ -3,6 +3,7 @@ import gzip
 import gradio as gr
 from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
 import pandas as pd
+import numpy as np
 from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import snapshot_download
 from io import StringIO
@@ -416,6 +417,18 @@ def init_leaderboard_mib_causalgraph(dataframe, track):
     # print(dataframe)
     renamed_df = dataframe.rename(columns=display_mapping)
     
+    # idx_to_method = {0: "Full Vector", 1: "DAS", 2: "DBM", 3: "PCA", 4: "SAE"}
+    # idx_to_scores = {0: [0.38, 0.36, 0.38, 0.42],
+    #                  1: [0.56, 0.62, 0.54, 0.51],
+    #                  2: [0.43, 0.41, 0.53, 0.49],
+    #                  3: [0.26, 0.20, 0.32, 0.40],
+    #                  4: ["-", "-", 0.33, "-"]}
+    # renamed_df.loc[0]["Method"] = "Full Vector"
+    # for i in range(5):
+    #     renamed_df.loc[i] = [idx_to_method[i]] + idx_to_scores[i]
+        
+    print(renamed_df)
+
     # Create only necessary columns
     return Leaderboard(
         value=renamed_df,
@@ -476,28 +489,81 @@ def process_json(temp_file):
 
 # Define the preset substrings for filtering
 PRESET_SUBSTRINGS = ["IOI", "MCQA", "Arithmetic", "ARC", "GPT-2", "Qwen-2.5", "Gemma-2", "Llama-3.1"]
+TASK_SUBSTRINGS = ["IOI", "MCQA", "Arithmetic", "ARC"]
+MODEL_SUBSTRINGS = ["GPT-2", "Qwen-2.5", "Gemma-2", "Llama-3.1"]
 
-def filter_columns_by_substrings(dataframe: pd.DataFrame, selected_substrings: List[str]) -> pd.DataFrame:
+def filter_columns_by_substrings(dataframe: pd.DataFrame, selected_task_substrings: List[str],
+                                 selected_model_substrings: List[str]) -> pd.DataFrame:
     """
     Filter columns based on the selected substrings.
     """
     original_dataframe = deepcopy(dataframe)
-    if not selected_substrings:
+    if not selected_task_substrings and not selected_model_substrings:
         return dataframe  # No filtering if no substrings are selected
 
-    # Filter columns that contain any of the selected substrings
+    if not selected_task_substrings:
+        # Filter columns that contain any of the selected model substrings
+        filtered_columns = [
+            col for col in dataframe.columns
+            if any(sub.lower() in col.lower() for sub in selected_model_substrings)
+            or col == "Method"
+        ]
+        return dataframe[filtered_columns]
+    elif not selected_model_substrings:
+        # Filter columns that contain any of the selected task substrings
+        filtered_columns = [
+            col for col in dataframe.columns
+            if any(sub.lower() in col.lower() for sub in selected_task_substrings)
+            or col == "Method"
+        ]
+        return dataframe[filtered_columns]
+
+    # Filter columns by task first. Use AND logic to combine with model filtering
     filtered_columns = [
         col for col in dataframe.columns
-        if any(sub.lower() in col.lower() for sub in selected_substrings)
+        if any(sub.lower() in col.lower() for sub in selected_task_substrings)
         or col == "Method"
     ]
+    filtered_columns = [
+        col for col in dataframe[filtered_columns].columns
+        if any(sub.lower() in col.lower() for sub in selected_model_substrings)
+        or col == "Method"
+    ]
+
     return dataframe[filtered_columns]
 
-def update_leaderboard(dataframe: pd.DataFrame, selected_substrings: List[str]):
+def update_leaderboard(dataframe: pd.DataFrame, selected_task_substrings: List[str],
+                       selected_model_substrings: List[str]):
     """
     Update the leaderboard based on the selected substrings.
     """
-    filtered_dataframe = filter_columns_by_substrings(dataframe, selected_substrings)
+    filtered_dataframe = filter_columns_by_substrings(dataframe, selected_task_substrings, selected_model_substrings)
+    if len(selected_task_substrings) >= 2 or len(selected_task_substrings) == 0:
+        if len(selected_model_substrings) >= 2 or len(selected_model_substrings) == 0:
+            show_average = True
+        else:
+            show_average = False
+    else:    
+        show_average = False
+
+    if show_average:
+        means = filtered_dataframe.replace("-", float("nan")).mean(axis=1, skipna=False)
+        filtered_dataframe["Average"] = means.round(2)
+        filtered_dataframe = filtered_dataframe.sort_values(by=["Average"], ascending=False, na_position='last')
+        filtered_dataframe = filtered_dataframe.replace(float("nan"), "-")
+
+
+    # if show_average:
+    #     print([row for index, row in filtered_dataframe.iterrows()])
+    #     filtered_dataframe["Average"] = [round(np.mean(row.values()), 2) if "-" not in row.values() else "-" for index, row in filtered_dataframe.iterrows()]
+    #     # Sort by Average score descending
+    #     if 'Average' in dataframe.columns:
+    #         # Convert '-' to NaN for sorting purposes
+    #         df['Average'] = pd.to_numeric(['Average'], errors='coerce')
+    #         df = df.sort_values(by=['Average'], ascending=True, na_position='last')
+    #         # Convert NaN back to '-'
+    #         df['Average'] = df['Average'].fillna('-')
+
     return filtered_dataframe
 
 demo = gr.Blocks(css=custom_css)
@@ -533,17 +599,33 @@ with demo:
                     You can combine filters to see specific task-model combinations.
                     """)
                     # CheckboxGroup for selecting substrings
-                    substring_checkbox = gr.CheckboxGroup(
-                        choices=PRESET_SUBSTRINGS,
-                        label="Filter results:",
-                        value=PRESET_SUBSTRINGS,  # Default to all substrings selected
+                    # substring_checkbox = gr.CheckboxGroup(
+                    #     choices=PRESET_SUBSTRINGS,
+                    #     label="Filter results:",
+                    #     value=PRESET_SUBSTRINGS,  # Default to all substrings selected
+                    # )
+                    task_substring_checkbox = gr.CheckboxGroup(
+                        choices=TASK_SUBSTRINGS,
+                        label="View tasks:",
+                        value=TASK_SUBSTRINGS,  # Default to all substrings selected
+                    )
+                    model_substring_checkbox = gr.CheckboxGroup(
+                        choices = MODEL_SUBSTRINGS,
+                        label = "View models:",
+                        value = MODEL_SUBSTRINGS
                     )
+
                     leaderboard, data = init_leaderboard_mib_subgraph(LEADERBOARD_DF_MIB_SUBGRAPH_FPL, "Subgraph")
                     original_leaderboard = gr.State(value=data)
                     # Update the leaderboard when the user selects/deselects substrings
-                    substring_checkbox.change(
+                    task_substring_checkbox.change(
                         fn=update_leaderboard,
-                        inputs=[original_leaderboard, substring_checkbox],
+                        inputs=[original_leaderboard, task_substring_checkbox, model_substring_checkbox],
+                        outputs=leaderboard
+                    )
+                    model_substring_checkbox.change(
+                        fn=update_leaderboard,
+                        inputs=[original_leaderboard, task_substring_checkbox, model_substring_checkbox],
                         outputs=leaderboard
                     )
                     print(f"Leaderboard is {leaderboard}")
@@ -555,17 +637,33 @@ with demo:
                     You can combine filters to see specific task-model combinations.
                     """)
                     # CheckboxGroup for selecting substrings
-                    substring_checkbox = gr.CheckboxGroup(
-                        choices=PRESET_SUBSTRINGS,
-                        label="Filter results:",
-                        value=PRESET_SUBSTRINGS,  # Default to all substrings selected
+                    # substring_checkbox = gr.CheckboxGroup(
+                    #     choices=PRESET_SUBSTRINGS,
+                    #     label="Filter results:",
+                    #     value=PRESET_SUBSTRINGS,  # Default to all substrings selected
+                    # )
+                    task_substring_checkbox = gr.CheckboxGroup(
+                        choices=TASK_SUBSTRINGS,
+                        label="View tasks:",
+                        value=TASK_SUBSTRINGS,  # Default to all substrings selected
+                    )
+                    model_substring_checkbox = gr.CheckboxGroup(
+                        choices = MODEL_SUBSTRINGS,
+                        label = "View models:",
+                        value = MODEL_SUBSTRINGS
                     )
+
                     leaderboard, data = init_leaderboard_mib_subgraph(LEADERBOARD_DF_MIB_SUBGRAPH_FEQ, "Subgraph")
                     original_leaderboard = gr.State(value=data)
                     # Update the leaderboard when the user selects/deselects substrings
-                    substring_checkbox.change(
+                    task_substring_checkbox.change(
                         fn=update_leaderboard,
-                        inputs=[original_leaderboard, substring_checkbox],
+                        inputs=[original_leaderboard, task_substring_checkbox, model_substring_checkbox],
+                        outputs=leaderboard
+                    )
+                    model_substring_checkbox.change(
+                        fn=update_leaderboard,
+                        inputs=[original_leaderboard, task_substring_checkbox, model_substring_checkbox],
                         outputs=leaderboard
                     )
                     print(f"Leaderboard is {leaderboard}")
@@ -584,19 +682,35 @@ with demo:
                     Use the dropdown menus below to filter results by specific tasks or models. 
                     You can combine filters to see specific task-model combinations.
                     """)
-                    substring_checkbox = gr.CheckboxGroup(
-                        choices=PRESET_SUBSTRINGS,
-                        label="Filter results:",
-                        value=PRESET_SUBSTRINGS,  # Default to all substrings selected
+                    # substring_checkbox = gr.CheckboxGroup(
+                    #     choices=PRESET_SUBSTRINGS,
+                    #     label="Filter results:",
+                    #     value=PRESET_SUBSTRINGS,  # Default to all substrings selected
+                    # )
+                    task_substring_checkbox = gr.CheckboxGroup(
+                        choices=TASK_SUBSTRINGS,
+                        label="View tasks:",
+                        value=TASK_SUBSTRINGS,  # Default to all substrings selected
+                    )
+                    model_substring_checkbox = gr.CheckboxGroup(
+                        choices = MODEL_SUBSTRINGS,
+                        label = "View models:",
+                        value = MODEL_SUBSTRINGS
                     )
+                    
                     leaderboard_aggregated, data = init_leaderboard_mib_causalgraph(
                         LEADERBOARD_DF_MIB_CAUSALGRAPH_AGGREGATED, 
                         "Causal Graph"
                     )
                     original_leaderboard = gr.State(value=data)
-                    substring_checkbox.change(
+                    task_substring_checkbox.change(
+                        fn=update_leaderboard,
+                        inputs=[original_leaderboard, task_substring_checkbox, model_substring_checkbox],
+                        outputs=leaderboard_aggregated
+                    )
+                    model_substring_checkbox.change(
                         fn=update_leaderboard,
-                        inputs=[original_leaderboard, substring_checkbox],
+                        inputs=[original_leaderboard, task_substring_checkbox, model_substring_checkbox],
                         outputs=leaderboard_aggregated
                     )
                 with gr.TabItem("Intervention Averaged", id=2):