caulsal-track debug

app.py

@@ -23,10 +23,8 @@ from src.display.utils import (
     BENCHMARK_COLS,
     BENCHMARK_COLS_MULTIMODAL,
     BENCHMARK_COLS_MIB_SUBGRAPH,
-    BENCHMARK_COLS_MIB_CAUSALGRAPH,
     COLS,
     COLS_MIB_SUBGRAPH,
-    COLS_MIB_CAUSALGRAPH,
     COLS_MULTIMODAL,
     EVAL_COLS,
     EVAL_TYPES,
@@ -281,9 +279,7 @@ LEADERBOARD_DF_MIB_SUBGRAPH_FEQ = get_leaderboard_df_mib_subgraph(EVAL_RESULTS_M
 # In app.py, modify the LEADERBOARD initialization
 LEADERBOARD_DF_MIB_CAUSALGRAPH_DETAILED, LEADERBOARD_DF_MIB_CAUSALGRAPH_AGGREGATED, LEADERBOARD_DF_MIB_CAUSALGRAPH_AVERAGED = get_leaderboard_df_mib_causalgraph(
     EVAL_RESULTS_MIB_CAUSALGRAPH_PATH, 
-    EVAL_REQUESTS_PATH, 
-    COLS_MIB_CAUSALGRAPH, 
-    BENCHMARK_COLS_MIB_CAUSALGRAPH
+    EVAL_REQUESTS_PATH
 )
 
 
@@ -300,95 +296,6 @@ LEADERBOARD_DF_MIB_CAUSALGRAPH_DETAILED, LEADERBOARD_DF_MIB_CAUSALGRAPH_AGGREGAT
 
 
 
-# def init_leaderboard_mib_subgraph(dataframe, track):
-#     # print(f"init_leaderboard_mib: dataframe head before loc is {dataframe.head()}\n")
-    
-#     if dataframe is None or dataframe.empty:
-#         raise ValueError("Leaderboard DataFrame is empty or None.")
-
-#     # filter for correct track
-#     # dataframe = dataframe.loc[dataframe["Track"] == track]
-
-#     # print(f"init_leaderboard_mib: dataframe head after loc is {dataframe.head()}\n")
-    
-#     return Leaderboard(
-#         value=dataframe,
-#         datatype=[c.type for c in fields(AutoEvalColumn_mib_subgraph)],
-#         select_columns=SelectColumns(
-#             default_selection=[c.name for c in fields(AutoEvalColumn_mib_subgraph) if c.displayed_by_default],
-#             cant_deselect=[c.name for c in fields(AutoEvalColumn_mib_subgraph) if c.never_hidden],
-#             label="Select Columns to Display:",
-#         ),
-#         search_columns=["Method"],  # Changed from AutoEvalColumn_mib_subgraph.model.name to "Method"
-#         hide_columns=[c.name for c in fields(AutoEvalColumn_mib_subgraph) if c.hidden],
-#         bool_checkboxgroup_label="Hide models",
-#         interactive=False,
-#     )
-
-
-
-
-
-# def init_leaderboard_mib_subgraph(dataframe, track):
-#     """Initialize the subgraph leaderboard with grouped column selection by benchmark."""
-#     if dataframe is None or dataframe.empty:
-#         raise ValueError("Leaderboard DataFrame is empty or None.")
-
-#     print("\nDebugging DataFrame columns:", dataframe.columns.tolist())
-    
-#     # Create groups of columns by benchmark
-#     benchmark_groups = []
-    
-#     # For each benchmark in our TasksMib_Subgraph enum...
-#     for task in TasksMib_Subgraph:
-#         benchmark = task.value.benchmark
-#         # Get all valid columns for this benchmark's models
-#         benchmark_cols = [
-#             f"{benchmark}_{model}"
-#             for model in task.value.models
-#             if f"{benchmark}_{model}" in dataframe.columns
-#         ]
-#         if benchmark_cols:  # Only add if we have valid columns
-#             benchmark_groups.append(benchmark_cols)
-#             print(f"\nBenchmark group for {benchmark}:", benchmark_cols)
-
-#     # Create model groups as well
-#     model_groups = []
-#     all_models = list(set(model for task in TasksMib_Subgraph for model in task.value.models))
-    
-#     # For each unique model...
-#     for model in all_models:
-#         # Get all valid columns for this model across benchmarks
-#         model_cols = [
-#             f"{task.value.benchmark}_{model}"
-#             for task in TasksMib_Subgraph
-#             if model in task.value.models
-#             and f"{task.value.benchmark}_{model}" in dataframe.columns
-#         ]
-#         if model_cols:  # Only add if we have valid columns
-#             model_groups.append(model_cols)
-#             print(f"\nModel group for {model}:", model_cols)
-
-#     # Combine all groups
-#     all_groups = benchmark_groups + model_groups
-    
-#     # Flatten groups for default selection (show everything initially)
-#     all_columns = [col for group in all_groups for col in group]
-#     print("\nAll available columns:", all_columns)
-
-#     return Leaderboard(
-#         value=dataframe,
-#         datatype=[c.type for c in fields(AutoEvalColumn_mib_subgraph)],
-#         select_columns=SelectColumns(
-#             default_selection=all_columns,  # Show all columns initially
-#             label="Select Results:"
-#         ),
-#         search_columns=["Method"],
-#         hide_columns=[],
-#         interactive=False,
-#     )
-
-
 def init_leaderboard_mib_subgraph(dataframe, track):
     """Initialize the subgraph leaderboard with display names for better readability."""
     if dataframe is None or dataframe.empty:
@@ -478,189 +385,6 @@ def init_leaderboard_mib_subgraph(dataframe, track):
 
 
 
-    # # Complete column groups for both benchmarks and models
-    # # Define keywords for filtering
-    # benchmark_keywords = ["ioi", "mcqa", "arithmetic_addition", "arithmetic_subtraction", "arc_easy", "arc_challenge"]
-    # model_keywords = ["qwen2_5", "gpt2", "gemma2", "llama3"]
-
-    # # Optional: Define display names
-    # mappings = {
-    #     "ioi_llama3": "IOI (LLaMA-3)",
-    #     "ioi_qwen2_5": "IOI (Qwen-2.5)",
-    #     "ioi_gpt2": "IOI (GPT-2)",
-    #     "ioi_gemma2": "IOI (Gemma-2)",
-    #     "mcqa_llama3": "MCQA (LLaMA-3)",
-    #     "mcqa_qwen2_5": "MCQA (Qwen-2.5)",
-    #     "mcqa_gemma2": "MCQA (Gemma-2)",
-    #     "arithmetic_addition_llama3": "Arithmetic Addition (LLaMA-3)",
-    #     "arithmetic_subtraction_llama3": "Arithmetic Subtraction (LLaMA-3)",
-    #     "arc_easy_llama3": "ARC Easy (LLaMA-3)",
-    #     "arc_easy_gemma2": "ARC Easy (Gemma-2)",
-    #     "arc_challenge_llama3": "ARC Challenge (LLaMA-3)",
-    #     "eval_name": "Evaluation Name",
-    #     "Method": "Method",
-    #     "Average": "Average Score"
-    # }
-    # # mappings = {}
-
-    # # Create SmartSelectColumns instance
-    # smart_columns = SmartSelectColumns(
-    #     benchmark_keywords=benchmark_keywords,
-    #     model_keywords=model_keywords,
-    #     column_mapping=mappings,
-    #     initial_selected=["Method", "Average"]
-    # )
-
-    # print("\nDebugging DataFrame columns:", renamed_df.columns.tolist())
-
-    # # Create Leaderboard
-    # leaderboard = Leaderboard(
-    #     value=renamed_df,
-    #     datatype=[c.type for c in fields(AutoEvalColumn_mib_subgraph)],
-    #     select_columns=smart_columns,
-    #     search_columns=["Method"],
-    #     hide_columns=[],
-    #     interactive=False
-    # )
-    # print(f"Successfully created leaderboard.")
-    # return leaderboard
-
-    # print("\nDebugging DataFrame columns:", dataframe.columns.tolist())
-
-    # # Define simple keywords for filtering
-    # benchmark_keywords = ["ioi", "mcqa", "arithmetic", "arc"]
-    # model_keywords = ["qwen2_5", "gpt2", "gemma2", "llama3"]
-
-    # # Create SmartSelectColumns instance with exact same parameters as working version
-    # smart_columns = SmartSelectColumns(
-    #     benchmark_keywords=benchmark_keywords,
-    #     model_keywords=model_keywords,
-    #     initial_selected=["Method", "Average"],
-    #     allow=True,
-    #     label=None,
-    #     show_label=True,
-    #     info=None
-    # )
-
-    # try:
-    #     print("\nCreating leaderboard...")
-    #     # Get groups before creating leaderboard
-    #     smart_columns.get_filtered_groups(dataframe.columns)
-        
-    #     leaderboard = Leaderboard(
-    #         value=dataframe,
-    #         datatype=[c.type for c in fields(AutoEvalColumn_mib_subgraph)],
-    #         select_columns=smart_columns,
-    #         search_columns=["Method"],
-    #         hide_columns=[],
-    #         interactive=False
-    #     )
-    #     print("Leaderboard created successfully")
-    #     return leaderboard
-        
-    # except Exception as e:
-    #     print("Error creating leaderboard:", str(e))
-    #     raise
-
-
-
-
-
-
-# def init_leaderboard_mib_subgraph(dataframe, track):
-#     """Initialize the subgraph leaderboard with group-based column selection."""
-#     if dataframe is None or dataframe.empty:
-#         raise ValueError("Leaderboard DataFrame is empty or None.")
-
-#     print("\nDebugging DataFrame columns:", dataframe.columns.tolist())
-    
-#     # Create selection mapping for benchmark groups
-#     selection_mapping = {}
-    
-#     # Create benchmark groups with descriptive names
-#     for task in TasksMib_Subgraph:
-#         benchmark = task.value.benchmark
-#         # Get all columns for this benchmark's models
-#         benchmark_cols = [
-#             f"{benchmark}_{model}"
-#             for model in task.value.models
-#             if f"{benchmark}_{model}" in dataframe.columns
-#         ]
-#         if benchmark_cols:
-#             # Use a descriptive group name as the key
-#             group_name = f"Benchmark: {benchmark.upper()}"
-#             selection_mapping[group_name] = benchmark_cols
-#             print(f"\n{group_name} maps to:", benchmark_cols)
-
-#     # Create model groups with descriptive names
-#     all_models = list(set(model for task in TasksMib_Subgraph for model in task.value.models))
-#     for model in all_models:
-#         # Get all columns for this model across benchmarks
-#         model_cols = [
-#             f"{task.value.benchmark}_{model}"
-#             for task in TasksMib_Subgraph
-#             if model in task.value.models
-#             and f"{task.value.benchmark}_{model}" in dataframe.columns
-#         ]
-#         if model_cols:
-#             # Use a descriptive group name as the key
-#             group_name = f"Model: {model}"
-#             selection_mapping[group_name] = model_cols
-#             print(f"\n{group_name} maps to:", model_cols)
-
-#     # The selection options are the group names
-#     selection_options = list(selection_mapping.keys())
-#     print("\nSelection options:", selection_options)
-
-#     return Leaderboard(
-#         value=dataframe,
-#         datatype=[c.type for c in fields(AutoEvalColumn_mib_subgraph)],
-#         select_columns=SelectColumns(
-#             default_selection=selection_options,  # Show all groups by default
-#             label="Select Benchmark or Model Groups:"
-#         ),
-#         search_columns=["Method"],
-#         hide_columns=[],
-#         interactive=False,
-#     )
-
-
-
-
-
-
-
-
-# def init_leaderboard_mib_causalgraph(dataframe, track):
-#     # print("Debugging column issues:")
-#     # print("\nActual DataFrame columns:")
-#     # print(dataframe.columns.tolist())
-    
-#     # print("\nExpected columns for Leaderboard:")
-#     expected_cols = [c.name for c in fields(AutoEvalColumn_mib_causalgraph)]
-#     # print(expected_cols)
-    
-#     # print("\nMissing columns:")
-#     missing_cols = [col for col in expected_cols if col not in dataframe.columns]
-#     # print(missing_cols)
-    
-#     # print("\nSample of DataFrame content:")
-#     # print(dataframe.head().to_string())
-
-#     return Leaderboard(
-#         value=dataframe,
-#         datatype=[c.type for c in fields(AutoEvalColumn_mib_causalgraph)],
-#         select_columns=SelectColumns(
-#             default_selection=[c.name for c in fields(AutoEvalColumn_mib_causalgraph) if c.displayed_by_default],
-#             cant_deselect=[c.name for c in fields(AutoEvalColumn_mib_causalgraph) if c.never_hidden],
-#             label="Select Columns to Display:",
-#         ),
-#         search_columns=["Method"],
-#         hide_columns=[c.name for c in fields(AutoEvalColumn_mib_causalgraph) if c.hidden],
-#         bool_checkboxgroup_label="Hide models",
-#         interactive=False,
-#     )
-
 def init_leaderboard_mib_causalgraph(dataframe, track):
     # print("Debugging column issues:")
     # print("\nActual DataFrame columns:")

caulsal_metric.py

@@ -135,6 +135,11 @@ def create_summary_dataframe(json_files: List[Dict[str, Any]]) -> pd.DataFrame:
     
     return df
 
+
+# averaged_cf = average_counterfactuals(json_files)
+# layer_averaged = find_layer_averages(averaged_cf)
+# detailed_df = create_summary_dataframe(layer_averaged)
+
 def aggregate_methods(df: pd.DataFrame) -> pd.DataFrame:
     """
     Aggregates rows with the same base method name by taking the max value for each column.

src/about.py

@@ -61,33 +61,27 @@ class TasksMib_Subgraph(Enum):
         return sorted(list(models))
 
 
+
+
 # @dataclass 
 # class TaskMIB_Causalgraph:
-#     benchmark: str      # MCQA
-#     models: list[str]   # List of all models
-#     layers: list[str]   # 0-31
-#     col_name: str       # display name in leaderboard
-#     interventions: list[str]  # output_token, output_location
-#     counterfactuals: list[str]  # symbol_counterfactual, etc.
-#     metrics: list[str]  # score
+#     benchmark: str      
+#     models: list[str]   
+#     layers: dict[str, list[str]]  # Different layers for each model
+#     col_name: str      
+#     interventions: list[str]  
+#     counterfactuals: list[str]  
+#     metrics: list[str]  
 
-# class TasksMib_Causalgraph(Enum):
-#     task0 = TaskMIB_Causalgraph(
-#         "MCQA", 
-#         ["LlamaForCausalLM", "Qwen2ForCausalLM", "Gemma2ForCausalLM"],  # Updated model list
-#         [str(i) for i in range(32)],  # 0-31 layers
-#         "mcqa",
-#         ["output_token", "output_location"],
-#         ["symbol_counterfactual", "randomLetter_counterfactual", 
-#          "answerPosition_counterfactual", "answerPosition_symbol_counterfactual"],
-#         ["score"]
-#     )
 
 # class TasksMib_Causalgraph(Enum):
-#     task0 = TaskMIB_Causalgraph(
-#         "MCQA", 
-#         ["Qwen2ForCausalLM", "Gemma2ForCausalLM", "LlamaForCausalLM"],  # Match exact model names with correct casing
-#         [str(i) for i in range(32)],  
+#     task0 = TaskMIB_Causalgraph("MCQA", 
+#         ["qwen2forcausallm", "gemma2forcausallm", "llamaforcausallm"],
+#         {
+#             "qwen2forcausallm": [str(i) for i in range(24)],    # 0-23
+#             "gemma2forcausallm": [str(i) for i in range(26)],   # 0-25
+#             "llamaforcausallm": [str(i) for i in range(32)]     # 0-31
+#         },
 #         "mcqa",
 #         ["output_token", "output_location"],
 #         ["randomLetter_counterfactual", "answerPosition_counterfactual", 
@@ -95,58 +89,32 @@ class TasksMib_Subgraph(Enum):
 #         ["score"]
 #     )
 
-# class TasksMib_Causalgraph(Enum):
-#     task0 = TaskMIB_Causalgraph(
-#         "MCQA", 
-#         ["qwen2forcausallm", "gemma2forcausallm", "llamaforcausallm"],  # Use lowercase consistently
-#         [str(i) for i in range(32)],
-#         "mcqa",
-#         ["output_token", "output_location"],
-#         ["randomLetter_counterfactual", "answerPosition_counterfactual", 
-#          "answerPosition_randomLetter_counterfactual"],
-#         ["score"]
-#     )
 
-@dataclass 
+@dataclass
 class TaskMIB_Causalgraph:
-    benchmark: str      
-    models: list[str]   
-    layers: dict[str, list[str]]  # Different layers for each model
-    col_name: str      
-    interventions: list[str]  
-    counterfactuals: list[str]  
-    metrics: list[str]  
+    benchmark: str      # task name in json (ioi/arithmetic)
+    models: list[str]   # list of models to show as sub-columns
+    col_name: str       # display name in leaderboard
+    metrics: list[str]  # metrics to store (average_score)
 
-# class TasksMib_Causalgraph(Enum):
-#     task0 = TaskMIB_Causalgraph(
-#         "MCQA", 
-#         ["Qwen2ForCausalLM", "Gemma2ForCausalLM", "LlamaForCausalLM"],
-#         {
-#             "Qwen2ForCausalLM": [str(i) for i in range(24)],    # 0-23
-#             "Gemma2ForCausalLM": [str(i) for i in range(26)],   # 0-25
-#             "LlamaForCausalLM": [str(i) for i in range(32)]     # 0-31
-#         },
-#         "mcqa",
-#         ["output_token", "output_location"],
-#         ["randomLetter_counterfactual", "answerPosition_counterfactual", 
-#          "answerPosition_randomLetter_counterfactual"],
-#         ["score"]
-#     )
 class TasksMib_Causalgraph(Enum):
-    task0 = TaskMIB_Causalgraph(
-        "MCQA", 
-        ["qwen2forcausallm", "gemma2forcausallm", "llamaforcausallm"],  # Use lowercase names to match actual columns
-        {
-            "qwen2forcausallm": [str(i) for i in range(24)],    # 0-23
-            "gemma2forcausallm": [str(i) for i in range(26)],   # 0-25
-            "llamaforcausallm": [str(i) for i in range(32)]     # 0-31
-        },
-        "mcqa",
-        ["output_token", "output_location"],
-        ["randomLetter_counterfactual", "answerPosition_counterfactual", 
-         "answerPosition_randomLetter_counterfactual"],
-        ["score"]
-    )
+    task0 = TaskMIB_Subgraph("mcqa", ["Qwen2ForCausalLM", "Gemma2ForCausalLM", "LlamaForCausalLM"], "MCQA", ["average_score"])
+
+    @classmethod
+    def get_all_tasks(cls):
+        """Returns a list of all task benchmarks"""
+        return [task.value.benchmark for task in cls]
+    
+    @classmethod
+    def get_all_models(cls):
+        """Returns a list of all unique models across all tasks"""
+        models = set()
+        for task in cls:
+            models.update(task.value.models)
+        return sorted(list(models))
+
+
+
 
 
 NUM_FEWSHOT = 0 # Change with your few shot

src/display/utils.py

@@ -58,64 +58,6 @@ AutoEvalColumnMultimodal = make_dataclass("AutoEvalColumnMultimodal", auto_eval_
 
 
 
-##############################################################################################################
-# Version 1
-
-# auto_eval_column_dict_mib_subgraph = []
-
-# # Method name column
-# auto_eval_column_dict_mib_subgraph.append(["method", ColumnContent, ColumnContent("Method", "markdown", True, never_hidden=True)])
-
-# # For each task and model combination
-# for task in TasksMib_Subgraph:
-#     for model in task.value.models:
-#         col_name = f"{task.value.benchmark}_{model}"  # ioi_gpt2, mcqa_qwen2.5, etc.
-#         auto_eval_column_dict_mib_subgraph.append([
-#             col_name, 
-#             ColumnContent, 
-#             ColumnContent(col_name, "number", True)
-#         ])
-
-# # Average column
-# auto_eval_column_dict_mib_subgraph.append(["average", ColumnContent, ColumnContent("Average", "number", True)])
-
-
-# ##############################################################################################################
-# # Version 2
-# auto_eval_column_dict_mib_subgraph = []
-
-# # Method name column
-# auto_eval_column_dict_mib_subgraph.append(["method", ColumnContent, ColumnContent("Method", "markdown", True, never_hidden=True)])
-
-# # Add task filter column
-# task_values = list(set(task.value.benchmark for task in TasksMib_Subgraph))
-# auto_eval_column_dict_mib_subgraph.append(
-#     ["task_filter", ColumnContent, ColumnContent("Task", "str", True, never_hidden=True)]
-# )
-
-# # Add model filter column
-# model_values = list(set(
-#     model 
-#     for task in TasksMib_Subgraph 
-#     for model in task.value.models
-# ))
-# auto_eval_column_dict_mib_subgraph.append(
-#     ["model_filter", ColumnContent, ColumnContent("Model", "str", True, never_hidden=True)]
-# )
-
-# # For each task and model combination
-# for task in TasksMib_Subgraph:
-#     for model in task.value.models:
-#         col_name = f"{task.value.benchmark}_{model}"
-#         auto_eval_column_dict_mib_subgraph.append([
-#             col_name, 
-#             ColumnContent, 
-#             ColumnContent(col_name, "number", True)
-#         ])
-
-# # Average column
-# auto_eval_column_dict_mib_subgraph.append(["average", ColumnContent, ColumnContent("Average", "number", True)])
-
 
 ##############################################################################################################
 # Version 3
@@ -185,133 +127,6 @@ BENCHMARK_COLS_MIB_CAUSALGRAPH = []
 
 
 
-
-# # Initialize the MIB causal graph columns
-# auto_eval_column_dict_mib_causalgraph = []
-
-# # Method name column
-# auto_eval_column_dict_mib_causalgraph.append(["method", ColumnContent, ColumnContent("Method", "markdown", True, never_hidden=True)])
-
-# # For each model-task-intervention combination
-# for task in TasksMib_Causalgraph:
-#     for model in task.value.models:
-#         for intervention in task.value.interventions:
-#             col_name = f"{model}_{task.value.benchmark}_{intervention}".lower()
-#             auto_eval_column_dict_mib_causalgraph.append([
-#                 col_name, 
-#                 ColumnContent, 
-#                 ColumnContent(col_name, "number", True)
-#             ])
-
-# auto_eval_column_dict_mib_causalgraph = []
-
-# # Method name column
-# auto_eval_column_dict_mib_causalgraph.append(["method", ColumnContent, ColumnContent("Method", "markdown", True, never_hidden=True)])
-
-# # For each model-task-intervention combination
-# for task in TasksMib_Causalgraph:
-#     for model in task.value.models:
-#         model_name = model.lower()  # Convert model name to lowercase
-#         for layer in task.value.layers:
-#             for intervention in task.value.interventions:
-#                 for counterfactual in task.value.counterfactuals:
-#                     # Include model name in the column name
-#                     col_name = f"{model_name}_layer{layer}_{intervention}_{counterfactual}"
-#                     field_name = col_name.lower()
-#                     auto_eval_column_dict_mib_causalgraph.append([
-#                         field_name, 
-#                         ColumnContent, 
-#                         ColumnContent(col_name, "number", True)
-#                     ])
-
-# # In utils.py, modify auto_eval_column_dict_mib_causalgraph:
-# auto_eval_column_dict_mib_causalgraph = []
-
-# # Method name column
-# auto_eval_column_dict_mib_causalgraph.append(["method", ColumnContent, ColumnContent("Method", "markdown", True, never_hidden=True)])
-
-# # For each model-task-intervention-counterfactual combination
-# for task in TasksMib_Causalgraph:
-#     for model in ["qwen2forcausallm", "gemma2forcausallm", "llamaforcausallm"]:  # exact model names
-#         for layer in task.value.layers:
-#             for intervention in task.value.interventions:
-#                 for counterfactual in task.value.counterfactuals:
-#                     # Match the exact format from the data
-#                     col_name = f"{model}_layer{layer}_{intervention}_{counterfactual}".lower()
-#                     auto_eval_column_dict_mib_causalgraph.append([
-#                         col_name, 
-#                         ColumnContent, 
-#                         ColumnContent(col_name, "number", True)
-#                     ])
-
-
-
-
-# auto_eval_column_dict_mib_causalgraph = []
-
-# # Method name column
-# auto_eval_column_dict_mib_causalgraph.append(["method", ColumnContent, ColumnContent("Method", "markdown", True, never_hidden=True)])
-
-# # Add eval_name column
-# auto_eval_column_dict_mib_causalgraph.append(["eval_name", ColumnContent, ColumnContent("eval_name", "str", True)])
-
-# # For each model-task-intervention-counterfactual combination
-# for task in TasksMib_Causalgraph:
-#     for model in task.value.models:  # Use exact model names with correct casing
-#         model_name = model  # Don't convert to lowercase
-#         for layer in task.value.layers:
-#             for intervention in task.value.interventions:
-#                 for counterfactual in task.value.counterfactuals:
-#                     # Match exact format from the actual data
-#                     col_name = f"{model_name}_layer{layer}_{intervention}_{counterfactual}"
-#                     # Use the exact column name as both the field name and display name
-#                     auto_eval_column_dict_mib_causalgraph.append([
-#                         col_name, 
-#                         ColumnContent, 
-#                         ColumnContent(col_name, "number", True)
-#                     ])
-
-
-
-# auto_eval_column_dict_mib_causalgraph = []
-
-# # Method name column
-# auto_eval_column_dict_mib_causalgraph.append(["method", ColumnContent, ColumnContent("Method", "markdown", True, never_hidden=True)])
-# auto_eval_column_dict_mib_causalgraph.append(["eval_name", ColumnContent, ColumnContent("eval_name", "str", True)])
-
-# # For each model-task-intervention-counterfactual combination
-# for task in TasksMib_Causalgraph:
-#     for model in task.value.models:
-#         for layer in task.value.layers[model]:  # Use model-specific layers
-#             for intervention in task.value.interventions:
-#                 for counterfactual in task.value.counterfactuals:
-#                     col_name = f"{model}_layer{layer}_{intervention}_{counterfactual}"
-#                     auto_eval_column_dict_mib_causalgraph.append([
-#                         col_name, 
-#                         ColumnContent, 
-#                         ColumnContent(col_name, "number", True)
-#                     ])
-
-# auto_eval_column_dict_mib_causalgraph = []
-
-# # Method name column
-# auto_eval_column_dict_mib_causalgraph.append(["method", ColumnContent, ColumnContent("Method", "markdown", True, never_hidden=True)])
-# auto_eval_column_dict_mib_causalgraph.append(["eval_name", ColumnContent, ColumnContent("eval_name", "str", True)])
-
-# # For each model-task-intervention-counterfactual combination
-# for task in TasksMib_Causalgraph:
-#     for model in task.value.models:  # model will already be lowercase
-#         for layer in task.value.layers[model]:
-#             for intervention in task.value.interventions:
-#                 for counterfactual in task.value.counterfactuals:
-#                     # Use exactly the same format as in DataFrame
-#                     col_name = f"{model}_layer{layer}_{intervention}_{counterfactual}"
-#                     auto_eval_column_dict_mib_causalgraph.append([
-#                         col_name, 
-#                         ColumnContent, 
-#                         ColumnContent(col_name, "number", True)
-#                     ])
-
 auto_eval_column_dict_mib_causalgraph = []
 
 # Only include Method column as required
@@ -320,15 +135,17 @@ auto_eval_column_dict_mib_causalgraph.append(["method", ColumnContent, ColumnCon
 # For each model-task-intervention-counterfactual combination
 for task in TasksMib_Causalgraph:
     for model in task.value.models:  # model will be lowercase
-        for layer in task.value.layers[model]:
-            for intervention in task.value.interventions:
-                for counterfactual in task.value.counterfactuals:
-                    col_name = f"{model}_layer{layer}_{intervention}_{counterfactual}"
-                    auto_eval_column_dict_mib_causalgraph.append([
-                        col_name, 
-                        ColumnContent, 
-                        ColumnContent(col_name, "number", True)
-                    ])
+        col_name = f"{task.value.benchmark}_{model}"
+        auto_eval_column_dict_mib_causalgraph.append([
+            col_name, 
+            ColumnContent, 
+            ColumnContent(col_name, "number", True)
+        ])
+
+# Add the Average column
+auto_eval_column_dict_mib_causalgraph.append(
+    ["average_score", ColumnContent, ColumnContent("Average", "number", True)]
+)
 
 # Create the dataclass
 AutoEvalColumn_mib_causalgraph = make_dataclass(
@@ -337,14 +154,20 @@ AutoEvalColumn_mib_causalgraph = make_dataclass(
     frozen=True
 )
 
-# Column selection for display 
-COLS_MIB_CAUSALGRAPH = [c.name for c in fields(AutoEvalColumn_mib_causalgraph) if not c.hidden]
 
 
-BENCHMARK_COLS_MIB_CAUSALGRAPH = [f"{model}_{task.value.benchmark}_{intervention}".lower()
-                                 for task in TasksMib_Causalgraph
-                                 for model in task.value.models
-                                 for intervention in task.value.interventions]
+
+
+
+
+# # Column selection for display 
+# COLS_MIB_CAUSALGRAPH = [c.name for c in fields(AutoEvalColumn_mib_causalgraph) if not c.hidden]
+
+
+# BENCHMARK_COLS_MIB_CAUSALGRAPH = [f"{model}_{task.value.benchmark}_{intervention}".lower()
+#                                  for task in TasksMib_Causalgraph
+#                                  for model in task.value.models
+#                                  for intervention in task.value.interventions]
 
 
 

src/leaderboard/read_evals.py

@@ -12,8 +12,10 @@ from src.display.utils import AutoEvalColumn, AutoEvalColumnMultimodal, Tasks, T
 from src.submission.check_validity import is_model_on_hub
 from src.about import TasksMib_Subgraph
 
-from typing import List, Dict
+from typing import List, Dict, Any
 from collections import defaultdict
+import pandas as pd
+
 
 
 def compute_area(edge_counts, faithfulnesses, log_scale=True):
@@ -65,21 +67,8 @@ class EvalResult_MIB_SUBGRAPH:
         for model_result in data.get("results", []):
             model_id = model_result.get("model_id", "")
             
-            # if "/" in model_id:
-            #     org = model_id.split("/")[0]
-            #     if org == "meta-llama":
-            #         model_name = "llama3"
-            #     elif org == "Qwen":
-            #         model_name = "qwen2_5"
-            #     elif "gpt" in model_id.lower():
-            #         model_name = "gpt2"
-            #     elif org == "google":
-            #         model_name = "gemma2"
-            # else:
-            #     model_name = model_id.replace(".", "_")
             model_name = model_id.replace(".", "_")
 
-
             # Keep exact scores structure from JSON
             scores = model_result.get("scores", {})
             
@@ -108,16 +97,7 @@ class EvalResult_MIB_SUBGRAPH:
     
         # Initialize all possible columns with '-'
         expected_models = TasksMib_Subgraph.get_all_models()
-        expected_tasks = TasksMib_Subgraph.get_all_tasks()
-        # for task in expected_tasks:
-        #     for model in task.value.models:
-        #         # if model == "gpt2" and task != "ioi":
-        #         #     continue
-        #         # if model == "qwen2_5" and task.startswith(("arithmetic", "arc")):
-        #         #     continue
-        #         # if model == "gemma2" and (task.startswith("arithmetic") or task == "arc_challenge"):
-        #         #     continue
-        #         data_dict[f"{task}_{model}"] = '-'
+        # expected_tasks = TasksMib_Subgraph.get_all_tasks()
 
         for task in TasksMib_Subgraph:
             for model in task.value.models:
@@ -145,23 +125,6 @@ class EvalResult_MIB_SUBGRAPH:
                 data_dict[col_name] = round(score, 2)
                 all_scores.append(score)
     
-        # All entries must be present for average
-        # required_entries = [
-        #     data_dict['ioi_llama3'] != '-',
-        #     data_dict['ioi_qwen2_5'] != '-',
-        #     data_dict['ioi_gpt2'] != '-',
-        #     data_dict['ioi_gemma2'] != '-',
-        #     data_dict['mcqa_llama3'] != '-',
-        #     data_dict['mcqa_qwen2_5'] != '-',
-        #     data_dict['mcqa_gemma2'] != '-',
-        #     data_dict['arithmetic_addition_llama3'] != '-',
-        #     data_dict['arithmetic_subtraction_llama3'] != '-',
-        #     data_dict['arc_easy_gemma2'] != '-',
-        #     data_dict['arc_easy_llama3'] != '-',
-        #     data_dict['arc_challenge_llama3'] != '-'
-        # ]
-        
-        # data_dict["Average"] = round(np.mean(all_scores), 2) if all(required_entries) else '-'
         data_dict["Average"] = round(np.mean(all_scores), 2) if '-' not in data_dict.values() else '-'
         return data_dict
 
@@ -207,9 +170,63 @@ def get_raw_eval_results_mib_subgraph(results_path: str, requests_path: str) ->
 
 
 
+
+def process_single_json(json_file: Dict[str, Any], method_counter: int) -> pd.DataFrame:
+    """
+    Process a single JSON file and convert it to a DataFrame.
+    
+    Args:
+        json_file: Dictionary containing the analysis results
+        method_counter: Counter for handling duplicate method names
+        
+    Returns:
+        pd.DataFrame: DataFrame for single method with MODEL_TASK_INTERVENTION as columns
+    """
+    method_name = json_file['method_name']
+    unique_method_name = f"{method_name}_{method_counter}"
+    method_scores = []
+    
+    for result in json_file['results']:
+        model = result['model_id']
+        
+        for task, scores in result['task_scores'].items():
+            # Process each layer's data
+            intervention_scores = defaultdict(list)
+            
+            for layer_data in scores:
+                for intervention_data in layer_data['layer_scores']:
+                    # Calculate average score for counterfactuals
+                    avg_cf_score = np.mean([
+                        cf['score'] 
+                        for cf in intervention_data['counterfactual_scores']
+                    ])
+                    
+                    if np.isnan(avg_cf_score):
+                        avg_cf_score = 0.0
+                        
+                    # Group scores by intervention
+                    intervention_key = '_'.join(intervention_data['intervention'])
+                    intervention_scores[intervention_key].append(avg_cf_score)
+            
+            # Average across layers for each intervention
+            for intervention, layer_scores in intervention_scores.items():
+                column = f"{model}_{task}_{intervention}"
+                avg_score = np.mean(layer_scores) if layer_scores else 0.0
+                method_scores.append((column, f"{avg_score:.3f}"))
+    
+    # Sort by column names for consistency
+    method_scores.sort(key=lambda x: x[0])
+    data = {
+        unique_method_name: {
+            col: score for col, score in method_scores
+        }
+    }
+    
+    return pd.DataFrame.from_dict(data, orient='index')
+
+
 # @dataclass
 # class EvalResult_MIB_CAUSALGRAPH:
-#     """Represents one full evaluation for a method in MIB causalgraph."""
 #     eval_name: str        
 #     method_name: str      
 #     results: Dict
@@ -222,31 +239,26 @@ def get_raw_eval_results_mib_subgraph(results_path: str, requests_path: str) ->
 #         method_name = data.get("method_name")
 #         results = {}
         
-#         # Get results for each model
+#         # Process each model's results
 #         for model_result in data.get("results", []):
-#             model_id = model_result.get("model_id", "")  # Will be one of the three models
+#             model_id = model_result.get("model_id", "")
+
+
+
 #             task_scores = model_result.get("task_scores", {})
             
-#             # Process MCQA task scores
-#             mcqa_scores = {}
+#             # Process MCQA scores
 #             for layer_data in task_scores.get("MCQA", []):
 #                 layer = layer_data.get("layer")
-#                 layer_scores = layer_data.get("layer_scores", [])
-                
-#                 # Store scores for each intervention and counterfactual
-#                 for intervention_data in layer_scores:
-#                     intervention = intervention_data["intervention"][0]
-#                     counterfactual_scores = intervention_data["counterfactual_scores"]
-                    
-#                     for cf_score in counterfactual_scores:
+#                 for score_data in layer_data.get("layer_scores", []):
+#                     intervention = score_data["intervention"][0]
+#                     for cf_score in score_data["counterfactual_scores"]:
 #                         counterfactual = cf_score["counterfactual"][0]
 #                         score = cf_score["score"]
                         
-#                         # Create key for this combination
-#                         key = f"layer{layer}_{intervention}_{counterfactual}"
-#                         mcqa_scores[key] = score
-
-#             results[model_id] = mcqa_scores
+#                         # Create key matching the expected column format
+#                         key = f"{model_id}_layer{layer}_{intervention}_{counterfactual}"
+#                         results[key] = score
 
 #         return EvalResult_MIB_CAUSALGRAPH(
 #             eval_name=method_name,
@@ -254,198 +266,70 @@ def get_raw_eval_results_mib_subgraph(results_path: str, requests_path: str) ->
 #             results=results
 #         )
 
-    # def to_dict(self):
-    #     """Converts the Eval Result to a dict for dataframe display"""
-    #     data_dict = {
-    #         "eval_name": self.eval_name,
-    #         "Method": self.method_name,
-    #     }
-        
-    #     # Process each model's results
-    #     for model_id, model_results in self.results.items():
-    #         for task, task_scores in model_results.items():
-    #             # Calculate layer-averaged scores for each intervention
-    #             intervention_scores = defaultdict(list)
-                
-    #             for layer_data in task_scores:
-    #                 for score_data in layer_data['scores']:
-    #                     intervention = score_data['intervention']
-    #                     intervention_scores[intervention].append(score_data['score'])
-                
-    #             # Average across layers for each intervention
-    #             for intervention, scores in intervention_scores.items():
-    #                 col_name = f"{model_id}_{task}_{intervention}".lower()
-    #                 data_dict[col_name] = round(np.mean(scores), 3)
-        
-    #     return data_dict
-
 
-    # def to_dict(self):
-    #     """Converts the Eval Result to a dict for dataframe display"""
-    #     data_dict = {
-    #         "eval_name": self.eval_name,
-    #         "Method": self.method_name,
-    #     }
+#         data = {}
+#         method_counters = defaultdict(int)
+        
+#         for json_file in json_files:
+#             # Handle method name and duplicates
+#             method_name = json_file['method_name']
+#             method_counters[method_name] += 1
+#             unique_method_name = f"{method_name}_{method_counters[method_name]}"
             
-    #     # Process each model's results
-    #     for model_id, task_scores in self.results.items():
-    #         model_name = model_id.lower()  # Lowercase for consistency
+#             method_scores = []
             
-    #         # Each task_scores contains layer data
-    #         for layer_data in task_scores.get("MCQA", []):
-    #             layer = layer_data.get("layer")
-    #             layer_scores = layer_data.get("layer_scores", [])
+#             for result in json_file['results']:
+#                 model = result['model_id']
                 
-    #             # Process each intervention and counterfactual
-    #             for intervention_data in layer_scores:
-    #                 intervention = intervention_data["intervention"][0]
-    #                 counterfactual_scores = intervention_data["counterfactual_scores"]
+#                 for task, scores in result['task_scores'].items():
+#                     # Process each layer's data
+#                     intervention_scores = defaultdict(list)
                     
-    #                 for cf_score in counterfactual_scores:
-    #                     counterfactual = cf_score["counterfactual"][0]
-    #                     score = cf_score["score"]
-                        
-    #                     # Column name matches what we defined in utils.py
-    #                     col_name = f"{model_name}_layer{layer}_{intervention}_{counterfactual}".lower()
-    #                     data_dict[col_name] = score
-
-    #     return data_dict
-
-    # def to_dict(self):
-    #     """Converts the Eval Result to a dict for dataframe display"""
-    #     print(f"Results in to_dict: {self.results}")  # Debug print
-        
-    #     data_dict = {
-    #         "eval_name": self.eval_name,
-    #         "Method": self.method_name,
-    #     }
-            
-    #     # Process each model's results
-    #     for model_id, scores in self.results.items():
-    #         model_name = model_id.lower()
-    #         for task, layer_scores in scores.items():
-    #             for layer_data in layer_scores:
-    #                 layer = layer_data.get("layer")
-    #                 intervention_scores = layer_data.get("scores", [])
+#                     for layer_data in scores:
+#                         for intervention_data in layer_data['layer_scores']:
+#                             # Calculate average score for counterfactuals
+#                             avg_cf_score = np.mean([
+#                                 cf['score'] 
+#                                 for cf in intervention_data['counterfactual_scores']
+#                             ])
+                            
+#                             if np.isnan(avg_cf_score):
+#                                 avg_cf_score = 0.0
+                                
+#                             # Group scores by intervention
+#                             intervention_key = '_'.join(intervention_data['intervention'])
+#                             intervention_scores[intervention_key].append(avg_cf_score)
                     
-    #                 for intervention_data in intervention_scores:
-    #                     col_name = f"{model_name}_layer{layer}_{intervention_data['intervention']}".lower()
-    #                     data_dict[col_name] = intervention_data['score']
-
-    #     return data_dict
-
-
-    # def to_dict(self):
-    #     """Converts the Eval Result to a dict for dataframe display"""
-    #     data_dict = {
-    #         "eval_name": self.eval_name,
-    #         "Method": self.method_name,
-    #     }
+#                     # Average across layers for each intervention
+#                     for intervention, layer_scores in intervention_scores.items():
+#                         column = f"{model}_{task}_{intervention}"
+#                         avg_score = np.mean(layer_scores) if layer_scores else 0.0
+#                         method_scores.append((column, f"{avg_score:.3f}"))
             
-    #     # Process each model's results
-    #     for model_id, scores in self.results.items():
-    #         model_name = model_id.lower()
-    #         # The scores are already in the format we want
-    #         for key, value in scores.items():
-    #             col_name = f"{model_name}_{key}"
-    #             data_dict[col_name] = value
-
-    #     return data_dict
-
-
-
-
-
-
-
-
-@dataclass
-class EvalResult_MIB_CAUSALGRAPH:
-    eval_name: str        
-    method_name: str      
-    results: Dict
-
-    def init_from_json_file(self, json_filepath):
-        """Inits results from the method result file"""
-        with open(json_filepath) as fp:
-            data = json.load(fp)
-
-        method_name = data.get("method_name")
-        results = {}
+#             # Sort by column names for consistency
+#             method_scores.sort(key=lambda x: x[0])
+#             data[unique_method_name] = {
+#                 col: score for col, score in method_scores
+#             }
         
-        # Process each model's results
-        for model_result in data.get("results", []):
-            model_id = model_result.get("model_id", "")
-            task_scores = model_result.get("task_scores", {})
-            
-            # Process MCQA scores
-            for layer_data in task_scores.get("MCQA", []):
-                layer = layer_data.get("layer")
-                for score_data in layer_data.get("layer_scores", []):
-                    intervention = score_data["intervention"][0]
-                    for cf_score in score_data["counterfactual_scores"]:
-                        counterfactual = cf_score["counterfactual"][0]
-                        score = cf_score["score"]
-                        
-                        # Create key matching the expected column format
-                        key = f"{model_id}_layer{layer}_{intervention}_{counterfactual}"
-                        results[key] = score
-
-        return EvalResult_MIB_CAUSALGRAPH(
-            eval_name=method_name,
-            method_name=method_name,
-            results=results
-        )
-
-    def to_dict(self):
-        """Converts the Eval Result to a dict for dataframe display"""
-        data_dict = {
-            "eval_name": self.eval_name,
-            "Method": self.method_name,
-        }
+#         return pd.DataFrame.from_dict(data, orient='index')
+
+#     def to_dict(self):
+#         """Converts the Eval Result to a dict for dataframe display"""
+#         data_dict = {
+#             "eval_name": self.eval_name,
+#             "Method": self.method_name,
+#         }
         
-        # Add all results directly
-        data_dict.update(self.results)
+#         # Add all results directly
+#         data_dict.update(self.results)
         
-        return data_dict
-
+#         return data_dict
 
 
 
 
 
-
-
-
-# def get_raw_eval_results_mib_causalgraph(results_path: str, requests_path: str) -> List[EvalResult_MIB_CAUSALGRAPH]:
-#     """Extract evaluation results for MIB causalgraph"""
-#     model_result_filepaths = []
-    
-#     for root, dirnames, files in os.walk(results_path):
-#         if len(files) == 0 or any([not f.endswith(".json") for f in files]):
-#             continue
-        
-#         try:
-#             files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
-#         except dateutil.parser._parser.ParserError:
-#             files = [files[-1]]
-
-#         for file in files:
-#             model_result_filepaths.append(os.path.join(root, file))
-
-#     eval_results = []
-#     for filepath in model_result_filepaths:
-#         try:
-#             eval_result = EvalResult_MIB_CAUSALGRAPH("", "", {})
-#             result = eval_result.init_from_json_file(filepath)
-#             result.to_dict()  # Verify conversion works
-#             eval_results.append(result)
-#         except Exception as e:
-#             print(f"Error processing {filepath}: {e}")
-#             continue
-
-#     return eval_results
-
 def get_raw_eval_results_mib_causalgraph(results_path: str, requests_path: str) -> List[EvalResult_MIB_CAUSALGRAPH]:
     model_result_filepaths = []
     
@@ -466,23 +350,30 @@ def get_raw_eval_results_mib_causalgraph(results_path: str, requests_path: str)
 
     # print(f"Found json files: {model_result_filepaths}")
 
-    eval_results = []
-    for filepath in model_result_filepaths:
+    method_counters = defaultdict(int)
+    dataframes = []
+    
+    for json_file in model_result_filepaths:
         try:
-            eval_result = EvalResult_MIB_CAUSALGRAPH("", "", {})
-            result = eval_result.init_from_json_file(filepath)
-            # print(f"Processed file {filepath}")
-            # print(f"Got result: {result}")
-            eval_results.append(result)
+            with open(filepath, 'r') as f:
+                json_data = json.load(f)
+                method_name = json_data['method_name']
+                method_counters[method_name] += 1
+            
+            # Process single JSON file
+            df = process_single_json(json_data, method_counters[method_name])
+            dataframes.append(df)
         except Exception as e:
-            print(f"Error processing {filepath}: {e}")
+            print(f"Error processing {json_file}: {e}")
             continue
-
-    # print(f"Total results processed: {len(eval_results)}")
-    return eval_results
-
-
-
+    
+    # # Concatenate all DataFrames
+    # if dataframes:
+    #     final_df = pd.concat(dataframes, axis=0)
+    #     return final_df
+    # else:
+    #     return pd.DataFrame()
+    return dataframes
 
 
 

src/populate.py

@@ -29,14 +29,6 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
 
     df = pd.DataFrame.from_records(all_data_json)
     df = df.sort_values(by=[AutoEvalColumn.text_average.name], ascending=False)
-    # df = df.sort_values(by=[Tasks.task0.value.col_name], ascending=False)
-    # df = df.sort_values(by=[AutoEvalColumn.track.name], ascending=False)
-    
-    # print(f"df is {df}")
-    
-    # df = df[cols].round(decimals=1)
-
-    # filter out if any of the benchmarks have not been produced
     df = df[has_no_nan_values(df, benchmark_cols)]
     return df
 
@@ -69,23 +61,6 @@ def get_leaderboard_df_mib_subgraph(results_path: str, requests_path: str, cols:
 
 
 
-# def aggregate_methods(df: pd.DataFrame) -> pd.DataFrame:
-#     """Aggregates rows with the same base method name by taking the max value for each column"""
-#     df_copy = df.copy()
-    
-#     # Extract base method names (remove _2, _3, etc. suffixes)
-#     base_methods = [name.split('_')[0] if '_' in name and name.split('_')[-1].isdigit() 
-#                    else name for name in df_copy.index]
-#     df_copy.index = base_methods
-    
-#     # Convert scores to numeric values
-#     numeric_df = df_copy.select_dtypes(include=['float64', 'int64'])
-    
-#     # Group by base method name and take the max
-#     aggregated_df = numeric_df.groupby(level=0).max().round(3)
-    
-#     return aggregated_df
-
 def aggregate_methods(df: pd.DataFrame) -> pd.DataFrame:
     """Aggregates rows with the same base method name by taking the max value for each column"""
     df_copy = df.copy()
@@ -111,63 +86,6 @@ def aggregate_methods(df: pd.DataFrame) -> pd.DataFrame:
     
     return aggregated_df
 
-# def create_intervention_averaged_df(df: pd.DataFrame) -> pd.DataFrame:
-#     """Creates a DataFrame where columns are model_task and cells are averaged over interventions"""
-#     df_copy = df.copy()
-    
-#     # Remove the Method column and eval_name if present
-#     columns_to_drop = ['Method', 'eval_name']
-#     df_copy = df_copy.drop(columns=[col for col in columns_to_drop if col in df_copy.columns])
-    
-#     # Group columns by model_task
-#     model_task_groups = {}
-#     for col in df_copy.columns:
-#         model_task = '_'.join(col.split('_')[:2])  # Get model_task part
-#         if model_task not in model_task_groups:
-#             model_task_groups[model_task] = []
-#         model_task_groups[model_task].append(col)
-    
-#     # Create new DataFrame with averaged intervention scores
-#     averaged_df = pd.DataFrame({
-#         model_task: df_copy[cols].mean(axis=1).round(3)
-#         for model_task, cols in model_task_groups.items()
-#     })
-    
-#     return averaged_df
-
-# def create_intervention_averaged_df(df: pd.DataFrame) -> pd.DataFrame:
-#     """Creates a DataFrame where columns are model_task and cells are averaged over interventions"""
-#     df_copy = df.copy()
-    
-#     # Store Method column if it exists
-#     method_col = None
-#     if 'Method' in df_copy.columns:
-#         method_col = df_copy['Method']
-#         df_copy = df_copy.drop('Method', axis=1)
-    
-#     # Remove eval_name if present
-#     if 'eval_name' in df_copy.columns:
-#         df_copy = df_copy.drop('eval_name', axis=1)
-    
-#     # Group columns by model_task
-#     model_task_groups = {}
-#     for col in df_copy.columns:
-#         model_task = '_'.join(col.split('_')[:2])  # Get model_task part
-#         if model_task not in model_task_groups:
-#             model_task_groups[model_task] = []
-#         model_task_groups[model_task].append(col)
-    
-#     # Create new DataFrame with averaged intervention scores
-#     averaged_df = pd.DataFrame({
-#         model_task: df_copy[cols].mean(axis=1).round(3)
-#         for model_task, cols in model_task_groups.items()
-#     })
-    
-#     # Add Method column back
-#     if method_col is not None:
-#         averaged_df.insert(0, 'Method', method_col)
-    
-#     return averaged_df
 
 def create_intervention_averaged_df(df: pd.DataFrame) -> pd.DataFrame:
     """Creates a DataFrame where columns are model_task and cells are averaged over interventions"""
@@ -203,69 +121,32 @@ def create_intervention_averaged_df(df: pd.DataFrame) -> pd.DataFrame:
     
     return averaged_df
 
-# def get_leaderboard_df_mib_causalgraph(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
-#     """Creates a dataframe from all the MIB causal graph experiment results"""
-#     print(f"results_path is {results_path}, requests_path is {requests_path}")
-#     raw_data = get_raw_eval_results_mib_causalgraph(results_path, requests_path)
-#     print(f"raw_data is {raw_data}")
-    
-#     # Convert each result to dict format for detailed df
-#     all_data_json = [v.to_dict() for v in raw_data]
-#     detailed_df = pd.DataFrame.from_records(all_data_json)
-#     print(f"detailed_df is: {detailed_df}")
-
-#     # Create and print other views for debugging/reference
-#     aggregated_df = aggregate_methods(detailed_df)
-#     print(f"aggregated_df is: {aggregated_df}")
-    
-#     intervention_averaged_df = create_intervention_averaged_df(aggregated_df)
-#     print(f"intervention_averaged_df is: {intervention_averaged_df}")
-    
-#     # Only return detailed_df for display
-#     return detailed_df
 
-# def get_leaderboard_df_mib_causalgraph(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
-#     print(f"results_path is {results_path}, requests_path is {requests_path}")
-#     raw_data = get_raw_eval_results_mib_causalgraph(results_path, requests_path)
-    
-#     # Convert each result to dict format for detailed df
-#     all_data_json = [v.to_dict() for v in raw_data]
-#     detailed_df = pd.DataFrame.from_records(all_data_json)
-#     print("Columns in detailed_df:", detailed_df.columns.tolist())  # Print actual columns
-    
-#     # Create aggregated df
-#     aggregated_df = aggregate_methods(detailed_df)
-#     print("Columns in aggregated_df:", aggregated_df.columns.tolist())
-    
-#     # Create intervention-averaged df
-#     intervention_averaged_df = create_intervention_averaged_df(aggregated_df)
-#     print("Columns in intervention_averaged_df:", intervention_averaged_df.columns.tolist())
-    
-#     return detailed_df, aggregated_df, intervention_averaged_df
 
-def get_leaderboard_df_mib_causalgraph(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
+def get_leaderboard_df_mib_causalgraph(results_path: str, requests_path: str) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
     # print(f"results_path is {results_path}, requests_path is {requests_path}")
-    raw_data = get_raw_eval_results_mib_causalgraph(results_path, requests_path)
-    
-    # Convert each result to dict format for detailed df
-    all_data_json = [v.to_dict() for v in raw_data]
-    detailed_df = pd.DataFrame.from_records(all_data_json)
+
+    # raw_data = get_raw_eval_results_mib_causalgraph(results_path, requests_path)
+    # all_data_json = [v.to_dict() for v in raw_data]
+    # detailed_df = pd.DataFrame.from_records(all_data_json)
+
+    detailed_df = get_raw_eval_results_mib_causalgraph(results_path, requests_path)
 
     # Print the actual columns for debugging
-    # print("Original columns:", detailed_df.columns.tolist())
-    
-    # Rename columns to match schema
-    column_mapping = {}
-    for col in detailed_df.columns:
-        if col in ['eval_name', 'Method']:
-            continue
-        # Ensure consistent casing for the column names
-        new_col = col.replace('Qwen2ForCausalLM', 'qwen2forcausallm') \
-                    .replace('Gemma2ForCausalLM', 'gemma2forcausallm') \
-                    .replace('LlamaForCausalLM', 'llamaforcausallm')
-        column_mapping[col] = new_col
-    
-    detailed_df = detailed_df.rename(columns=column_mapping)
+    print("Original columns:", detailed_df.columns.tolist())
+    
+    # # Rename columns to match schema
+    # column_mapping = {}
+    # for col in detailed_df.columns:
+    #     if col in ['eval_name', 'Method']:
+    #         continue
+    #     # Ensure consistent casing for the column names
+    #     new_col = col.replace('Qwen2ForCausalLM', 'qwen2forcausallm') \
+    #                 .replace('Gemma2ForCausalLM', 'gemma2forcausallm') \
+    #                 .replace('LlamaForCausalLM', 'llamaforcausallm')
+    #     column_mapping[col] = new_col
+    
+    # detailed_df = detailed_df.rename(columns=column_mapping)
     
     # Create aggregated df
     aggregated_df = aggregate_methods(detailed_df)