caulsal-track debug

src/leaderboard/read_evals.py

@@ -275,10 +275,12 @@ def get_raw_eval_results_mib_subgraph(results_path: str, requests_path: str) ->
 from dataclasses import dataclass
 import json
 import numpy as np
+import pandas as pd
 from typing import Dict, List, Any
 import os
 from datetime import datetime
 import dateutil
+from collections import defaultdict
 
 @dataclass
 class EvalResult_MIB_CAUSALGRAPH:
@@ -354,14 +356,86 @@ class EvalResult_MIB_CAUSALGRAPH:
                     continue
                 
                 avg_score = np.mean(scores)
-                data_dict[col_name] = round(avg_score, 3)
+                data_dict[col_name] = f"{avg_score:.3f}"
                 all_scores.append(avg_score)
     
-        data_dict["Average"] = round(np.mean(all_scores), 3) if all_scores else '-'
+        data_dict["Average"] = f"{np.mean(all_scores):.3f}" if all_scores else '-'
         return data_dict
 
 
-def get_raw_eval_results_mib_causalgraph(results_path: str, requests_path: str) -> List[EvalResult_MIB_CAUSALGRAPH]:
+def aggregate_methods(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Aggregates rows with the same base method name by taking the max value for each column.
+    """
+    # Create a copy of the DataFrame
+    df_copy = df.copy()
+    
+    # Extract base method names (remove _2, _3, etc. suffixes)
+    base_methods = [name.split('_')[0] if '_' in name and name.split('_')[-1].isdigit() 
+                   else name for name in df_copy.index]
+    df_copy.index = base_methods
+    
+    # Convert scores to numeric values
+    def extract_score(score_str):
+        if isinstance(score_str, str):
+            return float(score_str)
+        return 0.0
+    
+    numeric_df = df_copy.applymap(extract_score)
+    
+    # Group by base method name and take the max
+    aggregated_df = numeric_df.groupby(level=0).max().round(3)
+    
+    # Convert back to string format
+    aggregated_df = aggregated_df.applymap(lambda x: f"{x:.3f}")
+    
+    return aggregated_df
+
+
+def create_intervention_averaged_df(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Creates a DataFrame where columns are model_task and cells are averaged over interventions.
+    """
+    # Create a copy of the DataFrame
+    df_copy = df.copy()
+    
+    # Remove the Average column if it exists
+    if 'Average' in df_copy.columns:
+        df_copy = df_copy.drop('Average', axis=1)
+    
+    # Function to extract score value from string
+    def extract_score(score_str):
+        if isinstance(score_str, str):
+            return float(score_str)
+        return 0.0
+    
+    # Convert all scores to numeric values
+    numeric_df = df_copy.applymap(extract_score)
+    
+    # Group columns by model_task
+    model_task_groups = {}
+    for col in numeric_df.columns:
+        model_task = '_'.join(col.split('_')[:2])  # Get model_task part
+        if model_task not in model_task_groups:
+            model_task_groups[model_task] = []
+        model_task_groups[model_task].append(col)
+    
+    # Create new DataFrame with averaged intervention scores
+    averaged_df = pd.DataFrame({
+        model_task: numeric_df[cols].mean(axis=1).round(3)
+        for model_task, cols in model_task_groups.items()
+    })
+    
+    # Add overall average column
+    averaged_df['Average'] = averaged_df.mean(axis=1).round(3)
+    
+    # Sort by Average column
+    averaged_df = averaged_df.sort_values('Average', ascending=False)
+    
+    return averaged_df
+
+
+def get_raw_eval_results_mib_causalgraph(results_path: str, requests_path: str) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
     """From the path of the results folder root, extract all needed info for MIB causal graph results"""
     model_result_filepaths = []
     
@@ -377,24 +451,45 @@ def get_raw_eval_results_mib_causalgraph(results_path: str, requests_path: str)
         for file in files:
             model_result_filepaths.append(os.path.join(root, file))
 
-    eval_results = []
-    for model_result_filepath in model_result_filepaths:
+    method_counters = defaultdict(int)
+    data_dicts = []
+
+    for filepath in model_result_filepaths:
         try:
-            eval_result = EvalResult_MIB_CAUSALGRAPH("", "", {})  # Create empty instance
-            result = eval_result.init_from_json_file(model_result_filepath)
-            # Verify the result can be converted to dict format
-            result.to_dict()
-            eval_results.append(result)
+            with open(filepath, 'r') as f:
+                json_data = json.load(f)
+                method_name = json_data['method_name']
+                method_counters[method_name] += 1
+
+            eval_result = EvalResult_MIB_CAUSALGRAPH("", "", {})
+            result = eval_result.init_from_json_file(filepath)
+            data_dict = result.to_dict()
+            
+            # Add method counter to the method name if it's not the first instance
+            if method_counters[method_name] > 1:
+                data_dict["Method"] = f"{method_name}_{method_counters[method_name]}"
+            
+            data_dicts.append(data_dict)
         except Exception as e:
-            print(f"Error processing {model_result_filepath}: {e}")
+            print(f"Error processing {filepath}: {e}")
             continue
 
-    return eval_results
-
+    if not data_dicts:
+        return pd.DataFrame(), pd.DataFrame(), pd.DataFrame()
 
+    # Create the detailed DataFrame
+    detailed_df = pd.DataFrame(data_dicts)
+    detailed_df.set_index("Method", inplace=True)
+    if "eval_name" in detailed_df.columns:
+        detailed_df.drop("eval_name", axis=1, inplace=True)
 
+    # Create aggregated DataFrame
+    aggregated_df = aggregate_methods(detailed_df)
 
+    # Create intervention-averaged DataFrame
+    intervention_averaged_df = create_intervention_averaged_df(aggregated_df)
 
+    return detailed_df, aggregated_df, intervention_averaged_df
 
 
 

src/populate.py

@@ -127,14 +127,16 @@ def create_intervention_averaged_df(df: pd.DataFrame) -> pd.DataFrame:
 def get_leaderboard_df_mib_causalgraph(results_path: str, requests_path: str) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
     # print(f"results_path is {results_path}, requests_path is {requests_path}")
 
-    raw_data = get_raw_eval_results_mib_causalgraph(results_path, requests_path)
-    all_data_json = [v.to_dict() for v in raw_data]
+    raw_detailed_df, raw_aggregated_df, raw_intervention_averaged_df = get_raw_eval_results_mib_causalgraph(results_path, requests_path)
+
+    all_data_json = [v.to_dict() for v in raw_detailed_df]
     detailed_df = pd.DataFrame.from_records(all_data_json)
 
-    # detailed_df = get_raw_eval_results_mib_causalgraph(results_path, requests_path)
+    all_data_json = [v.to_dict() for v in raw_aggregated_df]
+    aggregated_df = pd.DataFrame.from_records(all_data_json)
 
-    # Print the actual columns for debugging
-    print("Original columns:", detailed_df.columns.tolist())
+    all_data_json = [v.to_dict() for v in raw_intervention_averaged_df]
+    intervention_averaged_df = pd.DataFrame.from_records(all_data_json)
     
     # # Rename columns to match schema
     # column_mapping = {}
@@ -149,11 +151,11 @@ def get_leaderboard_df_mib_causalgraph(results_path: str, requests_path: str) ->
     
     # detailed_df = detailed_df.rename(columns=column_mapping)
     
-    # Create aggregated df
-    aggregated_df = aggregate_methods(detailed_df)
+    # # Create aggregated df
+    # aggregated_df = aggregate_methods(detailed_df)
     
-    # Create intervention-averaged df
-    intervention_averaged_df = create_intervention_averaged_df(aggregated_df)
+    # # Create intervention-averaged df
+    # intervention_averaged_df = create_intervention_averaged_df(aggregated_df)
     
     # print("Transformed columns:", detailed_df.columns.tolist())