From 085db228adfc9a69e030532008d1c933e028fcb0 Mon Sep 17 00:00:00 2001 From: "Lennart J. Kurzweg (Nx2)" Date: Wed, 28 Aug 2024 20:45:12 +0200 Subject: [PATCH] vis --- visualize.py | 92 ++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 74 insertions(+), 18 deletions(-) diff --git a/visualize.py b/visualize.py index 6631993..ba31660 100644 --- a/visualize.py +++ b/visualize.py @@ -1,9 +1,12 @@ import json import matplotlib.pyplot as plt import pandas as pd -import numpy as np import seaborn as sns +from suite_settings.models import models +from suite_settings.techniques import techniques +from suite_settings.tests import tests + # Load the JSON data with open('saved_results.json', 'r') as f: data = json.load(f) @@ -13,9 +16,12 @@ results = [] for test_hash, test_data in data.items(): results.append({ "hash": test_hash, - "model": test_data['model_name'], + "model_name": models[test_data['model_id']].display_name, + "model_size": models[test_data['model_id']].parameter_count_in_b, + "technique_name": techniques[test_data['technique_id']].name, + "model_technique": f"{models[test_data['model_id']].display_name}:{ techniques[test_data['technique_id']].name}", "seed": test_data['seed'], - "test_name": test_data['test_name'], + "test_name": tests[test_data['test_id']].name, "validation": test_data['validation'] }) @@ -23,21 +29,34 @@ df = pd.DataFrame(results) +df['technique_name'] = pd.Categorical(df['technique_name'], categories=[techniques[1].name, techniques[572].name, techniques[903].name],ordered=True) +df['test_name'] = pd.Categorical(df['test_name'], categories=[tests[607].name, tests[693].name, tests[120].name, tests[283].name, tests[260].name, tests[856].name],ordered=True) +sorted_df = df.sort_values('model_size') + +# Perform the groupby and unstack operation +result_df = ( + sorted_df.groupby(['model_name', 'validation']).size() + .unstack(fill_value=0) # Unstack and fill NaN with 0 +) + ## 1st Chart -# Count the number of validation results for each model -validation_counts = df.groupby(['model', 'validation']).size().unstack().fillna(0) +# Count the number of validation results for each technique_name +validation_counts = result_df.loc[sorted_df['model_name'].drop_duplicates()] validation_counts.columns = ['Failed', 'Passed'] -# Plot the validation results by model +# Plot the validation results by technique_name plt.figure(figsize=(10, 6)) validation_counts.plot(kind='bar', stacked=True, color=['red', 'green'], ax=plt.gca()) -plt.title('Validation Results by Model') -plt.xlabel('Model') +plt.title('Validation Results by Model and Technique') +plt.xlabel('Model and Technique') plt.ylabel('Number of Tests') plt.xticks(rotation=45, ha='right') plt.legend(title='Validation') plt.tight_layout() -plt.savefig('validation_results_by_model.png') +plt.savefig('model-bar-chart.png') + + + @@ -54,25 +73,46 @@ plt.xlabel('Number of Tests') plt.ylabel('Test Name') plt.legend(title='Validation') plt.tight_layout() -plt.savefig('validation_results_by_test_name.png') +plt.savefig('test-bar-chart.png') +sorted_df = df.sort_values('model_size' ) -## 3rd Chart -pass_rate = pd.pivot_table(df, values='validation', index='model', columns='test_name', aggfunc="mean", fill_value=0) -# Create a heatmap -plt.figure(figsize=(8, 8)) -sns.heatmap(pass_rate*100, annot=True, fmt=".0f", cmap=sns.color_palette("blend:#100,#255,#4a3", as_cmap=True), cbar=True, annot_kws={"size": 10}) +# Get the unique order of 'model_technique' based on sorted_df +ordered_techniques = sorted_df['model_technique'].unique() +# Create the pivot table with the correct order of model_technique +pass_rate = pd.pivot_table( + sorted_df, + values='validation', + index='model_technique', + columns='test_name', + aggfunc="mean", + fill_value=0 +) + +# Reorder the rows in the pivot table based on the ordered techniques +pass_rate = pass_rate.loc[ordered_techniques] + +# Plot the heatmap +plt.figure(figsize=(8, 10)) +sns.heatmap( + pass_rate * 100, + annot=True, + fmt=".0f", + cmap=sns.color_palette("blend:#100,#255,#4a3", as_cmap=True), + cbar=True, + annot_kws={"size": 10} +) # Add percentage sign to annotations for text in plt.gca().texts: text.set_text(f"{text.get_text()}%") # Customize the plot with labels and a title -plt.title('Model Performance on Each Test', fontsize=16) +plt.title('Model Technique Performance on Each Test', fontsize=16) plt.xlabel('Test Name', fontsize=14) -plt.ylabel('Model', fontsize=14) +plt.ylabel('Model and Technique', fontsize=14) # Rotate x-axis labels by 45 degrees plt.xticks(rotation=45, ha='right') @@ -81,6 +121,22 @@ plt.xticks(rotation=45, ha='right') plt.tight_layout() # Save the heatmap -plt.savefig('model_performance_heatmap.png') +plt.savefig('modelTechnique_heatmap.png') +## 4th Chart: Technique Performance on Each Test (Aggregated Heatmap) +technique_pass_rate = pd.pivot_table(sorted_df, values='validation', index='test_name', columns='technique_name', aggfunc="mean", fill_value=0) +plt.figure(figsize=(8, 4)) +sns.heatmap(technique_pass_rate*100, annot=True, fmt=".0f", cmap=sns.color_palette("blend:#100,#255,#4a3", as_cmap=True), cbar=True, annot_kws={"size": 10}) + +# Add percentage sign to annotations +for text in plt.gca().texts: + text.set_text(f"{text.get_text()}%") + +# Customize the plot with labels and a title +plt.title('Technique Performance on Each Test', fontsize=16) +plt.ylabel('Test Name', fontsize=14) +plt.xlabel('Technique', fontsize=14) +plt.xticks(rotation=0) +plt.tight_layout() +plt.savefig('technique_heatmap.png')