Saving results (hash always diffrent) = skipping doesnt work

libs/run_tests.py

@@ -2,28 +2,69 @@ from libs.test_class import Test
 from libs.validators import system_human_answer_match
 from libs.runnables import basic
 
+import json
+
 def padd(list, element):
     longest = 0
     for s in list:
         longest = max(longest, len(str(s))) 
     return str(element).ljust(longest)
 
+           
+
 def run_tests(models: list[str], seeds: list[int], tests: list[Test], base_url: str):
-    results = []
-    esc = "\033"
+
+    # try:
+    with open("./saved_results.json", "r") as f:
+        saved_results = json.load(fp=f)
+    # except: 
+        # saved_results = {}
+
+    
+    # Get Results
+    run_results = {}
     for model in models:
         for seed in seeds:
             for test in tests:
-                try: 
-                    result = test.runnable(model=model, seed=seed, test=test, base_url=base_url)
-                    results.append({"test": test,"model": model, "seed": seed, "result": result})
+
+                # Init dict
+                combination = { 
+                    'test_name': test.name,
+                    'model': model,
+                    'seed': seed,
+                }
+                hash_key = str(hash(json.dumps(combination, sort_keys=True)))
+
+                if hash_key not in saved_results.keys():
+                    # try: 
+                    combination['answer'] = test.runnable(model=model, seed=seed, test=test, base_url=base_url)
+                    combination['test'] = test
+                    run_results[hash_key] = combination
                     print(f"Model {padd(models, model)} starting with seed {padd(seeds, seed)} is done with test '{test.name}'.")
-                except Exception as e:
-                    print("\033[0;31mError:\033[0m " + str(e))
+                    # except Exception as e:
+                        # print("\033[0;31mError:\033[0m " + str(e))
+                else: 
+                    print(f"Skipped {combination}")
 
-    for result in results:
-        result['validation'] = result['test'].validator(test=result['test'], answer=result['result'], base_url=base_url)
 
-        print(f"Validation of answer from test {result['test'].name} by {result['model']} with seed {result['seed']} evaluated to " + ('\033[0;32mcorrect\033[0m' if result['validation'] == True else '\033[0;31mincorrect\033[0m'))
+    # Validate Results
+    for hash_key in run_results:
+        result = run_results[hash_key]
+        
+        entry = {
+            'test_name':  result['test_name'],
+            'model':      result['model'],
+            'seed':       result['seed'],
+            'answer':     result['answer'],
+            'validation': result['test'].validator(test=result['test'], answer=result['answer'], base_url=base_url)
+        }
 
-    return results
+        saved_results[hash_key] = entry # add result with validation to saved results
+
+        print(f"Validation of answer from test {entry['test_name']} by {entry['model']} with seed {entry['seed']} evaluated to " + ('\033[0;32mcorrect\033[0m' if entry['validation'] == True else '\033[0;31mincorrect\033[0m'))
+
+    with open("./saved_results.json", "w") as f:
+        json.dump(fp=f, obj=saved_results, indent=4, ensure_ascii=False)
+        print("Dumped")
+
+    return saved_results