From 76ef8502d1f319a31d721e90d059ed5aa24576c8 Mon Sep 17 00:00:00 2001
From: "Lennart J. Kurzweg (Nx2)" <git@nx2.site>
Date: Thu, 8 Aug 2024 16:52:23 +0200
Subject: [PATCH] hash fixed + better run pirnts

---
 libs/run_tests.py | 103 +++++++++++++++++++++++++++++++++++++---------
 1 file changed, 83 insertions(+), 20 deletions(-)

diff --git a/libs/run_tests.py b/libs/run_tests.py
index da1389a..1c44ff4 100644
--- a/libs/run_tests.py
+++ b/libs/run_tests.py
@@ -10,19 +10,39 @@ def padd(list, element):
         longest = max(longest, len(str(s))) 
     return str(element).ljust(longest)
 
-           
+def nxhash(text:str): # @BenVida StackOverflow
+    hash=0
+    for ch in text:
+        hash = ( hash*281  ^ ord(ch)*997) & 0xFFFFFFFF
+    return hex(hash)[2:].upper().zfill(8)
 
-def run_tests(models: list[str], seeds: list[int], tests: list[Test], base_url: str):
-
-    # try:
-    with open("./saved_results.json", "r") as f:
-        saved_results = json.load(fp=f)
-    # except: 
-        # saved_results = {}
+def get_len(l: list) -> int:
+    m = 0
+    for e in l:
+        if isinstance(e, Test):
+            m = max(m, len(e.name))
+        elif isinstance(e, str):
+            m = max(m, len(e))
+        elif isinstance(e, int):
+            m = max(m, len(str(e)))
+        else:
+            raise Exception(f"get_len() only supports lits of Test, str or int but got {type(e)}")
+    return m
 
     
+    
+def run_tests(models: list[str], seeds: list[int], tests: list[Test], base_url: str):
+    try:
+        print("Trying to load saved_results.json")
+        with open("./saved_results.json", "r") as f:
+            saved_results = json.load(fp=f)
+        print("Loaded.")
+    except: 
+        print("saved_results.json not found. Initializing empty.")
+        saved_results = {}
     # Get Results
     run_results = {}
+    print("Starting to run Tests ... ")
     for model in models:
         for seed in seeds:
             for test in tests:
@@ -33,21 +53,50 @@ def run_tests(models: list[str], seeds: list[int], tests: list[Test], base_url:
                     'model': model,
                     'seed': seed,
                 }
-                hash_key = str(hash(json.dumps(combination, sort_keys=True)))
+                hash_key = str(nxhash(json.dumps(combination, sort_keys=True)))
 
                 if hash_key not in saved_results.keys():
-                    # try: 
-                    combination['answer'] = test.runnable(model=model, seed=seed, test=test, base_url=base_url)
-                    combination['test'] = test
-                    run_results[hash_key] = combination
-                    print(f"Model {padd(models, model)} starting with seed {padd(seeds, seed)} is done with test '{test.name}'.")
-                    # except Exception as e:
-                        # print("\033[0;31mError:\033[0m " + str(e))
+                    try: 
+                        combination['answer'] = test.runnable(model=model, seed=seed, test=test, base_url=base_url)
+                        combination['test'] = test
+                        run_results[hash_key] = combination
+                        print("\033[0;32mModel '\033[0m" +
+                              model +
+                              "\033[0;32m'" +
+                              (" " * (get_len(models) - len(model))) +
+                              " with seed \033[0m" +
+                              str(seed) +
+                              (" " * (get_len(seeds) - len(str(seed)))) +
+                              "\033[0;32m finished test '\033[0m" +
+                              test.name +
+                              "\033[0;32m'" +
+                              (" " * (get_len(tests) - len(test.name))) +
+                              " (\033[0m" +
+                              hash_key +
+                              "\033[0;32m)\033[0m"
+                        )
+                    except Exception as e:
+                          print("\033[0;31mError: <\033[0m " + str(e) + "\033[0;31m>\033[0m trying to continue...")
                 else: 
-                    print(f"Skipped {combination}")
+                    print("\033[0;34mModel '\033[0m" +
+                          model +
+                          "\033[0;34m'" +
+                          (" " * (get_len(models) - len(model))) +
+                          " with seed \033[0m" +
+                          str(seed) +
+                          (" " * (get_len(seeds) - len(str(seed)))) +
+                          "\033[0;34m skipped  test '\033[0m" +
+                          test.name +
+                          "\033[0;34m'" +
+                          (" " * (get_len(tests) - len(test.name))) +
+                          " (\033[0m" +
+                          hash_key +
+                          "\033[0;34m) becasue its results exists in saved_results.json\033[0m"
+                    )
 
 
     # Validate Results
+    if run_results != {}: print("\nStarting validation of tests ...")
     for hash_key in run_results:
         result = run_results[hash_key]
         
@@ -61,10 +110,24 @@ def run_tests(models: list[str], seeds: list[int], tests: list[Test], base_url:
 
         saved_results[hash_key] = entry # add result with validation to saved results
 
-        print(f"Validation of answer from test {entry['test_name']} by {entry['model']} with seed {entry['seed']} evaluated to " + ('\033[0;32mcorrect\033[0m' if entry['validation'] == True else '\033[0;31mincorrect\033[0m'))
+        print("\033[0;36mTest results of model '\033[0m" +
+              model +
+              "\033[0;36m'" +
+              (" " * (get_len(models) - len(entry['model']))) +
+              " with seed \033[0m" +
+              str(seed) +
+              (" " * (get_len(seeds) - len(str(entry['seed'])))) +
+              "\033[0;36m on test '\033[0m" +
+              test.name +
+              "\033[0;36m'" +
+              (" " * (get_len(tests) - len(entry['test_name']))) +
+              " (\033[0m" +
+              hash_key +
+              "\033[0;36m) evaluated to \033[0m" + 
+              ('\033[0;32mcorrect\033[0m' if entry['validation'] == True else '\033[0;31mincorrect\033[0m')
+        )
 
     with open("./saved_results.json", "w") as f:
-        json.dump(fp=f, obj=saved_results, indent=4, ensure_ascii=False)
-        print("Dumped")
+        json.dump(fp=f, obj=saved_results, indent=4, sort_keys=True, ensure_ascii=False)
 
     return saved_results