nx2/test-small-llms
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Mul test, valdidation works, but printing it doesnt

Browse Source
This commit is contained in:
Lennart J. Kurzweg (Nx2)
2024-08-05 14:09:06 +02:00
parent 52a180b936
commit abd6320ce9
6 changed files with 99 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
libs/run_tests.py
View File

@@ -19,10 +19,10 @@ def run_tests(models: list[str], seeds: list[int], tests: list[Test], base_url:
results.append({"test": test,"model": model, "seed": seed, "result": result})
print(f"Model {padd(models, model)} starting with seed {padd(seeds, seed)} is done with test '{test.name}'.")
except Exception as e:
print("\033[0;31mError:\033[0m" + e)
print("\033[0;31mError:\033[0m " + str(e))
for result in results:
result['validation'] = test.validator(test=result['test'], answer=result['result'], base_url=base_url)
result['validation'] = result['test'].validator(test=result['test'], answer=result['result'], base_url=base_url)
print(f"Validation of answer from test {result['test'].name} by {result['model']} with seed {result['seed']} evaluated to " + ('\033[0;32mcorrect\033[0m' if result['validation'] == True else '\033[0;31mincorrect\033[0m'))