mega commit

libs/validators.py

@@ -1,7 +1,7 @@
 from langchain_ollama.chat_models import ChatOllama
 from langchain_core.prompts import HumanMessagePromptTemplate, ChatPromptTemplate, SystemMessagePromptTemplate
 from langchain.tools import tool
-from libs.test_class import Test
+from libs.classes import Test
 from re import search
 from textwrap import dedent
 
@@ -16,7 +16,7 @@ def system_human_answer_match(test: Test, answer: str, base_url: str) -> bool:
         SystemMessagePromptTemplate.from_template(template=dedent("""You evaluate LLMs. Rate the LLM answer as correct, if the answer is  
         {validation_input}
 
-        else as incorrect. Only use the rate tool. Do not answer conversationally.""")),
+        else as incorrect. Only use the `rate` tool. You do not have accesss to any other tools. Do not answer conversationally.""")),
         HumanMessagePromptTemplate.from_template(template=dedent("""System Message:
         {system_msg}
 
@@ -50,7 +50,10 @@ def system_human_answer_match(test: Test, answer: str, base_url: str) -> bool:
         elif ret_str.lower() == 'false': return False
         else: raise Exception(f"rate tool retured {ret_str}")
     except IndexError as e:
-        print(f"\033[0;31mValidation Error \033[0mof {test.name} <{ai_msg.content[:20]}...> Retrying...")
+        print(f"\033[0;31mValidation Error of\033[0m {test.name} \033[0;31m<\033[0m{ai_msg.content[:20]}\033[0;31m...> Retrying...\033[0m")
+        return system_human_answer_match(test=test, answer=answer, base_url=base_url)
+    except Exception as e:
+        print(f"\033[0;31mValidation Error \033[0mof {test.name} \033[0;31m<\033[0m{e}\033[0;31m> Retrying...\033[0m")
         return system_human_answer_match(test=test, answer=answer, base_url=base_url)
 
 def regex_match_any(test: Test, answer: str, base_url: str) -> bool: