nx2/test-small-llms
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

more error feedback

Browse Source
This commit is contained in:
Lennart J. Kurzweg (Nx2)
2024-08-30 23:05:07 +02:00
parent e3df5d3e78
commit 065fff21e6
1 changed files with 19 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
libs/runnables.py
View File

@@ -87,25 +87,27 @@ def one_tool_call_answer(model: Model, seed: int, test: Test, technique: Techniq
tool_msg = selected_tool.invoke(call)
except KeyError:
tool_msg = SystemMessage(f"Tool '{call['name'].lower()}' does not exist. Available are {tools_dict.keys()}")
messages.append(tool_msg)
ai_msg = llm.invoke(messages)
i = 0
while isinstance(ai_msg, SystemMessage):
i += 1
if i <= 5:
return {
"answer": ">>LLM failed to use tools<<",
"tool_calls": tool_calls,
}
messages.append(ai_msg)
except Exception as e:
tool_msg = SystemMessage(f"Tool '{call['name'].lower()}' returned a input validation error:" + "\n" + str(e))
finally:
messages.append(tool_msg)
ai_msg = llm.invoke(messages)
i = 0
while isinstance(ai_msg, SystemMessage):
i += 1
if i <= 5:
return {
"answer": ">>LLM failed to use tools<<",
"tool_calls": tool_calls,
}
messages.append(ai_msg)
ai_msg = llm.invoke(messages)
tool_calls.append({
"tool": call["name"],
"args": call["args"],
"times_failed": i
})
tool_calls.append({
"tool": call["name"],
"args": call["args"],
"times_failed": i
})
except IndexError: # LLM didnt use a tool -> jsut return the content
tool_calls = []
if len(ai_msg.tool_calls) > 0: