Single Tool Reliability

Example showing how to evaluate reliability of single tool calls.

1. Create a Python file

   reliability_single_tool.py

   from typing import Optional
   
   from agno.agent import Agent
   from agno.eval.reliability import ReliabilityEval, ReliabilityResult
   from agno.models.openai import OpenAIResponses
   from agno.run.agent import RunOutput
   from agno.tools.calculator import CalculatorTools
   
   
   def factorial():
       agent = Agent(
           model=OpenAIResponses(id="gpt-5.2"),
           tools=[CalculatorTools()],
       )
       response: RunOutput = agent.run("What is 10!?")
       evaluation = ReliabilityEval(
           name="Tool Call Reliability",
           agent_response=response,
           expected_tool_calls=["factorial"],
       )
       result: Optional[ReliabilityResult] = evaluation.run(print_results=True)
       result.assert_passed()
   
   
   if __name__ == "__main__":
       factorial()
   
2. Set up your virtual environment

   Mac

   uv venv --python 3.12
   source .venv/bin/activate
   
3. Install dependencies

   uv pip install -U openai agno
   
4. Export your OpenAI API key

   Mac/Linux

     export OPENAI_API_KEY="your_openai_api_key_here"
   
5. Run Agent

   python reliability_single_tool.py