Examples

Each example below is a complete, copy-pasteable file.

End-to-end: trace, evaluate, assert, report

# end_to_end.py
import random
from entropy import (Suite, Dataset, Case, trace, event,
                     assert_stable, export_report)

# 1. Agent under test — returns an AgentRun with traced events.
def agent(inp):
    with trace():
        event("action", "tool:search", query=inp)
        event("observation", "top_result", n=5)
        event("action", "reply")
    # 70% of the time we answer correctly, otherwise we drift.
    return "A" if random.random() < 0.7 else "Z"

# 2. Dataset.
dataset = Dataset([
    Case(input="weather", expected="A"),
    Case(input="news", expected="A"),
])

# 3. Monte Carlo evaluation.
results = Suite(seed=42).run(agent, dataset, trials=200)
print("success_rate:", results["success_rate"])
print("behavioral_entropy:", results["behavioral_entropy"])
print("drift_score:", results["drift_score"])

# 4. Regression gate.
assert_stable(results, success_rate__gt=0.5, drift_score__lt=0.3)

# 5. Export an HTML report.
export_report(results, "html", "report.html")
print("wrote report.html")

Evaluating a stochastic agent

A non-deterministic agent that mostly returns "A" but sometimes "B" or "C":

import random
from entropy import Suite, Dataset, Case, assert_stable

def fickle_agent(inp):
    r = random.random()
    if r < 0.6:
        return "A"
    if r < 0.85:
        return "B"
    return "C"

dataset = Dataset([
    Case(input="q1", expected="A"),
    Case(input="q2", expected="B"),
])
results = Suite(seed=42).run(fickle_agent, dataset, trials=300)

for k, v in results.items():
    print(f"  {k}: {v}")

assert_stable(
    results,
    success_rate__gt=0.3,
    behavioral_entropy__gt=0.0,   # non-deterministic -> non-zero entropy
    drift_score__lt=0.2,
)
print("\nassert_stable: OK")

This mirrors examples/basic.py in the repository.

Tracing a tool-using agent

from entropy import trace, event, Suite, Dataset, Case

def agent(inp):
    with trace():
        event("action", "tool:search", query=inp)
        event("observation", "top_result", n=5)
        event("action", "reply")
    return f"searched: {inp}"

dataset = Dataset([Case(input="weather", expected=None)])
res = Suite(seed=0).run(agent, dataset, trials=20)
print(res["tool_reliability"], res["behavioral_entropy"])

Chaos resilience gate

from entropy import ChaosRunner, assert_stable

cr = ChaosRunner(seed=0)
out = cr.run(agent, dataset, ["api_fail", "timeout"], trials=50)
base = out["baseline"]["success_rate"]
for fault, r in out["faults"].items():
    print(f"{fault}: success dropped {base - r['success_rate']:.3f}, "
          f"drift={r['drift_score']:.3f}")

CLI end-to-end

# 1. scaffold
entropy init

# 2. edit agent.py / dataset.json, then run
entropy run --agent agent:agent --dataset dataset.json --trials 200

# 3. lock a baseline, then guard it in CI
entropy test --agent agent:agent --dataset dataset.json --baseline base.json --update
entropy test --agent agent:agent --dataset dataset.json --baseline base.json

LangChain agent (Ollama / minimax-m3)

A real LangChain agent built with create_agent on a local Ollama model, wrapped and evaluated. Mirrors examples/langchain_ollama_e2e.py:

from langchain.agents import create_agent
from langchain_core.tools import tool
from langchain_ollama import ChatOllama
from entropy import from_langgraph, Suite, Dataset, Case, assert_stable

@tool
def get_weather(city: str) -> str:
    return f"It is sunny in {city}."

agent = create_agent(ChatOllama(model="minimax-m3:cloud", temperature=0.0),
                     tools=[get_weather],
                     system_prompt="You are a helpful assistant. Answer concisely.")
wrapped = from_langgraph(agent)   # create_agent -> CompiledStateGraph

dataset = Dataset([
    Case(input="What is the weather in Paris?",
         check=lambda o: "sunny" in (o or "").lower()),
    Case(input="Tell me a joke.",
         check=lambda o: bool(o) and len(o) > 15),
])

results = Suite(seed=42).run(wrapped, dataset, trials=3)
for k, v in results.items():
    print(f"  {k}: {v}")

assert_stable(results, success_rate__gt=0.5, drift_score__lt=0.6)
print("\nassert_stable: OK")