Run an A/B evaluation

import re
from langchain.agents import create_agent
from langchain_anthropic import ChatAnthropic
from langchain_core.tools import tool
from reasonblocks import ReasonBlocks

rb = ReasonBlocks(api_key="rb_live_...")
model = ChatAnthropic(model="claude-haiku-4-5-20251001", max_tokens=512)

@tool
def calc(expression: str) -> str:
    """Evaluate a simple arithmetic expression, e.g. '17 * 23'."""
    if not re.fullmatch(r"[0-9+\-*/(). ]+", expression or ""):
        return "invalid expression"
    return str(eval(expression, {"__builtins__": {}}, {}))

# A fixed eval set with checkable answers -> a mechanical outcome label.
TASKS = [("t-1", "compute 17 * 23", "391"), ("t-2", "compute 256 + 389", "645")]

for task_id, prompt, expected in TASKS:
    mw = rb.ab_middleware(
        experiment_id="cust-acme-q2",
        unit_id=task_id,            # stable id -> retries stay in the same arm
        on_fraction=0.5,            # 50/50 split
        agent_name="calc-agent",
        task=prompt,
        org_id="acme",
    )
    agent = create_agent(
        model=model,
        tools=[calc],
        system_prompt="You are a precise calculator. Use the calc tool, then state the result.",
        middleware=[mw],
    )
    outcome = "failure"
    try:
        result = agent.invoke({"messages": [("user", prompt)]})
        final = next(
            (m.content for m in reversed(result["messages"])
             if getattr(m, "content", None) and not getattr(m, "tool_calls", None)),
            "",
        )
        outcome = "success" if expected in str(final) else "failure"
    finally:
        mw.flush_session(outcome_status=outcome)   # records the run's outcome
        mw.close(timeout=10)

curl -sS \
  "https://rb-api.reasonblocks.com/v1/monitor/experiments/cust-acme-q2/report?org_id=acme&on_fraction=0.5" \
  -H "Authorization: Bearer $RB_API_KEY"