Run an agent in a simulated world

npm install @hue-run/sdk@0.2.0 zod

import { createHue } from "@hue-run/sdk";
import { createEnvironmentClient } from "@hue-run/sdk/environment";
import { createEvaluationClient, runSimulation } from "@hue-run/sdk/evals";

const connection = { apiKey: process.env.HUE_API_KEY! };
const hue = createHue({ ...connection, serviceName: "agent-test", captureContent: true });

try {
  const report = await runSimulation({
    client: createEvaluationClient(connection),
    environmentClient: createEnvironmentClient(connection),
    hue,
    checkpointDirectory: ".hue-checkpoints/refund-scenario",
    scenario: { kind: "experiment", experimentId: process.env.HUE_EXPERIMENT_ID! },
    persistResultContent: true,
    traceEvidence: { mode: "required" },
    // runMyExistingAgent is your application function, not an SDK method.
    target: (inputs, { tools, mcp, config, signal }) =>
      runMyExistingAgent({ inputs, config, tools, mcp, signal }),
    onProgress(event) {
      if (event.type === "run_created") console.log(`Inspect this run: ${event.runUrl}`);
    },
  });
  console.log(report.runUrl);
} finally {
  await hue.shutdownSafe();
}

import { defineLocalScorer } from "@hue-run/sdk/evals";
import type { EnvironmentDefinition } from "@hue-run/sdk/environment";

// A minimal world: one collection and one read action.
const recordsWorld: EnvironmentDefinition = {
  schemaVersion: 1,
  state: { collections: { records: { r_1: { title: "Quarterly summary", status: "draft" } } } },
  actions: [
    {
      name: "list_records",
      description: "List every record in the world.",
      params: [],
      semantics: { entry: "hue.collection.list@1", config: { collection: "records" } },
    },
  ],
};

// Scores the sealed journal: did the agent inspect the world before answering?
const listedRecords = defineLocalScorer({
  source: "listed-records-v1",
  entrypoint: "score",
  metrics: [{ name: "listed_records", type: "boolean" }],
  score(context) {
    const listed =
      context.environment?.steps.some((step) => step.action === "list_records") ?? false;
    return {
      state: "scored",
      metrics: [{ name: "listed_records", value: listed, passed: listed }],
      explanation: listed ? "The agent listed the records." : "The agent never listed the records.",
    };
  },
});

const scenario = {
  kind: "repository" as const,
  name: "Summarize the records",
  slug: "summarize-records",
  environment: { name: "Records fixture", slug: "records-fixture", definition: recordsWorld },
  cases: [
    {
      externalKey: "one-draft",
      inputs: { request: "Summarize every record." },
      metadata: { suite: "reporting" },
    },
  ],
  scorers: [{ name: "Listed records", slug: "listed-records", scorer: listedRecords }],
  config: { agentMode: "reporting" },
};

await runSimulation({
  client,
  environmentClient,
  hue,
  checkpointDirectory: ".hue-checkpoints/summarize-records",
  scenario,
  persistResultContent: false,
  traceEvidence: { mode: "required" },
  target: (inputs, context) => runMyExistingAgent({ inputs, ...context }),
});

import { randomUUID } from "node:crypto";
import { bindEnvironmentTools, createEnvironmentClient } from "@hue-run/sdk/environment";

const client = createEnvironmentClient(connection);
const environment = await client.createEnvironment({ name: "Records fixture", slug: "records-fixture" });
const version = await client.publishVersion(environment.id, recordsWorld);
const run = await client.createRun({ idempotencyKey: randomUUID(), environmentVersionId: version.id });
const tools = bindEnvironmentTools({ hue, client, run });
const observation = await tools.list_records!.execute({});
console.log(observation.status); // "ok" or "error", a recorded world answer either way
await client.finishRun(run.id, { idempotencyKey: randomUUID(), status: "completed" });