Running experiments

agentmark run-experiment agentmark/classifier.prompt.mdx

agentmark run-experiment <filepath> [options]

Options:
  --server <url>          Webhook server URL (default: http://localhost:9417)
  --skip-eval             Skip running evals even if they exist
  --format <format>       Output format: table, csv, json, jsonl, or junit (default: table)
  --threshold <percent>   Fail if pass percentage is below threshold (0-100)
  --truncate <chars>      Truncate long cells in table output (default 1000; 0 = unlimited)
  --concurrency <number>  Dataset rows to run in parallel (default: 20)
  --baseline-commit <ref> Git ref (or tree hash) of a prior run to compare against; enables the
                          regression gate via test_settings.regression_tolerance (see /deploy/regression-gates)

Dataset sampling (pick at most one):
  --sample <percent>    Run on a random N% of rows (1-100)
  --rows <spec>         Select specific rows by index or range (e.g., 0,3-5,9)
  --split <spec>        Train/test split (e.g., train:80 or test:80)
  --seed <number>       Seed for reproducible sampling/splitting

agentmark run-experiment agentmark/test.prompt.mdx --skip-eval

agentmark run-experiment agentmark/test.prompt.mdx --format table   # Default
agentmark run-experiment agentmark/test.prompt.mdx --format csv     # Spreadsheets
agentmark run-experiment agentmark/test.prompt.mdx --format json    # Structured
agentmark run-experiment agentmark/test.prompt.mdx --format jsonl   # Line-delimited
agentmark run-experiment agentmark/test.prompt.mdx --format junit   # JUnit XML for CI gating

agentmark run-experiment agentmark/test.prompt.mdx --threshold 85

agentmark run-experiment agentmark/test.prompt.mdx --format junit > results.xml

agentmark run-experiment agentmark/test.prompt.mdx --sample 20
agentmark run-experiment agentmark/test.prompt.mdx --rows 0,3-5,9
agentmark run-experiment agentmark/test.prompt.mdx --split train:80

agentmark run-experiment agentmark/test.prompt.mdx --server http://staging:9417

# Run on ~20% of rows (random, non-reproducible)
agentmark run-experiment agentmark/test.prompt.mdx --sample 20

# Reproducible: same 20% every time
agentmark run-experiment agentmark/test.prompt.mdx --sample 20 --seed 42

# Row 0 only
agentmark run-experiment agentmark/test.prompt.mdx --rows 0

# Rows 0, 3, 4, 5, and 9
agentmark run-experiment agentmark/test.prompt.mdx --rows 0,3-5,9

# Run on the first 80% (train portion), positional split
agentmark run-experiment agentmark/test.prompt.mdx --split train:80

# Run on the remaining 20% (test portion), positional split
agentmark run-experiment agentmark/test.prompt.mdx --split test:80

# Seeded split — random assignment, reproducible across runs
agentmark run-experiment agentmark/test.prompt.mdx --split train:80 --seed 42
agentmark run-experiment agentmark/test.prompt.mdx --split test:80 --seed 42

# These two runs always process the exact same rows
agentmark run-experiment agentmark/test.prompt.mdx --sample 30 --seed 99
agentmark run-experiment agentmark/test.prompt.mdx --sample 30 --seed 99

✅ Experiment passed threshold check
   Pass rate: 100% (3/3 evaluations passed)
   Threshold: 85%

---
name: sentiment-classifier
test_settings:
  dataset: ./datasets/sentiment.jsonl
  evals:
    - sentiment_check
---

<System>Classify the sentiment</System>
<User>{props.text}</User>

test_settings:
  props:
    language: en
    verbose: false
  dataset: ./datasets/sentiment.jsonl
  evals:
    - sentiment_check

{"input": {"text": "I love this!"}, "expected_output": "positive"}
{"input": {"text": "Terrible product"}, "expected_output": "negative"}
{"input": {"text": "It's okay"}, "expected_output": "neutral"}

agentmark run-experiment agentmark/my-prompt.prompt.mdx

import { client } from './agentmark.client';
import { generateText } from 'ai';  // Or your SDK's generation function

const prompt = await client.loadTextPrompt('classifier.prompt.mdx');

// Returns a stream of formatted inputs from the dataset
const datasetStream = await prompt.formatWithDataset();

// Process each test case
for await (const item of datasetStream) {
  // The stream can yield error chunks ({ error, type: 'error' }) — handle them first
  if (item.type === 'error') {
    console.error(`Dataset error: ${item.error}`);
    continue;
  }

  const { dataset, formatted, evals } = item;

  // Run the prompt with your AI SDK
  const result = await generateText(formatted);

  // Check results
  const passed = result.text === dataset.expected_output;
  console.log(`Input: ${JSON.stringify(dataset.input)}`);
  console.log(`Expected: ${dataset.expected_output}`);
  console.log(`Got: ${result.text}`);
  console.log(`Result: ${passed ? 'PASS' : 'FAIL'}\n`);
}

# List experiments
curl "http://localhost:9418/v1/experiments?limit=10"

# Get a specific experiment, including its runs and evaluation results
curl "http://localhost:9418/v1/experiments/<experimentId>"

# List traces for a specific experiment run — filter `/v1/traces` by `dataset_run_id`
curl "http://localhost:9418/v1/traces?dataset_run_id=<runId>"

# List prompt files registered with the local dev server
curl "http://localhost:9418/v1/prompts?limit=10"

# Same call against AgentMark Cloud — set auth + app headers
curl "https://api.agentmark.co/v1/experiments?limit=10" \
  -H "Authorization: Bearer $AGENTMARK_API_KEY" \
  -H "X-Agentmark-App-Id: $AGENTMARK_APP_ID"