Testing Guide

Abbyfile has three levels of testing: unit tests for individual packages, integration tests against built binaries, and MCP bridge tests for the protocol layer.

Unit Testing Tools

CLI Tools

CLI tool tests use real system commands. The Executor runs them as subprocesses:

func TestExecutor_Run_CLI(t *testing.T) {
    exec := tools.NewExecutor(5*time.Second, nil)
    def := tools.CLI("echo", "echo", "Echo text")
    def.Args = []string{"hello"}

    result, err := exec.Run(context.Background(), def, nil)
    if err != nil {
        t.Fatalf("Run() error: %v", err)
    }
    if result != "hello" {
        t.Errorf("Run() = %q, want %q", result, "hello")
    }
}

Test argument passing via the args input field:

func TestExecutor_Run_WithArgs(t *testing.T) {
    exec := tools.NewExecutor(5*time.Second, nil)
    def := tools.CLI("echo", "echo", "Echo text")

    result, err := exec.Run(context.Background(), def, map[string]any{
        "args": "hello world",
    })
    // result == "hello world"
}

Builtin Tools

Builtin tool handlers are plain functions. Test them directly:

func TestBuiltinTool(t *testing.T) {
    exec := tools.NewExecutor(5*time.Second, nil)
    def := tools.BuiltinTool("test", "A test tool", nil,
        func(input map[string]any) (string, error) {
            return "builtin result", nil
        },
    )

    result, err := exec.Run(context.Background(), def, nil)
    // result == "builtin result"
}

Sandboxed Builtins

The shipped builtins (read_file, run_command, …) are context-aware (HandlerCtx) and read their sandbox from the context. Build a sandbox rooted at a temp dir and call HandlerCtx directly:

func TestReadFile_OutsideDenied(t *testing.T) {
    root, _ := filepath.EvalSymlinks(t.TempDir()) // resolve e.g. macOS /var -> /private/var
    sb, err := sandbox.New(sandbox.Config{}, root) // zero Config = sandbox.Default(): allowed_dirs ["."] = root
    if err != nil {
        t.Fatal(err)
    }
    ctx := sandbox.NewContext(context.Background(), sb)

    var readFile *tools.Definition
    for _, d := range builtins.All() {
        if d.Name == "read_file" {
            readFile = d
        }
    }
    _, err = readFile.HandlerCtx(ctx, map[string]any{"path": "/etc/hosts"})
    // err: path "/etc/hosts" is outside the allowed directories (...)
}

Calling Handler instead runs under the default sandbox with no deadline. See pkg/builtins/sandbox_helpers_test.go and files_sandbox_test.go for the helpers and the full suite.

Timeout Testing 

func TestExecutor_Run_Timeout(t *testing.T) {
    exec := tools.NewExecutor(100*time.Millisecond, nil)
    def := tools.CLI("sleep", "sleep", "Sleep")
    def.Args = []string{"10"}

    _, err := exec.Run(context.Background(), def, nil)
    // err contains "timed out"
}

Input Validation

Test schema validation without running the tool:

func TestValidateInput(t *testing.T) {
    def := &tools.Definition{
        Name: "test_tool",
        InputSchema: map[string]any{
            "type":     "object",
            "required": []any{"key"},
            "properties": map[string]any{
                "key": map[string]any{"type": "string"},
            },
        },
    }

    // Missing required field
    err := def.ValidateInput(map[string]any{})
    // err: missing required field "key"

    // Wrong type
    err = def.ValidateInput(map[string]any{"key": 123.0})
    // err: field "key": expected string, got number
}

See pkg/tools/validate_test.go for the full table-driven test suite.

Unit Testing Memory

Use t.TempDir() for isolated file stores:

func TestMemory(t *testing.T) {
    dir := filepath.Join(t.TempDir(), "memory")
    store, err := memory.NewFileStoreAt(dir, memory.Limits{})
    if err != nil {
        t.Fatal(err)
    }

    store.Write("greeting", "Hello, world!")
    got, _ := store.Read("greeting")
    // got == "Hello, world!"
}

Test limits enforcement:

func TestMemory_MaxKeys(t *testing.T) {
    dir := filepath.Join(t.TempDir(), "memory")
    store, _ := memory.NewFileStoreAt(dir, memory.Limits{MaxKeys: 2})

    store.Write("a", "1")
    store.Write("b", "2")
    err := store.Write("c", "3")
    // err: key count 3 would exceed limit of 2 keys
}

Test the Manager’s built-in tool handlers:

func TestManager_HandleWrite_Read(t *testing.T) {
    store, _ := memory.NewFileStoreAt(t.TempDir(), memory.Limits{})
    mgr := memory.NewManager(store)

    memTools := mgr.Tools()
    toolMap := make(map[string]*tools.Definition)
    for _, tool := range memTools {
        toolMap[tool.Name] = tool
    }

    // Call the write handler directly
    result, err := toolMap["memory_write"].Handler(map[string]any{
        "key":   "test",
        "value": "hello world",
    })
    // result: "Stored 11 bytes under key \"test\""

    // Call the read handler directly
    result, err = toolMap["memory_read"].Handler(map[string]any{"key": "test"})
    // result: "hello world"
}

Unit Testing Prompts

Use t.Setenv("HOME", ...) to isolate override behavior:

func TestLoader_Override(t *testing.T) {
    tmpHome := t.TempDir()
    t.Setenv("HOME", tmpHome)

    overrideDir := filepath.Join(tmpHome, ".abbyfile", "test-agent")
    os.MkdirAll(overrideDir, 0o755)
    os.WriteFile(
        filepath.Join(overrideDir, "override.md"),
        []byte("Override prompt content"),
        0o644,
    )

    loader := prompt.NewLoader("test-agent", testFS, "testdata/system.md")
    got, _ := loader.Load()
    // got == "Override prompt content"
}

MCP Bridge Testing

Use gomcp.NewInMemoryTransports() for in-process client/server testing:

func TestBridge(t *testing.T) {
    registry := tools.NewRegistry()
    registry.Register(tools.BuiltinTool("echo", "Echo input",
        map[string]any{
            "type": "object",
            "properties": map[string]any{
                "message": map[string]any{"type": "string"},
            },
            "required": []string{"message"},
        },
        func(input map[string]any) (string, error) {
            msg, _ := input["message"].(string)
            return "echo: " + msg, nil
        },
    ))

    bridge := mcp.NewBridge(mcp.BridgeConfig{
        Name:     "test-agent",
        Version:  "v0.1.0",
        Registry: registry,
        Executor: tools.NewExecutor(30*time.Second, nil),
        Loader:   loader,
    })

    serverTransport, clientTransport := gomcp.NewInMemoryTransports()

    ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
    defer cancel()

    go func() { bridge.ServeTransport(ctx, serverTransport) }()

    client := gomcp.NewClient(&gomcp.Implementation{
        Name: "test-client", Version: "v0.1.0",
    }, nil)
    session, _ := client.Connect(ctx, clientTransport, nil)
    defer session.Close()

    // List tools
    listResult, _ := session.ListTools(ctx, nil)
    // Verify tool count, names, annotations

    // Call a tool
    callResult, _ := session.CallTool(ctx, &gomcp.CallToolParams{
        Name:      "echo",
        Arguments: map[string]any{"message": "hello"},
    })
    // Verify result content
}

Test memory resources, prompts, and error handling in the same pattern. See pkg/mcp/bridge_test.go for the full test suite.

Agents speak both protocol eras (MCP 2026-07-28 via server/discover, and initialize for 2025-11-25 / 2025-06-18 clients). To pin a client to one era, pass &gomcp.ClientSessionOptions{ProtocolVersion: "2025-11-25"} to client.Connect and check session.InitializeResult().ProtocolVersion. pkg/mcp/conformance_test.go covers this (TestDualEra) and checks that tool, prompt and resource lists don’t change between connections (TestListsInvariantAcrossConnections); pkg/mcp/wire_test.go holds a golden tools/list wire test.

Integration Testing

Integration tests build the binary and exercise all subcommands. They live in internal/integration/ and use the //go:build integration tag.

# Run integration tests
make integration
# Equivalent to:
go test -tags integration -race -count=1 -timeout 120s ./internal/integration/

Integration tests are not included in the normal go test ./... run.

Test Setup

The TestMain function builds the abby CLI once, then uses it to build a test agent from a temp Abbyfile against the local module:

//go:build integration

func TestMain(m *testing.M) {
    // Never touch a real runtime: force the file method (no `claude`/`gemini`
    // CLI calls) and ignore the developer's own runtime config dirs.
    os.Setenv("ABBY_CONFIG_METHOD", "file")
    os.Unsetenv("CLAUDE_CONFIG_DIR")
    os.Unsetenv("CODEX_HOME")

    projectRoot := findProjectRoot()
    tmp, _ := os.MkdirTemp("", "abbyfile-integration-*")
    defer os.RemoveAll(tmp)

    abbyBin = filepath.Join(tmp, "abby-cli")
    cmd := exec.Command("go", "build", "-o", abbyBin, "./cmd/abby")
    cmd.Dir = projectRoot
    cmd.Run()

    // ... write tmp/Abbyfile and tmp/agents/test-agent.md ...

    buildDir := filepath.Join(tmp, "build")
    cmd = exec.Command(abbyBin, "build", "-f", filepath.Join(tmp, "Abbyfile"),
        "-o", buildDir, "--module-dir", projectRoot)
    cmd.Dir = tmp
    cmd.Run()

    binaryPath = filepath.Join(buildDir, "test-agent")
    os.Exit(m.Run())
}

Tests that run abby commands also give each child process its own HOME=<t.TempDir()>, so the registry and user-scope configs (~/.claude.json, ~/.codex/, ~/.gemini/) stay in the temp dir.

Important: Use cmd.Output(), not CombinedOutput()

Agent binaries log to stderr via slog. When parsing stdout output (like JSON from --describe), use cmd.Output() to get only stdout. CombinedOutput() mixes in log lines and corrupts the output.

func runAgentStdout(t *testing.T, args ...string) string {
    cmd := exec.CommandContext(ctx, binaryPath, args...)
    out, err := cmd.Output() // stdout only
    // ...
}

MCP Integration with CommandTransport 

func TestServeMCP(t *testing.T) {
    cmd := exec.CommandContext(ctx, binaryPath, "serve-mcp")
    client := gomcp.NewClient(&gomcp.Implementation{
        Name: "integration-test", Version: "v0.1.0",
    }, nil)

    session, _ := client.Connect(ctx, &gomcp.CommandTransport{Command: cmd}, nil)
    defer session.Close()

    // List tools, call tools, verify responses
}

The make all Pipeline 

make all

Runs four stages in order:

  1. fmtcheck – verify all files are gofmt-formatted
  2. vet – go vet ./... for static analysis
  3. test – go test -race ./... (all unit tests)
  4. build – build the abby CLI

Individual stages:

make fmt          # auto-format all files
make fmtcheck     # check formatting (CI-friendly, fails on unformatted)
make vet          # static analysis
make test         # unit tests with race detector
make build        # build the abby CLI → build/abby
make integration  # integration tests (builds binary first)
make clean        # remove build/, .abbyfile/, .mcp.json, .codex/config.toml, .gemini/settings.json

make clean deletes those runtime config files in the repo root outright, including any entries you added by hand.

CI Setup Patterns

A typical CI workflow:

steps:
  - uses: actions/setup-go@v6
    with:
      go-version-file: go.mod

  - name: Lint and test
    run: make all

  - name: Integration tests
    run: make integration

Key considerations:

  • Always use -race flag (already set in the Makefile)
  • Integration tests need a working Go toolchain (they go build the binary)
  • Memory tests are isolated via t.TempDir() – no filesystem cleanup needed

Plugin Testing

Plugin generation is tested at two levels:

Unit Tests (pkg/plugin/plugin_test.go)

Test the plugin.Generate() function with a fake binary and in-memory definitions:

func TestGenerate_DirectoryStructure(t *testing.T) {
    tmp := t.TempDir()
    binaryPath := filepath.Join(tmp, "my-agent")
    os.WriteFile(binaryPath, []byte("#!/bin/sh\necho hello"), 0o755)

    def := &definition.AgentDef{
        Name:    "my-agent",
        Version: "1.0.0",
    }
    skills := []plugin.SkillFile{
        {Name: "review-pr", Description: "Review a PR", Content: "Review content."},
    }

    outputDir := filepath.Join(tmp, "build")
    os.MkdirAll(outputDir, 0o755)

    err := plugin.Generate(def, skills, plugin.GenerateConfig{
        OutputDir:  outputDir,
        BinaryPath: binaryPath,
    })
    // Verify: .claude-plugin/plugin.json, .mcp.json, binary, skills/review-pr/SKILL.md
}

Tests cover directory structure, plugin.json content, .mcp.json content, SKILL.md frontmatter + body, no-skills case, and binary permissions.

Integration Tests (internal/integration/plugin_test.go)

End-to-end test that builds a real agent with --plugin and verifies the full output:

func TestBuildPlugin(t *testing.T) {
    // Create Abbyfile, agent .md with skills, skill .md files
    // Run: abby build --plugin
    // Verify: plugin directory structure, plugin.json, .mcp.json, SKILL.md content, binary executable
}

Run with make integration.

Distribution Testing

The distribution layer (install, uninstall, list, publish) has integration tests in internal/integration/distribution_test.go:

func TestList(t *testing.T) {
    // Verifies list shows "No agents installed." with empty registry.
    // Uses isolated HOME via t.TempDir().
}

func TestInstallLocalWithRegistry(t *testing.T) {
    // Installs a binary from ./build/, verifies:
    // - Binary copied to .abbyfile/bin/
    // - Registry entry created with source="local" and correct version
    // - abby list shows the agent
}

func TestUninstall(t *testing.T) {
    // Installs, then uninstalls. Verifies:
    // - Binary removed
    // - Registry cleaned up
    // - List shows empty again
}

func TestPublishDryRun(t *testing.T) {
    // Runs publish --dry-run. Verifies:
    // - Cross-compiled binaries created for all 4 targets
    // - No GitHub Release created
}

Distribution tests use HOME override for registry isolation, run with ABBY_CONFIG_METHOD=file, and reuse the test agent binary built by TestMain. The same suite also covers:

  • TestInstallAllLocal (install --all from ./build/)
  • install_config_test.go – install --dry-run --runtime all writes nothing, and install through a fake claude CLI on PATH
  • doctor_test.go – abby doctor after an install, including from an unrelated directory
  • sandbox_test.go – the sandbox end to end through a built agent
  • stdio_test.go – stdout carries only JSON-RPC in both protocol eras, even with sandbox warnings

Unit Testing the Registry 

func TestSaveAndLoad(t *testing.T) {
    path := filepath.Join(t.TempDir(), "registry.json")
    r, _ := registry.Load(path)

    r.Set(registry.Entry{
        Name: "test", Source: "local", Version: "1.0.0",
        Path: "/bin/test", Scope: "local",
    })
    r.Save()

    r2, _ := registry.Load(path)
    e, ok := r2.Get("test")
    // ok == true, e.Version == "1.0.0"
}

Unit Testing the GitHub Client

Tests use httptest to mock the GitHub API:

func TestGetRelease(t *testing.T) {
    srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
        json.NewEncoder(w).Encode(github.Release{
            TagName: "myagent/v1.0.0",
            Assets:  []github.Asset,
        })
    }))
    defer srv.Close()

    c := &github.Client{HTTPClient: srv.Client(), BaseURL: srv.URL}
    ref := github.ReleaseRef{Owner: "owner", Repo: "repo", Agent: "myagent", Version: "1.0.0"}
    release, _ := c.GetRelease(context.Background(), ref)
    // release.TagName == "myagent/v1.0.0"
}

Test Count

The project currently has 500+ unit tests across all packages:

pkg/agent       -- agent creation, options, defaults, sandbox wiring
pkg/tools       -- registry, executor, validation, shaping, spill, process groups
pkg/memory      -- file store, limits, concurrency, manager tools
pkg/prompt      -- loader, override, paths
pkg/builtins    -- builtin tool implementations, sandbox confinement
pkg/sandbox     -- path confinement, command allowlist, argv parsing
pkg/definition  -- Abbyfile + agent .md parsing (skills, context_budget, sandbox)
pkg/builder     -- code generation templates, build notes
pkg/config      -- config.yaml loading and overrides
pkg/mcp         -- bridge, tools, annotations, resources, prompts, protocol eras
pkg/plugin      -- plugin directory generation
pkg/subagent    -- sub-agent file emission
pkg/registry    -- installed agents tracking, atomic save/load
pkg/runtimecfg  -- runtime config writers (CLI + file), JSON/TOML edits, redaction
pkg/fsutil      -- atomic writes, ownership, snapshots
pkg/github      -- GitHub Releases client, checksums, version comparison, ref parsing
internal/cli    -- root command, validate, config, flags
cmd/abby        -- install, update, doctor, checksum, config-change planning

Plus integration tests that exercise the full binary end-to-end, including distribution commands (list, install, uninstall, doctor, publish –dry-run), and benchmarks under benchmarks/ (see make bench).

Abbyfile is an open-source project licensed under MIT.

This site uses Just the Docs, a documentation theme for Jekyll.