Test Configuration Schema

interface MCPTestConfig {
  name: string                    // Required: Test suite name
  description?: string            // Optional: Test suite description  
  version?: string               // Optional: Test suite version
  server: ServerConfig           // Required: Server configuration
  timeout?: number               // Optional: Global timeout (ms)
  retries?: number              // Optional: Number of retries
  bail?: boolean                // Optional: Stop on first failure
  tests: TestDefinition[]       // Required: Array of tests
  snapshots?: SnapshotConfig    // Optional: Snapshot configuration
  config?: TestSuiteConfig      // Optional: Advanced configuration
}

interface ServerConfig {
  command: string               // Required: Command to run
  args?: string[]              // Optional: Command arguments
  cwd?: string                 // Optional: Working directory
  env?: Record<string, string> // Optional: Environment variables
  timeout?: number             // Optional: Startup timeout (ms)
  killSignal?: string         // Optional: Signal to kill server
  killTimeout?: number        // Optional: Time to wait before force kill
  stdio?: StdioOptions        // Optional: stdio configuration
  shell?: boolean             // Optional: Run in shell
  detached?: boolean          // Optional: Run detached
}

interface TestDefinition {
  name: string                 // Required: Test name
  description?: string         // Optional: Test description
  type: TestType              // Required: Type of test
  timeout?: number            // Optional: Test-specific timeout
  retries?: number           // Optional: Test-specific retries
  skip?: boolean             // Optional: Skip this test
  only?: boolean             // Optional: Only run this test
  
  // Tool-specific properties
  tool?: string              // Tool name (for tool tests)
  arguments?: any            // Tool arguments (for tool tests)
  
  // Resource-specific properties  
  resource?: string          // Resource URI (for resource tests)
  
  // Expectations
  expect?: any               // Expected response
  expectError?: boolean      // Expect an error response
  
  // Snapshot testing
  snapshot?: boolean | SnapshotOptions
}

type TestType = 
  | 'capabilities'    // Test server capabilities
  | 'tools'          // Test tools/list endpoint  
  | 'tool'           // Test tools/call endpoint
  | 'resources'      // Test resources/list endpoint
  | 'resource'       // Test resources/read endpoint
  | 'prompts'        // Test prompts/list endpoint
  | 'prompt'         // Test prompts/get endpoint

interface ExpectationValue {
  // Exact value matching
  value?: any
  
  // Type validation
  type?: 'string' | 'number' | 'boolean' | 'object' | 'array' | 'null'
  
  // Pattern matching
  pattern?: string              // Regular expression
  
  // Range validation (for numbers)
  min?: number
  max?: number
  
  // Array validation
  minLength?: number
  maxLength?: number
  items?: ExpectationValue     // Expected item structure
  
  // Object validation
  properties?: Record<string, ExpectationValue>
  required?: string[]
  
  // Conditional expectations
  oneOf?: ExpectationValue[]
  anyOf?: ExpectationValue[]
  
  // Custom validators
  validator?: string | CustomValidator
  schema?: JSONSchema
}

interface SnapshotConfig {
  updateSnapshots?: boolean     // Update all snapshots
  snapshotDir?: string         // Snapshot directory
  snapshotFormat?: 'json' | 'yaml'
  prettify?: boolean           // Format snapshots nicely
  sortKeys?: boolean          // Sort object keys
  
  // Global filters
  globalExcludes?: string[]    // Fields to exclude globally
  globalIncludes?: string[]    // Fields to include globally
  globalReplacements?: ReplacementPattern[]
}

interface SnapshotOptions {
  inline?: boolean            // Inline snapshot
  name?: string              // Custom snapshot name
  include?: string[]         // Fields to include
  exclude?: string[]         // Fields to exclude
  replacements?: ReplacementPattern[]
  condition?: SnapshotCondition
}

interface ReplacementPattern {
  pattern: string            // Regular expression
  replacement: string        // Replacement text
  flags?: string            // Regex flags
}

interface SnapshotCondition {
  env?: string              // Environment variable name
  value?: string            // Expected value
  platform?: string        // Operating system
}

interface TestSuiteConfig {
  parallel?: boolean           // Run tests in parallel
  maxWorkers?: number         // Number of worker processes
  isolateWorkers?: boolean    // Isolate each test
  setupTimeout?: number       // Setup timeout
  teardownTimeout?: number    // Teardown timeout
  
  // Hooks
  beforeAll?: string          // Script to run before all tests
  afterAll?: string          // Script to run after all tests
  beforeEach?: string        // Script to run before each test
  afterEach?: string         // Script to run after each test
  
  // Reporting
  reporters?: Reporter[]      // Test reporters
  verbose?: boolean          // Verbose output
  colors?: boolean          // Colored output
  
  // Coverage
  coverage?: CoverageConfig  // Coverage configuration
  
  // Custom matchers
  matchers?: MatcherConfig   // Custom matcher configuration
}

interface Reporter {
  name: string              // Reporter name
  outputFile?: string       // Output file path
  outputDir?: string        // Output directory  
  config?: any             // Reporter-specific config
}

interface CoverageConfig {
  enabled: boolean         // Enable coverage
  outputDir: string       // Coverage output directory
  reporters: string[]     // Coverage reporters
  thresholds?: {          // Coverage thresholds
    global?: CoverageThresholds
    'per-file'?: CoverageThresholds
  }
  include?: string[]      // Files to include
  exclude?: string[]      // Files to exclude
}

interface CoverageThresholds {
  branches: number        // Branch coverage %
  functions: number       // Function coverage %
  lines: number          // Line coverage %
  statements: number     // Statement coverage %
}

{
  "name": "Complete MCP Server Test Suite",
  "description": "Comprehensive tests for our MCP server",
  "version": "1.0.0",
  
  "server": {
    "command": "node",
    "args": ["dist/server.js"],
    "env": {
      "NODE_ENV": "test",
      "LOG_LEVEL": "error"
    },
    "timeout": 15000
  },
  
  "timeout": 10000,
  "retries": 1,
  "bail": false,
  
  "snapshots": {
    "snapshotDir": "__snapshots__",
    "prettify": true,
    "globalExcludes": ["timestamp", "id"],
    "globalReplacements": [
      {
        "pattern": "\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z",
        "replacement": "TIMESTAMP"
      }
    ]
  },
  
  "config": {
    "parallel": true,
    "maxWorkers": 2,
    "verbose": true,
    "reporters": [
      "default",
      {
        "name": "junit",
        "outputFile": "test-results.xml"
      }
    ]
  },
  
  "tests": [
    {
      "name": "Server should start correctly",
      "type": "capabilities",
      "expect": {
        "name": "my-server",
        "version": { "type": "string" }
      }
    },
    {
      "name": "Calculator should add numbers",
      "type": "tool",
      "tool": "add",
      "arguments": { "a": 5, "b": 3 },
      "expect": {
        "result": 8
      }
    },
    {
      "name": "Complex response snapshot",
      "type": "tool", 
      "tool": "complex-operation",
      "snapshot": {
        "exclude": ["operationId"],
        "replacements": [
          {
            "pattern": "tmp_\\w+",
            "replacement": "tmp_FILE"
          }
        ]
      }
    }
  ]
}