{
  "title": "A bounded AI workflow evaluation",
  "author": "Joey Victorino",
  "date": "2026-11-07",
  "version": "1.0",
  "status": "proposed_not_executed",
  "purpose": "Test whether a candidate improves the full cost of an independently accepted result without violating declared security and operating requirements.",
  "scenario": "An assistant reads a synthetic security ticket and proposes an action through tools restricted to an isolated test environment.",
  "claim": "The candidate reduces cost per accepted result relative to the strongest feasible baseline under the same acceptance rubric and operating constraints.",
  "must_be_fixed_before_running": [
    "Named decision owner and technical owner",
    "Precise allowed actions, prohibited effects and independent evidence checks",
    "Task distribution, rights to the data, held-out sample size and deduplication rules",
    "Quality, security, latency and capacity thresholds for this workload",
    "Equal tuning allowance and declared inference budgets for each arm",
    "Compute, reviewer time, cost ceiling and stopping conditions",
    "Candidate version and ablation that isolates its proposed mechanism"
  ],
  "comparison_arms": [
    {
      "id": "current",
      "description": "Existing implementation and configuration"
    },
    {
      "id": "stronger_model",
      "description": "Competently configured stronger model within the declared budget"
    },
    {
      "id": "candidate",
      "description": "Proposed change, with all unrelated factors fixed where possible"
    },
    {
      "id": "simple_baseline",
      "description": "Deterministic rules or retrieval workflow where applicable"
    }
  ],
  "evaluation_sets": [
    {
      "name": "development",
      "use": "Tuning only; never included in the final held-out estimate"
    },
    {
      "name": "representative_holdout",
      "use": "Paired evaluation under a specified sampling model; report uncertainty and task slices"
    },
    {
      "name": "adversarial",
      "use": "Find failure mechanisms; do not treat adaptive attacks as a representative estimate of production prevalence"
    }
  ],
  "measures": [
    "Independently accepted results / all attempted tasks",
    "Prohibited actions, abstentions, timeouts and missing evidence",
    "End-to-end cost including model calls, tools and reviewer work",
    "Median and p95 latency, throughput and capacity limits",
    "Results by task slice and by repeated run",
    "One-time integration cost and continuing maintenance cost"
  ],
  "controls": [
    "Lock the held-out set and acceptance rubric before tuning",
    "Prevent duplicated cases and common templates crossing data splits",
    "Use the same tasks and record paired outcomes",
    "Choose repeated runs and seeds in advance; record every run",
    "Judge evidence separately from the model's self-report",
    "Ablate the proposed component in the isolated test environment",
    "Keep failed runs and supported negative conclusions"
  ],
  "stop_conditions": [
    "Budget or pre-agreed time ceiling reached",
    "Prohibited action observed: investigate and contain the test before continuing",
    "Required evidence missing: no acceptance decision",
    "Candidate cannot meet the declared quality, security, latency or capacity constraints",
    "The next experiment cannot plausibly change the decision at an acceptable cost"
  ],
  "reproduction_record": [
    "Code revision and dependency lock",
    "Model identifier, weights or provider version where available",
    "Data manifest and hashes with appropriate access controls",
    "Environment, hardware, configuration and random seeds",
    "Raw outcomes, checker verdicts, tool traces and missing evidence",
    "Cost measurements, analysis script and decision record"
  ],
  "adoption_gate": "Technical and decision owners review the evidence, then define a bounded rollout, monitoring thresholds, maintainer and rollback trigger. A positive cost model alone does not authorize deployment.",
  "illustrative_cost_inputs": {
    "monthlyAcceptedTasks": 100000,
    "baselineAttemptCost": 0.12,
    "baselineYield": 0.95,
    "candidateAttemptCost": 0.08,
    "candidateYield": 0.9,
    "researchCost": 30000,
    "monthlyOverhead": 500,
    "horizonMonths": 12
  },
  "illustrative_statistical_inputs": {
    "safetyTrials": 300,
    "observedViolations": 0,
    "riskTolerance": 0.001,
    "confidence": 0.95
  },
  "statistical_limit": "The zero-event binomial bound assumes independent representative trials of a fixed system with complete event detection. It is not valid as a production-rate estimate from an adaptive attack set. If events are observed, the calculator requests investigation instead of applying the zero-event formula. Choose the sample size in advance; repeatedly sampling until a favorable bound appears requires a sequential analysis.",
  "results": null
}