#!/usr/bin/env node
/**
 * Spec runner: judges `*.spec.md` assertions against the codebase.
 *
 * Each `*.spec.md` file is split into assertions — one per level-2 heading —
 * and every assertion is handed to an LLM judge that reads whatever the
 * assertion implicates and returns a verdict (pass, fail, or refused) with its
 * reasoning. Orchestration is plain code; only the judging is an LLM.
 *
 * Usage:
 *   node spec.js                          judge every **\/*.spec.md
 *   node spec.js skills/testing           narrow to a path or glob
 *   node spec.js a.spec.md --consensus 3  sample 3 judges per assertion
 *
 * Run `node spec.js --help` for the full list of flags.
 */

import { spawn } from "node:child_process";
import { readFileSync, writeFileSync } from "node:fs";
import { dirname, relative, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { globSync } from "glob";
import matter from "gray-matter";

// Child claude processes detect CLAUDECODE and refuse to launch as a nested
// session, so clear it before spawning any. Mirrors eval.js.
delete process.env.CLAUDECODE;

const VERDICTS = ["pass", "fail", "refused"];

const VERDICT_LABELS = {
  pass: "PASS",
  fail: "FAIL",
  refused: "REFUSED",
  error: "ERROR",
};

/**
 * Entry point: load the matched specs, judge every assertion, write the report,
 * and exit non-zero if any assertion failed or errored.
 */
async function main() {
  const specs = loadSpecs(specGlobs);
  announcePlan(specs);
  const fileResults = await judgeSpecs(specs);
  const summary = await writeReport(fileResults);
  reportSummary(summary);
  process.exit(summary.failed > 0 || summary.errored > 0 ? 1 : 0);
}

const HELP = `\
Usage: node spec.js [paths-or-globs...] [options]

Judges every \`*.spec.md\` assertion against the current codebase.

Options:
  --consensus N    Judge calls per assertion; majority wins (default: 1)
  --agree K        Votes required to settle a verdict (default: majority of N)
  --concurrency N  Max judge calls in flight at once (default: 8)
  --retries N      Retries on a malformed judge response (default: 2)
  --report PATH    Also write the markdown report to this path
  --json PATH      Write full results as JSON
  --help           Show this help message`;

const NUMBER_FLAGS = {
  "--consensus": "consensus",
  "--agree": "agree",
  "--concurrency": "concurrency",
  "--retries": "retries",
};

const PATH_FLAGS = {
  "--report": "reportPath",
  "--json": "jsonPath",
};

const DEFAULT_OPTIONS = {
  specGlobs: [],
  consensus: 1,
  agree: 0,
  concurrency: 8,
  retries: 2,
  reportPath: "",
  jsonPath: "",
};

/**
 * Turn raw argv into a validated options object. Folds the flags in via
 * `readFlags`, rejects a `--consensus` below 1, and defaults `--agree` to a
 * strict majority of the sample count when the caller left it unset.
 */
function parseArgs(argv) {
  const options = readFlags(argv, DEFAULT_OPTIONS);
  if (options.consensus < 1) {
    console.error("Error: --consensus must be at least 1.");
    process.exit(1);
  }
  if (options.agree) return options;
  return { ...options, agree: strictMajority(options.consensus) };
}

/** Smallest vote count that is a strict majority of `sampleCount`. */
function strictMajority(sampleCount) {
  return Math.floor(sampleCount / 2) + 1;
}

/**
 * Read flags off the front of `argv`, folding each into a fresh options object:
 * a value flag also consumes the token after it, and any bare argument is a
 * spec path/glob. Recurs on what's left rather than advancing a cursor, so
 * nothing is mutated in place.
 */
function readFlags(argv, options) {
  if (argv.length === 0) return options;
  const [arg, ...rest] = argv;
  if (arg === "--help") {
    console.log(HELP);
    process.exit(0);
  }
  if (arg in NUMBER_FLAGS) {
    const [value, ...remaining] = rest;
    return readFlags(remaining, {
      ...options, [NUMBER_FLAGS[arg]]: Number(value),
    });
  }
  if (arg in PATH_FLAGS) {
    const [value, ...remaining] = rest;
    return readFlags(remaining, {
      ...options, [PATH_FLAGS[arg]]: value,
    });
  }
  return readFlags(rest, {
    ...options, specGlobs: [...options.specGlobs, arg],
  });
}

/**
 * Discover the spec files for `globs` and parse each into assertions, exiting
 * with a message when nothing matches or the matched files hold no assertions.
 */
function loadSpecs(globs) {
  const specPaths = discoverSpecs(globs);
  if (specPaths.length === 0) {
    console.error("Error: no *.spec.md files matched.");
    process.exit(1);
  }
  const specs = specPaths.map((filePath) => ({
    filePath,
    specRelPath: relative(process.cwd(), filePath),
    assertions: parseSpec(filePath),
  }));
  if (countAssertions(specs) === 0) {
    console.error("Error: spec files matched but contain no assertions.");
    process.exit(1);
  }
  return specs;
}

/** Total number of assertions across every spec. */
function countAssertions(specs) {
  return specs.reduce((sum, spec) => sum + spec.assertions.length, 0);
}

/**
 * Print the run plan — file and assertion counts, plus consensus settings — to
 * stderr before judging starts.
 */
function announcePlan(specs) {
  const count = countAssertions(specs);
  console.error(`Specs: ${specs.length} file(s), ${count} assertion(s)`);
  console.error(
    `Consensus: ${consensus} sample(s)` +
    (consensus > 1 ? `, ${agree} to agree` : "") +
    `, concurrency ${concurrency}\n`,
  );
}

// A trailing slash on a directory glob, dropped before the spec suffix.
const TRAILING_SLASH_RE = /\/$/;

/**
 * Expand the caller's paths/globs into spec file paths. A bare directory means
 * "every spec under it"; an explicit file is taken as-is. With no arguments,
 * glob the whole repo. `node_modules` is always excluded.
 */
function discoverSpecs(globs) {
  const patterns = globs.length ? globs : ["**/*.spec.md"];
  const expanded = patterns.flatMap((pattern) => {
    if (pattern.endsWith(".spec.md")) return [pattern];
    return [`${pattern.replace(TRAILING_SLASH_RE, "")}/**/*.spec.md`];
  });
  const matches = globSync(expanded, { ignore: "**/node_modules/**" });
  return [...new Set(matches.map((path) => resolve(path)))].sort();
}

// A level-2 ATX heading (`## …`); capture group 1 is the heading text.
const H2_HEADING_RE = /^##\s+(.*?)\s*$/;

/**
 * Each level-2 heading is an assertion; the prose beneath it (up to the next
 * `##`) is the body the judge uses as context. Content above the first `##`
 * (title, intro) is not an assertion.
 */
function parseSpec(filePath) {
  const { content } = matter(readFileSync(filePath, "utf-8"));
  const assertions = [];
  let current = null;
  for (const line of content.split("\n")) {
    const match = line.match(H2_HEADING_RE);
    if (match) {
      current = { heading: match[1], slug: slugify(match[1]), body: [] };
      assertions.push(current);
      continue;
    }
    if (current) current.body.push(line);
  }
  return assertions.map((assertion) => ({
    heading: assertion.heading,
    slug: assertion.slug,
    body: assertion.body.join("\n").trim(),
  }));
}

// Strip the characters VitePress drops when it slugs a heading, so a slug here
// matches the anchor the rendered page would emit.
const SLUG_STRIP_RE = /[\s~`!@#$%^&*()\-_+=[\]{}|\\;:"'“”‘’<>,.?/]+/g;
// A run of two or more dashes, collapsed to one so separators don't pile up.
const REPEATED_DASHES_RE = /-{2,}/g;
// Leading or trailing dashes, trimmed so a slug never starts or ends with one.
const EDGE_DASHES_RE = /^-+|-+$/g;

/**
 * Matches VitePress's heading-anchor slugs so a slug here lines up with the
 * anchor the rendered page emits.
 */
function slugify(heading) {
  return heading
    .replace(/`/g, "")
    .normalize("NFKD")
    .replace(SLUG_STRIP_RE, "-")
    .replace(REPEATED_DASHES_RE, "-")
    .replace(EDGE_DASHES_RE, "")
    .toLowerCase();
}

/**
 * Judge every assertion across all specs, capping concurrent judge calls, then
 * regroup the verdicts back under their originating spec file.
 */
async function judgeSpecs(specs) {
  const jobs = specs.flatMap((spec) =>
    spec.assertions.map((assertion) => async () => {
      console.error(`  judging: ${spec.specRelPath} › ${assertion.heading}`);
      const result = await judgeAssertion(assertion, spec.specRelPath);
      const label = VERDICT_LABELS[result.verdict];
      console.error(`  ${label}: ${assertion.heading}`);
      return { specFilePath: spec.filePath, result };
    }),
  );
  const judged = await pool(jobs, concurrency);
  return regroupBySpec(specs, judged);
}

/** Regroup judged assertions back under their spec file, preserving order. */
function regroupBySpec(specs, judged) {
  return specs.map((spec) => ({
    specRelPath: spec.specRelPath,
    assertions: judged
      .filter((job) => job.specFilePath === spec.filePath)
      .map((job) => job.result),
  }));
}

/**
 * Sample `consensus` times and settle on the verdict with the most votes. With
 * consensus > 1, a winner that fails to reach the `agree` threshold is reported
 * as "refused" — the judges could not agree, so the assertion is not
 * confidently settled. Ties break toward the outcome that demands attention
 * (fail, then refused, then pass).
 */
async function judgeAssertion(assertion, specRelPath) {
  // A sample that still fails after its retries is an operational error, not a
  // verdict. Drop it and keep the samples that did return; only when every
  // sample fails does the assertion become an "error" — that way one flaky
  // judge call never aborts the whole run or discards the other verdicts.
  const samples = [];
  let remaining = consensus;
  while (remaining-- > 0) {
    try {
      samples.push(await judgeWithRetry(assertion, specRelPath));
    } catch (error) {
      console.error(`  error ${assertion.slug}: ${error.message}`);
    }
  }
  if (samples.length === 0) {
    return {
      ...assertion,
      verdict: "error",
      votes: { pass: 0, fail: 0, refused: 0 },
      samples: [],
      reasoning: "The judge failed to return a valid verdict after retries.",
    };
  }
  const votes = { pass: 0, fail: 0, refused: 0 };
  for (const sample of samples) votes[sample.verdict]++;
  const winner = leadingVerdict(votes);
  const settled = consensus > 1 && votes[winner] < agree ? "refused" : winner;
  // Surface a representative rationale: the first sample that matched the
  // settled verdict, else the first sample overall.
  const match = samples.find((sample) => sample.verdict === settled);
  return {
    ...assertion,
    verdict: settled,
    votes,
    samples,
    reasoning: (match ?? samples[0]).reasoning,
  };
}

/**
 * Verdict with the most votes, breaking ties toward the outcome that demands
 * attention: fail, then refused, then pass.
 */
function leadingVerdict(votes) {
  const order = ["fail", "refused", "pass"];
  let winner = order[0];
  for (const candidate of order) {
    if (votes[candidate] > votes[winner]) winner = candidate;
  }
  return winner;
}

/**
 * Retry on a malformed judge response, which is a transient defect rather than
 * a verdict.
 */
async function judgeWithRetry(assertion, specRelPath) {
  let lastError;
  let attemptsLeft = retries + 1;
  while (attemptsLeft-- > 0) {
    try {
      return await judgeOnce(assertion, specRelPath);
    } catch (error) {
      lastError = error;
      console.error(`  retry ${assertion.slug}: ${error.message}`);
    }
  }
  throw lastError;
}

// A JSON object anywhere in the judge's response (it may wrap the object in
// prose or fences despite instructions to the contrary).
const JSON_OBJECT_RE = /\{[\s\S]*\}/;

/**
 * Run one judge call for an assertion and parse its verdict. Throws when the
 * response carries no JSON object or an unrecognized verdict, so the caller can
 * retry the malformed sample.
 */
async function judgeOnce(assertion, specRelPath) {
  // The prompt is passed first, before the flags: `--allowedTools` is variadic
  // (`<tools...>`) and greedily consumes every trailing argument, so a prompt
  // placed after it would be swallowed as another tool name.
  const stdout = await exec("claude", [
    judgePrompt(assertion, specRelPath),
    "-p",
    "--output-format", "json",
    "--permission-mode", "bypassPermissions",
    "--allowedTools", "Read", "Grep", "Glob",
  ]);
  const text = parseResult(stdout);
  const jsonMatch = text.match(JSON_OBJECT_RE);
  if (!jsonMatch) {
    throw new Error(`no JSON object in judge response: ${text.slice(0, 200)}`);
  }
  const parsed = JSON.parse(jsonMatch[0]);
  if (!VERDICTS.includes(parsed.verdict)) {
    throw new Error(`malformed verdict: ${JSON.stringify(parsed.verdict)}`);
  }
  return { reasoning: parsed.reasoning ?? "", verdict: parsed.verdict };
}

const MODULE_DIR = dirname(fileURLToPath(import.meta.url));

/**
 * The `file()` pattern from skills/nodejs/files/file, copied in and adapted for
 * ESM: read a path relative to this module with a default UTF-8 encoding that
 * callers can override, so it resolves the same no matter where the process is
 * started. The published `@chriscalo/file` resolves its caller through the V8
 * stack as a `file://` URL and never converts it back, so it throws from an ESM
 * module like this one; anchoring on `import.meta.url` is the ESM-correct form.
 */
function file(filePath, options = {}) {
  return readFileSync(resolve(MODULE_DIR, filePath), {
    encoding: "utf-8",
    ...options,
  });
}

// The judge prompt lives in its own file so it reads and edits as prose, not as
// a string literal buried in code. `{{name}}` tokens are filled per assertion.
const JUDGE_PROMPT_TEMPLATE = file("./spec-judge.prompt.md").trimEnd();

/** Fill the judge prompt template with one assertion's fields. */
function judgePrompt(assertion, specRelPath) {
  return fillTemplate(JUDGE_PROMPT_TEMPLATE, {
    specRelPath,
    heading: assertion.heading,
    body: assertion.body || "(no elaboration provided)",
  });
}

const TEMPLATE_TOKEN_RE = /\{\{(\w+)\}\}/g;

/**
 * Replace every `{{name}}` token with `values[name]` in a single pass, so a
 * value that itself contains `{{...}}` is never re-scanned. An unknown token is
 * left intact rather than silently dropped — that surfaces a template typo.
 */
function fillTemplate(template, values) {
  return template.replace(TEMPLATE_TOKEN_RE, (token, name) =>
    name in values ? values[name] : token,
  );
}

/**
 * Spawn a command, collect stdout, and resolve when it exits. detached: true so
 * child claude processes survive this runner's process-group management when it
 * runs inside Claude Code. Mirrors eval.js.
 */
function exec(cmd, cmdArgs, opts = {}) {
  return new Promise((resolveExec, reject) => {
    const child = spawn(cmd, cmdArgs, {
      stdio: ["ignore", "pipe", "pipe"],
      detached: true,
      ...opts,
    });
    let stdout = "";
    let stderr = "";
    child.stdout.on("data", (chunk) => (stdout += chunk));
    child.stderr.on("data", (chunk) => (stderr += chunk));
    child.on("error", reject);
    child.on("exit", (code, signal) => {
      if (signal) return reject(new Error(`${cmd} killed by ${signal}`));
      if (code !== 0) {
        return reject(
          new Error(`${cmd} exited ${code}: ${stderr.slice(0, 500)}`),
        );
      }
      resolveExec(stdout);
    });
  });
}

/**
 * Unwrap the JSON result from `claude -p --output-format json`, falling back to
 * the raw stdout when it is not the expected envelope.
 */
function parseResult(stdout) {
  try {
    const parsed = JSON.parse(stdout);
    return parsed.result || stdout;
  } catch {
    return stdout;
  }
}

/**
 * Run `tasks` (thunks returning promises) at most `limit` at a time so a large
 * run does not spawn hundreds of claude processes at once.
 */
async function pool(tasks, limit) {
  const results = new Array(tasks.length);
  let next = 0;
  async function worker() {
    while (next < tasks.length) {
      const index = next++;
      results[index] = await tasks[index]();
    }
  }
  const workers = Array.from(
    { length: Math.min(limit, tasks.length) },
    () => worker(),
  );
  await Promise.all(workers);
  return results;
}

/**
 * Render the markdown report, print it to stdout, optionally persist it and a
 * JSON dump, and return the verdict tally for the exit-code decision.
 */
async function writeReport(fileResults) {
  const meta = await gitMeta();
  const timestamp = new Date().toISOString();
  const { report, passed, failed, refused, errored } = generateReport(
    fileResults,
    meta,
    timestamp,
  );
  process.stdout.write(report);
  if (reportPath) {
    writeFileSync(resolve(reportPath), report);
    console.error(`\nReport: ${reportPath}`);
  }
  if (jsonPath) {
    const jsonOutput = {
      timestamp,
      commit: meta.commitHash,
      consensus,
      agree,
      summary: { passed, failed, refused, errored },
      specs: fileResults,
    };
    writeFileSync(
      resolve(jsonPath),
      JSON.stringify(jsonOutput, null, 2) + "\n",
    );
    console.error(`JSON: ${jsonPath}`);
  }
  return { passed, failed, refused, errored };
}

/**
 * Commit hash and dirty-tree state for the report header. Unknown outside a git
 * repo rather than fatal — the report still stands on its own.
 */
async function gitMeta() {
  let commitHash = "unknown";
  let uncommitted = "unknown";
  try {
    commitHash = (await exec("git", ["rev-parse", "--short", "HEAD"])).trim();
  } catch { /* not a git repo */ }
  try {
    const status = (await exec("git", ["status", "--porcelain"])).trim();
    const changed = status ? status.split("\n").length : 0;
    uncommitted = changed ? `${changed} files changed` : "none";
  } catch { /* not a git repo */ }
  return { commitHash, uncommitted };
}

/** Build the markdown report and the verdict tally from the judged results. */
function generateReport(fileResults, meta, timestamp) {
  const lines = [
    `# Spec Run — ${timestamp}`,
    "",
    `**Commit:** ${meta.commitHash}`,
    `**Uncommitted changes:** ${meta.uncommitted}`,
    `**Consensus:** ${consensus} sample(s) per assertion` +
      (consensus > 1 ? `, ${agree} to agree` : ""),
  ];
  for (const { specRelPath, assertions } of fileResults) {
    lines.push("", "---", "", `## ${specRelPath}`, "");
    for (const assertion of assertions) {
      const label = VERDICT_LABELS[assertion.verdict];
      lines.push(`### ${label} — ${assertion.heading}`, "");
      if (consensus > 1) {
        lines.push(
          `Votes: ${assertion.votes.pass} pass, ` +
          `${assertion.votes.fail} fail, ${assertion.votes.refused} refused`,
          "",
        );
      }
      lines.push(assertion.reasoning || "(no reasoning returned)", "");
    }
  }
  const tally = tallyVerdicts(fileResults);
  lines.push(...summaryLines(tally));
  return { report: lines.join("\n") + "\n", ...tally };
}

/** Count assertions in each verdict bucket across every spec. */
function tallyVerdicts(fileResults) {
  const all = fileResults.flatMap((file) => file.assertions);
  function count(verdict) {
    return all.filter((assertion) => assertion.verdict === verdict).length;
  }
  return {
    total: all.length,
    passed: count("pass"),
    failed: count("fail"),
    refused: count("refused"),
    errored: count("error"),
  };
}

/** Render the summary table and one-line tally that close the report. */
function summaryLines({ total, passed, failed, refused, errored }) {
  const lines = ["", "---", "", "## Summary", ""];
  lines.push("| Bucket  | Count |");
  lines.push("| ------- | ----- |");
  lines.push(`| Passed  | ${passed} |`);
  lines.push(`| Failed  | ${failed} |`);
  lines.push(`| Refused | ${refused} |`);
  if (errored) lines.push(`| Errored | ${errored} |`);
  lines.push(
    "",
    `${total} assertions: ${passed} passed, ${failed} failed, ` +
    `${refused} refused` + (errored ? `, ${errored} errored` : ""),
  );
  return lines;
}

/** Print the one-line verdict tally to stderr after the report is written. */
function reportSummary({ passed, failed, refused, errored }) {
  console.error(
    `\n${passed} passed, ${failed} failed, ${refused} refused` +
    (errored ? `, ${errored} errored` : "") + ".",
  );
}

const {
  specGlobs, consensus, agree, concurrency, retries, reportPath, jsonPath,
} = parseArgs(process.argv.slice(2));

await main();