batchCooking/apps/api/src/scripts/calibrate-tech-step-threshold.ts

import { prisma } from "../db/prisma.js";
import { TECH_STEP_EVAL_DATASET } from "../lib/recipe-matching/tech-step-eval-dataset.js";
import {
  computeTechStepMetrics,
  type TechStepEvalOutcome,
} from "../lib/recipe-matching/tech-step-evaluator.js";
import { techStepClassifier } from "../lib/recipe-matching/tech-step-matcher.js";

/**
 * Candidate thresholds to sweep, `0.05` to `0.95` in `0.05` steps — fine
 * enough to find a good value without an unreasonable number of full
 * `TECH_STEP_EVAL_DATASET` passes (each threshold only needs one
 * {@link techStepClassifier.classifyClauses} call per eval case, not a
 * retrain — see this file's own doc comment for why).
 */
const CANDIDATE_THRESHOLDS = Array.from({ length: 19 }, (_, i) => Math.round((i + 1) * 5) / 100);

/**
 * One-off maintainer tool for recalibrating `CONFIDENCE_THRESHOLD`
 * (`tech-step-matcher.ts`) after a change to the underlying intent
 * classifier — most notably, the migration from `node-nlp` to
 * `services/tech-step-intent-service` (spaCy): a different model produces a
 * differently-shaped confidence score distribution, so a threshold tuned
 * against the old classifier has no reason to still be the right cutoff for
 * the new one.
 *
 * Reuses `techStepClassifier.classifyClauses` — already public, and
 * deliberately *not* threshold-applied (see that method's own doc comment)
 * — to get every eval case's raw `{anchorUid, intentUid, score}` per clause
 * exactly once, then replays `_classifyClause`'s own decision rule
 * (`intentUid` if confident enough, `anchorUid` otherwise) locally in this
 * script for every candidate threshold. This is what makes a full sweep
 * cheap: one classifier pass per eval case regardless of how many
 * thresholds are being compared, rather than one full pass *per threshold*.
 *
 * Prints a threshold -> precision/recall/F1 table and the threshold that
 * maximizes aggregate F1 — does **not** edit `tech-step-matcher.ts` itself.
 * A maintainer reads the table, updates `CONFIDENCE_THRESHOLD` by hand (with
 * an updated doc comment recording what run/F1 the new value was calibrated
 * against, same as the existing comment's own format), then re-runs
 * `retrain-tech-steps.ts` to confirm the change clears `MIN_OVERALL_F1`.
 *
 * Usage:
 *
 *   pnpm --filter api exec tsx src/scripts/calibrate-tech-step-threshold.ts
 */
async function calibrateTechStepThreshold(): Promise<void> {
  console.info(`Classifying ${TECH_STEP_EVAL_DATASET.length} eval case(s)...`);

  // One classifier pass per eval case, all clauses' raw verdicts kept
  // alongside the case's own `expectedKeys` — reused for every candidate
  // threshold in the loop below.
  const casesWithClauses = await Promise.all(
    TECH_STEP_EVAL_DATASET.map(async (evalCase) => ({
      expectedKeys: evalCase.expectedKeys,
      clauses: await techStepClassifier.classifyClauses(evalCase.description, evalCase.locale),
    })),
  );

  console.info("\nthreshold  precision  recall     f1");
  let bestThreshold = CANDIDATE_THRESHOLDS[0] ?? 0;
  let bestF1 = -1;

  for (const threshold of CANDIDATE_THRESHOLDS) {
    const outcomes: TechStepEvalOutcome[] = casesWithClauses.map(({ expectedKeys, clauses }) => {
      const actualKeys = clauses
        // Mirrors `_classifyClause`'s own decision rule exactly (see that
        // method, `tech-step-matcher.ts`) — the classifier's own verdict
        // when confident enough, otherwise its clause's NER anchor, `null`
        // when neither applies (no keyword, no confident classification).
        .map((clause) =>
          clause.intentUid !== null && clause.score >= threshold
            ? clause.intentUid
            : clause.anchorUid,
        )
        .filter((key): key is string => key !== null);
      return { expectedKeys, actualKeys };
    });

    const { overall } = computeTechStepMetrics(outcomes);
    console.info(
      `${threshold.toFixed(2)}       ${overall.precision.toFixed(3)}      ${overall.recall.toFixed(3)}      ${overall.f1.toFixed(3)}`,
    );
    if (overall.f1 > bestF1) {
      bestF1 = overall.f1;
      bestThreshold = threshold;
    }
  }

  console.info(
    `\nBest aggregate F1 ${bestF1.toFixed(3)} at threshold ${bestThreshold.toFixed(2)} — update CONFIDENCE_THRESHOLD in tech-step-matcher.ts by hand if this differs from the current value.`,
  );
}

calibrateTechStepThreshold()
  .then(() => prisma.$disconnect())
  .catch(async (err) => {
    console.error(err);
    await prisma.$disconnect();
    process.exit(1);
  });