import { prisma } from "../../db/prisma.js";
import { TECH_STEP_EVAL_DATASET } from "./tech-step-eval-dataset.js";
import {
  computeTechStepMetrics,
  type TechStepEvalOutcome,
  type TechStepEvalResult,
} from "./tech-step-evaluator.js";
import { techStepClassifier } from "./tech-step-matcher.js";

/**
 * Regression floor (not a target) both `runTechStepEvalSuite`'s consumers
 * gate on — exported from here (not defined separately in each consumer)
 * so the CI regression gate and `scripts/retrain-tech-steps.ts`'s
 * pre-backfill gate can never silently drift to different thresholds.
 *
 * Calibrated against a real run: the classifier trained on the corpus as
 * of this constant's introduction scored **0.815** aggregate F1
 * (33 TP / 9 FP / 6 FN) against `TECH_STEP_EVAL_DATASET` — `0.8` leaves a
 * small margin below that for run-to-run noise while still catching a
 * real regression (not a floor picked blind before ever running this
 * suite — see this feature's plan document for that earlier state). The
 * mismatches this run surfaced (e.g. "Blanchissez les haricots verts..."
 * misclassified as `peel`, a handful of anchor-less sentences expected to
 * match nothing instead scoring confidently as some technique) are real,
 * known classifier weaknesses — evidence this harness is doing its job,
 * not something to quietly paper over by loosening the dataset's own
 * expectations. Improving them is corpus work for a future change, gated
 * by this same suite.
 */
export const MIN_OVERALL_F1 = 0.8;

/**
 * Runs {@link TECH_STEP_EVAL_DATASET} against the real, currently-trained
 * `techStepClassifier` and returns the aggregate/per-technique metrics
 * (`computeTechStepMetrics`, `tech-step-evaluator.ts`) — the one place this
 * DB-touching "resolve ids to keys, then score" logic lives, shared by
 * `test/recipe-matching/tech-step-eval.test.ts` (this feature's CI
 * regression gate) and `scripts/retrain-tech-steps.ts` (the same gate, run
 * by a maintainer before applying a corpus change). Kept out of
 * `tech-step-evaluator.ts` itself, which is deliberately pure/DB-free (see
 * that module's own doc comment) so its scoring logic stays unit-testable
 * without a database.
 */
export async function runTechStepEvalSuite(): Promise<TechStepEvalResult> {
  const techSteps = await prisma.techStep.findMany({ select: { id: true, key: true } });
  const keyById = new Map(techSteps.map((techStep) => [techStep.id, techStep.key]));

  const outcomes: TechStepEvalOutcome[] = [];
  for (const evalCase of TECH_STEP_EVAL_DATASET) {
    const techStepIds = await techStepClassifier.matchTechSteps(
      evalCase.description,
      evalCase.locale,
    );
    const actualKeys = techStepIds.map((id) => {
      const key = keyById.get(id);
      // A `techStepId` the classifier resolved that isn't in the seeded
      // catalog would be a bug in the classifier or the seed data, not
      // this dataset — fail loudly rather than silently dropping it (see
      // `_train`'s own comment in `tech-step-matcher.ts` on the
      // equivalent, deliberately silent `undefined` case it has to
      // tolerate for a different reason).
      if (key === undefined) {
        throw new Error(`Unknown TechStep id ${id} returned for "${evalCase.description}"`);
      }
      return key;
    });
    outcomes.push({ expectedKeys: evalCase.expectedKeys, actualKeys });
  }

  return computeTechStepMetrics(outcomes);
}