Nicolas
0f33aaa76c
Docker etant redevenu disponible dans cette session, j'ai pu lancer pour de vrai la suite Mocha d'apps/api (334/334, y compris les tests Phase 1/2 qui n'avaient pu etre executes precedemment) ainsi que les scripts de la Phase 5 contre une vraie base de test. - tech-step-eval-dataset.ts : corrige un vrai bug d'auteur - "Take the plates..." collisionnait avec le synonyme anglais enregistre "plates" (technique plate), invalidant ce cas negatif. Remplace par "dishes". - tech-step-eval-runner.ts : F1 reel mesure = 0.815 (33 TP / 9 FP / 6 FN). Documente ce chiffre et les vraies erreurs de classification decouvertes (ex: "Blanchissez les haricots verts..." classifie a tort comme "peel") - des faiblesses reelles du classifieur que ce harness est cense detecter, pas a masquer en ajustant le jeu de test. - retrain-tech-steps.ts : le script loggait `appliedIds.length`/ `rejectedIds.length` (ce qui a ete demande) au lieu du `count` reel retourne par `updateMany` (ce qui a vraiment ete modifie) - un id inexistant faisait afficher un faux succes. Decouvert en executant le script pour de vrai avec des ids partiellement invalides. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
71 lines
3.4 KiB
TypeScript
71 lines
3.4 KiB
TypeScript
import { prisma } from "../../db/prisma.js";
|
|
import { TECH_STEP_EVAL_DATASET } from "./tech-step-eval-dataset.js";
|
|
import {
|
|
computeTechStepMetrics,
|
|
type TechStepEvalOutcome,
|
|
type TechStepEvalResult,
|
|
} from "./tech-step-evaluator.js";
|
|
import { techStepClassifier } from "./tech-step-matcher.js";
|
|
|
|
/**
|
|
* Regression floor (not a target) both `runTechStepEvalSuite`'s consumers
|
|
* gate on — exported from here (not defined separately in each consumer)
|
|
* so the CI regression gate and `scripts/retrain-tech-steps.ts`'s
|
|
* pre-backfill gate can never silently drift to different thresholds.
|
|
*
|
|
* Calibrated against a real run: the classifier trained on the corpus as
|
|
* of this constant's introduction scored **0.815** aggregate F1
|
|
* (33 TP / 9 FP / 6 FN) against `TECH_STEP_EVAL_DATASET` — `0.8` leaves a
|
|
* small margin below that for run-to-run noise while still catching a
|
|
* real regression (not a floor picked blind before ever running this
|
|
* suite — see this feature's plan document for that earlier state). The
|
|
* mismatches this run surfaced (e.g. "Blanchissez les haricots verts..."
|
|
* misclassified as `peel`, a handful of anchor-less sentences expected to
|
|
* match nothing instead scoring confidently as some technique) are real,
|
|
* known classifier weaknesses — evidence this harness is doing its job,
|
|
* not something to quietly paper over by loosening the dataset's own
|
|
* expectations. Improving them is corpus work for a future change, gated
|
|
* by this same suite.
|
|
*/
|
|
export const MIN_OVERALL_F1 = 0.8;
|
|
|
|
/**
|
|
* Runs {@link TECH_STEP_EVAL_DATASET} against the real, currently-trained
|
|
* `techStepClassifier` and returns the aggregate/per-technique metrics
|
|
* (`computeTechStepMetrics`, `tech-step-evaluator.ts`) — the one place this
|
|
* DB-touching "resolve ids to keys, then score" logic lives, shared by
|
|
* `test/recipe-matching/tech-step-eval.test.ts` (this feature's CI
|
|
* regression gate) and `scripts/retrain-tech-steps.ts` (the same gate, run
|
|
* by a maintainer before applying a corpus change). Kept out of
|
|
* `tech-step-evaluator.ts` itself, which is deliberately pure/DB-free (see
|
|
* that module's own doc comment) so its scoring logic stays unit-testable
|
|
* without a database.
|
|
*/
|
|
export async function runTechStepEvalSuite(): Promise<TechStepEvalResult> {
|
|
const techSteps = await prisma.techStep.findMany({ select: { id: true, key: true } });
|
|
const keyById = new Map(techSteps.map((techStep) => [techStep.id, techStep.key]));
|
|
|
|
const outcomes: TechStepEvalOutcome[] = [];
|
|
for (const evalCase of TECH_STEP_EVAL_DATASET) {
|
|
const techStepIds = await techStepClassifier.matchTechSteps(
|
|
evalCase.description,
|
|
evalCase.locale,
|
|
);
|
|
const actualKeys = techStepIds.map((id) => {
|
|
const key = keyById.get(id);
|
|
// A `techStepId` the classifier resolved that isn't in the seeded
|
|
// catalog would be a bug in the classifier or the seed data, not
|
|
// this dataset — fail loudly rather than silently dropping it (see
|
|
// `_train`'s own comment in `tech-step-matcher.ts` on the
|
|
// equivalent, deliberately silent `undefined` case it has to
|
|
// tolerate for a different reason).
|
|
if (key === undefined) {
|
|
throw new Error(`Unknown TechStep id ${id} returned for "${evalCase.description}"`);
|
|
}
|
|
return key;
|
|
});
|
|
outcomes.push({ expectedKeys: evalCase.expectedKeys, actualKeys });
|
|
}
|
|
|
|
return computeTechStepMetrics(outcomes);
|
|
}
|