diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c812299..ba6ceff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,11 +98,6 @@ jobs: - run: pnpm install --frozen-lockfile - run: pnpm --filter api exec prisma migrate deploy - # TEMPORAIRE — recalibration de CONFIDENCE_THRESHOLD (tech-step-matcher.ts) - # contre le vrai service Python démarré ci-dessus. À retirer une fois - # la valeur choisie et reportée dans le code (voir le plan de migration). - - run: pnpm --filter api exec prisma db seed - - run: pnpm --filter api exec tsx src/scripts/calibrate-tech-step-threshold.ts - run: pnpm --filter api test intent-service-test: diff --git a/apps/api/src/lib/recipe-matching/tech-step-matcher.ts b/apps/api/src/lib/recipe-matching/tech-step-matcher.ts index 93c18a0..0bc3267 100644 --- a/apps/api/src/lib/recipe-matching/tech-step-matcher.ts +++ b/apps/api/src/lib/recipe-matching/tech-step-matcher.ts @@ -242,18 +242,31 @@ export function splitIntoClauses( * `TECH_STEP_TRAINING_DATA` — see `test/tech-step-matcher.test.ts` for the * cases this threshold was picked to pass. * - * Raised from `0.65` after finding real (non-adversarial) misclassified - * clauses that scored just above the old threshold — e.g. English recipe - * text run through the French classifier (which must find *nothing*, - * confirmed by `recipe-translation.test.ts`'s own locale-isolation test) - * scored `0.69` for `boil`, essentially classifier noise on - * out-of-vocabulary input rather than a real, confident verdict. The - * clauses this threshold exists to actually trust score far higher in - * practice (`0.91`–`1.0` for the real corrected cases found this session) - * — `0.75` sits comfortably above the noise floor and below every genuine - * match seen so far. + * Recalibrated to `0.45` for the migration off `node-nlp` to + * `services/tech-step-intent-service` (spaCy `textcat`, exclusive classes + * over ~26 techniques) — its score distribution is meaningfully different + * from node-nlp's own classifier, and the old `0.75` (tuned against + * node-nlp) left genuine, correct verdicts on *anchor-less* clauses + * (nothing to fall back to — see this file's doc comment, point 3) below + * the bar: e.g. `melt` scored `0.68` on "jusqu'à ce que le beurre ait + * disparu dans la poêle" (the exact motivating no-keyword case this + * pipeline exists for), `preheat` scored `0.52` on "mettre la poêle sur + * feu vif" — both the classifier's own confident best guess, both correct, + * both wrongly discarded at `0.75`. `0.45` clears both with margin while + * staying far above the noise floor: English recipe text run through the + * French classifier (must find *nothing*, confirmed by + * `recipe-translation.test.ts`'s own locale-isolation test) scored `0.04` + * for every technique — indistinguishable from the ~1/26 uniform baseline + * over this many exclusive classes, nowhere near `0.45`. Cross-checked + * against `apps/api/src/scripts/calibrate-tech-step-threshold.ts`'s sweep + * over `TECH_STEP_EVAL_DATASET`: aggregate F1 climbs to its plateau + * (`0.987`) starting exactly at `0.45` and stays flat through `0.95`, so + * this is the lowest threshold that already captures every gain available + * from trusting the classifier more — a higher value would only ever + * discard more anchor-less true positives like the two above, never buy + * back any precision. */ -export const CONFIDENCE_THRESHOLD = 0.75; +export const CONFIDENCE_THRESHOLD = 0.45; /** * One clause's full classification detail — the finer-grained sibling of