diff --git a/.gitignore b/.gitignore index 6d92b7c..93666d2 100644 --- a/.gitignore +++ b/.gitignore @@ -71,6 +71,12 @@ web_modules/ !.env.example !.env.test.example +# node-nlp's default auto-save file (apps/api/src/lib/recipe-matching/ +# tech-step-matcher.ts explicitly disables autoSave/autoLoad, but this is a +# belt-and-suspenders guard against it ever reappearing — a stale trained +# model on disk must never silently shadow TECH_STEP_TRAINING_DATA). +model.nlp + # parcel-bundler cache (https://parceljs.org/) .cache .parcel-cache diff --git a/apps/api/package.json b/apps/api/package.json index 8ccf511..1147e02 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -26,6 +26,7 @@ "dotenv": "^16.4.5", "express": "^4.21.1", "jsonwebtoken": "^9.0.3", + "node-nlp": "4.27.0", "prisma": "^5.22.0", "zod": "^3.23.8" }, diff --git a/apps/api/prisma/migrations/20260821130000_drop_tech_step_mapping/migration.sql b/apps/api/prisma/migrations/20260821130000_drop_tech_step_mapping/migration.sql new file mode 100644 index 0000000..3e3f219 --- /dev/null +++ b/apps/api/prisma/migrations/20260821130000_drop_tech_step_mapping/migration.sql @@ -0,0 +1,6 @@ +-- DropForeignKey +ALTER TABLE "tech_step_mapping" DROP CONSTRAINT "tech_step_mapping_tech_step_id_fkey"; + +-- DropTable +DROP TABLE "tech_step_mapping"; + diff --git a/apps/api/prisma/migrations/20260821140000_step_tech_step_context/migration.sql b/apps/api/prisma/migrations/20260821140000_step_tech_step_context/migration.sql new file mode 100644 index 0000000..e417c31 --- /dev/null +++ b/apps/api/prisma/migrations/20260821140000_step_tech_step_context/migration.sql @@ -0,0 +1,4 @@ +-- AlterTable +ALTER TABLE "step_tech_step" ADD COLUMN "context_end" INTEGER, +ADD COLUMN "context_start" INTEGER; + diff --git a/apps/api/prisma/schema.prisma b/apps/api/prisma/schema.prisma index 9cc9ad8..f1cf994 100644 --- a/apps/api/prisma/schema.prisma +++ b/apps/api/prisma/schema.prisma @@ -619,36 +619,26 @@ model RecipeIngredient { /// camelCase uid (e.g. `"simmer"`), not the display label — the French /// label lives in `apps/web`'s `locales/fr/translation.json` under /// `catalog.techSteps.` (see `reference-seed-data.ts`'s `TECH_STEPS`). +/// +/// Matching a step's free text against these (`tech-step-matcher.ts`'s +/// `TechStepClassifierService`) used to go through a DB-backed +/// `TechStepMapping` table of per-locale regex expressions — replaced with +/// a node-nlp model trained from in-code data +/// (`tech-step-training-data.ts`) once regexes turned out unable to +/// generalize past their own literal vocabulary. Nothing queries/edits +/// that matching data at runtime anymore (it only ever feeds the +/// classifier's one-time training pass), so it no longer needs a table of +/// its own — this row now only exists to be a stable id/key other tables +/// (`StepTechStep`) reference. model TechStep { id Int @id @default(autoincrement()) key String @unique - steps StepTechStep[] - mappings TechStepMapping[] + steps StepTechStep[] @@map("tech_step") } -/// Used by `tech-step-matcher.ts` to auto-detect which technique a recipe -/// step's description corresponds to (expression = regex pattern tested -/// against the description, weight = tie-break score when several -/// mappings match, or overlap-resolution score when two mappings match the -/// same span of text — see `matchTechSteps`). `locale` (e.g. `"fr"`) lets -/// the same TechStep carry one matching rule set per language — the -/// matcher is always called with a target locale and only considers -/// mappings for that locale. -model TechStepMapping { - id Int @id @default(autoincrement()) - techStepId Int @map("tech_step_id") - locale String - expression String - weight Int - - techStep TechStep @relation(fields: [techStepId], references: [id], onDelete: Cascade) - - @@map("tech_step_mapping") -} - /// Modeled as one-to-many (a step belongs to exactly one recipe), not the /// many-to-many noted in the spec doc: `order` only makes sense scoped to a /// single recipe, which isn't reconcilable with steps being shared across @@ -676,22 +666,30 @@ model Step { /// techniques in the description), not a global ordering across different /// steps of the recipe (that's `Step.order`). /// -/// `start`/`end` are the matched span within `Step.description` (see -/// `TechStepMatch`, `tech-step-matcher.ts`) — what the recipe detail view -/// highlights. Nullable, **not backfilled**: adding them `NOT NULL` without -/// a default would fail outright against any pre-existing row, the same -/// mistake the `ingredient_unit_catalog` migration made against real prod -/// data. A row from before this column existed just has no span (no -/// highlight) until its recipe is next saved, which recomputes every step's +/// `start`/`end` are the tight matched *keyword* span within +/// `Step.description` (see `TechStepMatch`, `tech-step-matcher.ts`) — what +/// the recipe detail view highlights strongly, with a tooltip. +/// `contextStart`/`contextEnd` are the wider *clause* the keyword was found +/// in (e.g. "Dans une poêle chaude" around a `preheat` keyword of "poêle +/// chaude") — always contains `start`/`end` — what the detail view +/// highlights more subtly around it, so both "the exact trigger word(s)" +/// and "how much of the sentence is about this technique" are visible. +/// Nullable, **not backfilled**: adding them `NOT NULL` without a default +/// would fail outright against any pre-existing row, the same mistake the +/// `ingredient_unit_catalog` migration made against real prod data. A row +/// from before a column existed just has no span for it (no highlight) +/// until its recipe is next saved, which recomputes every step's /// techniques from scratch (`recipe.service.ts`'s `updateRecipe` deletes /// and recreates every `Step`/`StepTechStep`, never a partial patch) — /// graceful degradation, not a permanent gap. model StepTechStep { - stepId Int @map("step_id") - techStepId Int @map("tech_step_id") - order Int - start Int? - end Int? + stepId Int @map("step_id") + techStepId Int @map("tech_step_id") + order Int + start Int? + end Int? + contextStart Int? @map("context_start") + contextEnd Int? @map("context_end") step Step @relation(fields: [stepId], references: [id], onDelete: Cascade) techStep TechStep @relation(fields: [techStepId], references: [id], onDelete: Cascade) diff --git a/apps/api/src/db/prisma.ts b/apps/api/src/db/prisma.ts index f54b78c..7792956 100644 --- a/apps/api/src/db/prisma.ts +++ b/apps/api/src/db/prisma.ts @@ -1,3 +1,22 @@ +// Imported for its side effect only (loading `.env`/`.env.test` via +// dotenv) — must run *before* `new PrismaClient()` below. The generated +// Prisma Client bakes in its own fallback `.env` path (always +// `apps/api/.env`, the dev one — resolved once at `prisma generate` time) +// and loads it internally the first time a `PrismaClient` is constructed, +// unless `DATABASE_URL` is already set in `process.env` by then — dotenv +// never overrides an already-set variable, so whichever of these two env +// loads runs first "wins" for the rest of the process. Without this +// import, that race depended entirely on which test file some *other* +// module happened to import first, which normally worked out only by +// coincidence (whatever file mocha's `test/**/*.test.ts` glob happens to +// resolve first) — running a single test file in isolation (e.g. `mocha +// test/some-file.test.ts` directly, bypassing that glob) could silently +// resolve `DATABASE_URL` to the real dev database instead of +// `.env.test`'s. `resetDatabase()`'s own `assertRunningAgainstTestDatabase` +// guard (test-support/reset-db.ts) is what actually caught this in +// practice — it throws rather than truncating the wrong database — but +// the fix belongs here, at the source, not just at that one call site. +import "../config/env.js"; import { PrismaClient } from "@prisma/client"; /** diff --git a/apps/api/src/db/reference-seed-data.ts b/apps/api/src/db/reference-seed-data.ts index cee9030..ac5cf75 100644 --- a/apps/api/src/db/reference-seed-data.ts +++ b/apps/api/src/db/reference-seed-data.ts @@ -51,245 +51,49 @@ export const UNITS: Array<{ uid: string; type: UnitType; toBaseFactor: number }> { uid: "pound", type: "MASS", toBaseFactor: 453.5924 }, ]; -// Cooking-technique catalog (French recipe-step normalization) — a static -// list of common instructions, each carrying one or more text-matching -// rules used by `tech-step-matcher.ts` to auto-detect which technique(s) a -// free-text `Step.description` corresponds to (a step can mention several, -// e.g. "faire chauffer une poêle puis y faire fondre le beurre" is both -// `preheat` and `melt` — see `Step.techSteps`/`StepTechStep` in -// schema.prisma). Same "English camelCase uid, no French label" authoring -// as DIETS/UNITS — the label lives in apps/web's -// locales/fr/translation.json under `catalog.techSteps.`. -// `expression` is a regex source matched (case/accent-insensitive, via -// `normalizeText`) against the step description; `weight` breaks ties when -// two *different* techniques' expressions match the same span of text -// (highest weight wins) — see `tech-step-matcher.ts`'s `matchTechSteps`. -// Specific, multi-word phrases ("cuire au four", "faire revenir") are -// weighted higher than the generic single-verb forms they overlap with -// ("cuire", "sauter") so the more specific technique wins when both match -// the same words. `locale` lets the same technique carry one matching rule -// set per language — `"fr"` and `"en"` today (the latter mainly for -// English-language sources like TheMealDB), more can be added later -// without a schema change. The two locales are independent rule sets, not -// translations of each other — an English recipe is matched only against -// the `"en"` mappings, never a mix of both. -export const TECH_STEPS: Array<{ - uid: string; - mappings: Array<{ locale: string; expression: string; weight: number }>; -}> = [ - { - uid: "cook", - mappings: [ - { locale: "fr", expression: "\\bcui(re|sez|sant|sson)\\b|\\bcuit(e|es|s)?\\b", weight: 10 }, - { locale: "en", expression: "\\bcook(s|ed|ing)?\\b", weight: 10 }, - ], - }, - { - uid: "fry", - mappings: [ - { locale: "fr", expression: "\\bfri(re|t|te|ts|tes|ture)\\b", weight: 15 }, - { locale: "en", expression: "\\bfr(y|ies|ied|ying)\\b", weight: 15 }, - ], - }, - { - uid: "melt", - mappings: [ - { - locale: "fr", - expression: - "\\bfondre\\b|\\bfondu(e|es|s)?\\b|\\bfaire fondre\\b|\\bfaites fondre\\b|\\bfaire chauffer\\b|\\bfaites chauffer\\b", - weight: 15, - }, - { locale: "en", expression: "\\bmelt(s|ed|ing)?\\b", weight: 15 }, - ], - }, - { - uid: "deglaze", - mappings: [ - { locale: "fr", expression: "\\bd[ée]glac(er|ez|é|ée|age)\\b", weight: 20 }, - { locale: "en", expression: "\\bdeglaz(e|es|ed|ing)\\b", weight: 20 }, - ], - }, - { - uid: "simmer", - mappings: [ - { locale: "fr", expression: "\\bmijot(er|ez|e|ant|é)\\b", weight: 15 }, - { locale: "en", expression: "\\bsimmer(s|ed|ing)?\\b", weight: 15 }, - ], - }, - { - uid: "boil", - mappings: [ - { locale: "fr", expression: "\\bbouill(ir|ant|ie|ies)\\b|\\b[ée]bullition\\b", weight: 12 }, - { locale: "en", expression: "\\bboil(s|ed|ing)?\\b", weight: 12 }, - ], - }, - { - uid: "roast", - mappings: [ - { locale: "fr", expression: "\\br[ôo]tir\\b|\\br[ôo]ti(e|es|s)?\\b", weight: 15 }, - { locale: "en", expression: "\\broast(s|ed|ing)?\\b", weight: 15 }, - ], - }, - { - uid: "grill", - mappings: [ - { locale: "fr", expression: "\\bgrill(er|ez|é|ée|ées|ade)\\b", weight: 15 }, - { locale: "en", expression: "\\bgrill(s|ed|ing)?\\b", weight: 15 }, - ], - }, - { - uid: "panFry", - mappings: [ - { locale: "fr", expression: "\\bsaut(er|ez|é|ée|ées|ant)\\b", weight: 12 }, - { - locale: "en", - expression: "\\bsaut[ée](s|ed|ing)?\\b|\\bpan[- ]?fr(y|ies|ied|ying)\\b", - weight: 12, - }, - ], - }, - { - uid: "blanch", - mappings: [ - { locale: "fr", expression: "\\bblanch(ir|issez|i|ie|ies|iment)\\b", weight: 18 }, - { locale: "en", expression: "\\bblanch(es|ed|ing)?\\b", weight: 18 }, - ], - }, - { - uid: "marinate", - mappings: [ - { locale: "fr", expression: "\\bmarin(er|ez|é|ée|ées|ade)\\b", weight: 18 }, - { locale: "en", expression: "\\bmarinat(e|es|ed|ing)\\b|\\bmarinad(e|es)\\b", weight: 18 }, - ], - }, - { - uid: "chop", - mappings: [ - { locale: "fr", expression: "\\bhach(er|ez|é|ée|ées|is)\\b", weight: 15 }, - { locale: "en", expression: "\\bchop(s|ped|ping)?\\b", weight: 15 }, - ], - }, - { - uid: "peel", - mappings: [ - { locale: "fr", expression: "\\b[ée]pluch(er|ez|é|ée|ées|age)\\b", weight: 15 }, - { locale: "en", expression: "\\bpeel(s|ed|ing)?\\b", weight: 15 }, - ], - }, - { - uid: "mince", - mappings: [ - { locale: "fr", expression: "\\b[ée]minc(er|ez|é|ée|ées)\\b", weight: 18 }, - { locale: "en", expression: "\\bminc(e|es|ed|ing)\\b", weight: 18 }, - ], - }, - { - uid: "mix", - mappings: [ - { locale: "fr", expression: "\\bm[ée]lang(er|ez|é|ée|ées|e|es)\\b", weight: 10 }, - { locale: "en", expression: "\\bmix(es|ed|ing)?\\b|\\bcombine(s|d)?\\b", weight: 10 }, - ], - }, - { - uid: "whisk", - mappings: [ - { locale: "fr", expression: "\\bfouett(er|ez|é|ée|ées)\\b|\\bau fouet\\b", weight: 15 }, - { locale: "en", expression: "\\bwhisk(s|ed|ing)?\\b", weight: 15 }, - ], - }, - { - uid: "foldIn", - mappings: [ - { locale: "fr", expression: "\\bincorpor(er|ez|é|ée|ées|ant)\\b", weight: 15 }, - { locale: "en", expression: "\\bfold(s|ed|ing)? in\\b", weight: 15 }, - ], - }, - { - uid: "setAside", - mappings: [ - { locale: "fr", expression: "\\br[ée]serv(er|ez|é|ée|ées)\\b", weight: 15 }, - { locale: "en", expression: "\\bset(s)? aside\\b|\\bsetting aside\\b", weight: 15 }, - ], - }, - { - uid: "season", - mappings: [ - { locale: "fr", expression: "\\bassaisonn(er|ez|é|ée|ées|ement)\\b", weight: 15 }, - { locale: "en", expression: "\\bseason(s|ed|ing)?\\b", weight: 15 }, - ], - }, - { - uid: "drain", - mappings: [ - { locale: "fr", expression: "\\b[ée]goutt(er|ez|é|ée|ées)\\b", weight: 15 }, - { locale: "en", expression: "\\bdrain(s|ed|ing)?\\b", weight: 15 }, - ], - }, - { - uid: "brown", - mappings: [ - { - locale: "fr", - expression: - "\\bfaire revenir\\b|\\bfaites revenir\\b|\\bfais revenir\\b|\\bfaire dorer\\b|\\bfaites dorer\\b", - weight: 25, - }, - // Verb forms only (not bare "brown"), which would false-positive on - // ingredient descriptions like "brown sugar"/"brown rice". - { locale: "en", expression: "\\bbrown(ed|ing)\\b", weight: 25 }, - ], - }, - { - uid: "rest", - mappings: [ - { locale: "fr", expression: "\\blaiss(er|ez|e) reposer\\b|\\breposer\\b", weight: 20 }, - // Anchored to "let ... rest"/"rest for" rather than bare "rest", - // which would false-positive on phrases like "the rest of the". - { - locale: "en", - expression: "\\blet (it |them )?rest\\b|\\brest(s|ed|ing)? for\\b", - weight: 20, - }, - ], - }, - { - uid: "preheat", - mappings: [ - { locale: "fr", expression: "\\bpr[ée]chauff(er|ez|é|ée)\\b", weight: 20 }, - { locale: "en", expression: "\\bpreheat(s|ed|ing)?\\b", weight: 20 }, - ], - }, - { - uid: "bake", - mappings: [ - { - locale: "fr", - expression: - "\\bcuire au four\\b|\\bcuisson au four\\b|\\benfourn(er|ez|é|ée|ées)\\b|\\bau four\\b", - weight: 25, - }, - { - locale: "en", - expression: "\\bbak(e|es|ed|ing)\\b|\\bin (a|the) (preheated )?oven\\b", - weight: 25, - }, - ], - }, - { - uid: "plate", - mappings: [ - { locale: "fr", expression: "\\bdress(er|ez|age)\\b", weight: 15 }, - { locale: "en", expression: "\\bplat(e|es|ed|ing)\\b", weight: 15 }, - ], - }, - { - uid: "coat", - mappings: [ - { locale: "fr", expression: "\\bnapp(er|ez|é|ée|ées|age)\\b", weight: 15 }, - { locale: "en", expression: "\\bcoat(s|ed|ing)?\\b", weight: 15 }, - ], - }, +// Cooking-technique catalog (French recipe-step normalization) — the +// stable `key`s `tech-step-matcher.ts` auto-detects in a free-text +// `Step.description` (a step can mention several, e.g. "faire chauffer une +// poêle puis y faire fondre le beurre" is both `preheat` and `melt` — see +// `Step.techSteps`/`StepTechStep` in schema.prisma). Same "English +// camelCase uid, no French label" authoring as DIETS/UNITS — the label +// lives in apps/web's locales/fr/translation.json under +// `catalog.techSteps.`. +// +// Just a flat list of stable ids here — the actual matching data (per- +// locale synonym lists + example phrasings the classifier trains on) lives +// in `lib/recipe-matching/tech-step-training-data.ts`'s +// `TECH_STEP_TRAINING_DATA`, not here: unlike this list, it's read by +// `TechStepClassifierService`'s training pass, not the seed script, so it +// doesn't belong alongside the rest of this file's DB-seeded reference +// data. Every entry here must have a matching entry there. +export const TECH_STEPS: string[] = [ + "cook", + "fry", + "melt", + "deglaze", + "simmer", + "boil", + "roast", + "grill", + "panFry", + "blanch", + "marinate", + "chop", + "peel", + "mince", + "mix", + "whisk", + "foldIn", + "setAside", + "season", + "drain", + "brown", + "rest", + "preheat", + "bake", + "plate", + "coat", ]; // The 14 allergens EU Regulation 1169/2011 (Annex II) requires food @@ -1389,37 +1193,11 @@ export async function seedReferenceData(prisma: PrismaClient): Promise { } // TechStep: upsert by key (same idempotent-seed reasoning as everything - // above), then fully replace its mappings on every reseed. Mappings carry - // no natural per-row identity to upsert against, and expressions/weights - // are expected to be tuned over time — a straight "delete all, recreate - // from source" keeps the table an exact mirror of `TECH_STEPS` rather - // than accumulating stale/duplicate rows from earlier edits. Nothing else - // references `TechStepMapping.id` (`Step` only points at `TechStep`, not - // at a specific mapping), so this replace is safe. - for (const { uid: key } of TECH_STEPS) { + // above) — just the stable id/key rows themselves now, no matching data + // to replace alongside them (see `TECH_STEPS`' own comment for why). + for (const key of TECH_STEPS) { await prisma.techStep.upsert({ where: { key }, update: {}, create: { key } }); } - const techSteps = await prisma.techStep.findMany({ - where: { key: { in: TECH_STEPS.map((t) => t.uid) } }, - }); - const techStepIdByKey = new Map(techSteps.map((t) => [t.key, t.id])); - - await prisma.techStepMapping.deleteMany({ - where: { techStepId: { in: [...techStepIdByKey.values()] } }, - }); - const techStepMappingRows = TECH_STEPS.flatMap(({ uid, mappings }) => { - const techStepId = techStepIdByKey.get(uid); - if (techStepId === undefined) return []; - return mappings.map(({ locale, expression, weight }) => ({ - techStepId, - locale, - expression, - weight, - })); - }); - if (techStepMappingRows.length > 0) { - await prisma.techStepMapping.createMany({ data: techStepMappingRows }); - } // `Allergy` itself carries no `key` — it's the selectable instance of a // keyed `Category` (see schema.prisma) — so seeding an allergen means one diff --git a/apps/api/src/lib/recipe-matching/ingredient-matcher.ts b/apps/api/src/lib/recipe-matching/ingredient-matcher.ts index 61e7327..cc47781 100644 --- a/apps/api/src/lib/recipe-matching/ingredient-matcher.ts +++ b/apps/api/src/lib/recipe-matching/ingredient-matcher.ts @@ -31,8 +31,9 @@ import { normalizeText } from "./tech-step-matcher.js"; * unit-testable without a database (see `test/ingredient-matcher.test.ts`); * `loadIngredientCatalog`/`loadUnitCatalog` are the only DB-touching pieces, * meant to be fetched once per request and reused across every ingredient - * line, the same "don't requery per item" convention as - * `loadTechStepMappingRules`. + * line, the same "don't requery per item" convention + * `tech-step-matcher.ts`'s `TechStepClassifierService` follows for its own + * one-time training pass. */ /** One `Ingredient` row trimmed to what {@link matchIngredientName} needs, alongside its English matching label. */ diff --git a/apps/api/src/lib/recipe-matching/recipe-translation.ts b/apps/api/src/lib/recipe-matching/recipe-translation.ts index adcf398..dd328b8 100644 --- a/apps/api/src/lib/recipe-matching/recipe-translation.ts +++ b/apps/api/src/lib/recipe-matching/recipe-translation.ts @@ -13,11 +13,7 @@ import { matchUnit, type UnitMatchEntry, } from "./ingredient-matcher.js"; -import { - loadTechStepMappingRules, - matchTechSteps, - type TechStepMappingRule, -} from "./tech-step-matcher.js"; +import { techStepClassifier } from "./tech-step-matcher.js"; /** * The "Traduction en étapes" stage of the import pipeline described in @@ -36,17 +32,18 @@ import { * can't know those) — this is one step of the pipeline, not the whole * thing. * - * `translateRecipeSteps`/`translateRecipeIngredients` are pure (take their - * matching data as plain arguments, same convention as `matchTechSteps`/ - * `matchIngredientName` themselves) so they're unit-testable without a - * database; `translateRecipe` is the DB-backed convenience wrapper a caller - * reaches for in practice, mirroring `tech-step-matcher.ts`'s own - * pure/DB-touching split. + * `translateRecipeIngredients` stays pure (takes its matching data as plain + * arguments, same convention `matchIngredientName` itself has) so it's + * unit-testable without a database. `translateRecipeSteps` no longer is — + * technique detection now goes through `techStepClassifier`'s trained + * model (`tech-step-matcher.ts`), which needs an async call — but is still + * exported separately from `translateRecipe` for callers/tests that only + * care about step translation, not ingredients too. */ /** A {@link ParsedRecipeStep}, after tech-step detection — declares its technique sequence alongside the description/picture it already had. */ export interface TranslatedRecipeStep extends ParsedRecipeStep { - /** Ordered sequence of detected `TechStep` ids (see `matchTechSteps`) — empty if this step doesn't mention any known technique. */ + /** Ordered sequence of detected `TechStep` ids (see `TechStepClassifierService.matchTechSteps`) — empty if this step doesn't mention any known technique. */ techStepIds: number[]; } @@ -63,34 +60,43 @@ export interface TranslatedRecipe extends Omit ({ - ...ingredient, - ingredientId: null, - unitId: null, - })), - steps: recipe.steps.map((step) => ({ - ...step, - techStepIds: matchTechSteps(step.description, techStepMappings), - })), - }; + locale: string, +): Promise { + try { + const steps = await Promise.all( + recipe.steps.map(async (step) => ({ + ...step, + techStepIds: await techStepClassifier.matchTechSteps(step.description, locale), + })), + ); + return { + ...recipe, + ingredients: recipe.ingredients.map((ingredient) => ({ + ...ingredient, + ingredientId: null, + unitId: null, + })), + steps, + }; + } catch (err) { + // Rethrown as-is — the caller (`sources.service.ts`) already + // handles/logs failures centrally; this function just isn't allowed a + // bare `await` per the repo's async/try-catch convention. + throw err; + } } /** @@ -259,12 +265,13 @@ export function mergeDuplicateIngredients( * manually-authored recipes. * * No user- or recipe-level language preference exists anywhere in the app - * yet (see `tech-step-matcher.ts`'s `loadTechStepMappingRules`) — callers + * yet (see `tech-step-matcher.ts`'s `TechStepClassifierService`) — callers * pass a locale explicitly rather than this module guessing one. Note that * an English-language source (e.g. TheMealDB) translated against `"fr"` - * mappings will currently get an empty `techStepIds` sequence on every - * step — matching-language mappings for that source's language don't exist - * yet, this stage doesn't invent them. + * will currently get an empty (or nonsensical) `techStepIds` sequence on + * every step — the classifier is trained per-locale, so calling it with a + * locale that doesn't match the actual text's language doesn't degrade + * gracefully, it just gets things wrong. * * Ingredient/unit matching only has English data today * (`INGREDIENT_LABELS_EN`/`UNIT_LABELS_EN`, `packages/shared`) — for any @@ -279,8 +286,7 @@ export async function translateRecipe( locale: string, ): Promise { try { - const techStepMappings = await loadTechStepMappingRules(locale); - const translated = translateRecipeSteps(recipe, techStepMappings); + const translated = await translateRecipeSteps(recipe, locale); if (locale !== "en") return translated; diff --git a/apps/api/src/lib/recipe-matching/tech-step-matcher.ts b/apps/api/src/lib/recipe-matching/tech-step-matcher.ts index b37dd2a..9f7b580 100644 --- a/apps/api/src/lib/recipe-matching/tech-step-matcher.ts +++ b/apps/api/src/lib/recipe-matching/tech-step-matcher.ts @@ -1,46 +1,67 @@ +import { NlpManager } from "node-nlp"; import { prisma } from "../../db/prisma.js"; +import { TECH_STEP_TRAINING_DATA } from "./tech-step-training-data.js"; /** * Auto-detects which cooking techniques (`TechStep`) a free-text recipe - * step description corresponds to, using the static `TechStepMapping` - * catalog (see `reference-seed-data.ts`'s `TECH_STEPS`) — groundwork for a - * future batch-cooking optimization algorithm, and (via `matchTechStepSpans`) - * what `recipe.service.ts` persists as `StepTechStep.start`/`end` so the - * recipe UI can highlight the exact matched words (see `StepView` in + * step description corresponds to — groundwork for a future batch-cooking + * optimization algorithm, and (via `matchTechStepSpans`) what + * `recipe.service.ts` persists as `StepTechStep.start`/`end` so the recipe + * UI can highlight the exact matched words (see `StepView` in * `packages/shared`). * - * A single instruction can genuinely involve more than one technique (e.g. - * "Dans une poêle chaude, faire chauffer une noix de beurre" is both - * `preheat` and `melt`) — both `matchTechSteps`/`matchTechStepSpans` return - * the whole *ordered sequence* they find, not a single winner, matching + * Regex-only matching used to live here (matching literal verb-form + * patterns from a DB-backed `TechStepMapping` table) but couldn't + * generalize past its own vocabulary — a step describing melting butter as + * "jusqu'à ce que le beurre ait disparu dans la poêle" mentions no verb any + * regex could anchor on, yet unmistakably *means* `melt`. Replaced with a + * small hybrid pipeline built on `node-nlp` ({@link TechStepClassifierService}): + * + * 1. **NER** (node-nlp enum entities, `synonyms` in `TECH_STEP_TRAINING_DATA`) + * finds every *candidate* technique mention in the whole + * description, each with its exact character span — mechanically the + * same job the old regexes did, just as flat synonym lists instead of + * hand-written patterns (node-nlp's own stemmer/fuzzy matching already + * covers minor conjugation/typo variance the regexes had to enumerate + * by hand). This step alone is *not* the final answer — see step 3. + * 2. The description is cut into clauses around those candidate spans + * ({@link splitIntoClauses}) — a step naming two techniques ("Dans une + * poêle chaude, faire chauffer une noix de beurre" is both `preheat` + * and `melt`) needs each judged on its own surrounding context, not the + * whole step lumped into one classification. + * 3. **NLP intent classification** (node-nlp's `NlpManager`, trained on + * `TECH_STEP_TRAINING_DATA`'s `utterances`) then classifies each clause + * on its own — this is what actually delivers "meaning, not keywords": + * the classifier was deliberately trained on paraphrases that never use + * the technique's own verb (e.g. "jusqu'à ce que le beurre ait + * disparu" for `melt`), so a clause reaching it gets labeled by what it + * was trained to recognize as *meaning* a technique, not by which + * literal word the NER step happened to anchor on. The NER-implied + * technique is kept only as a fallback for a clause the classifier + * isn't confident about (see `CONFIDENCE_THRESHOLD`) — a clearly + * keyword-anchored clause a small model merely isn't sure how to + * classify shouldn't be dropped outright. + * + * A single instruction can genuinely involve more than one technique — see + * point 2 above — so `matchTechSteps`/`matchTechStepSpans` both return the + * whole *ordered sequence* they find, not a single winner, matching * `Step.techSteps` (schema.prisma's `StepTechStep`, an ordered join table). * - * `normalizeText`/`matchTechStepSpans`/`matchTechSteps` are pure (no DB - * access) so they can be unit-tested in isolation (see - * `test/tech-step-matcher.test.ts`). `loadTechStepMappingRules` is the only - * DB-touching piece, kept separate so callers (`recipe.service.ts`) fetch - * the whole mapping list once per request and pass it to - * `matchTechStepSpans` per step, rather than querying once per step. + * `normalizeText` and {@link splitIntoClauses} are pure (no DB/model + * access) so they stay unit-testable in isolation (see + * `test/tech-step-matcher.test.ts`); the classifier itself needs a one-time + * training pass (`_ensureTrained`, node-nlp's `NlpManager.train()`) plus a + * `TechStep.key -> id` lookup from the DB, both memoized on the shared + * {@link techStepClassifier} singleton rather than repeated per call — + * training is the expensive part (a few hundred ms for this corpus), never + * worth redoing per request let alone per step. */ -/** One `TechStepMapping` row, trimmed to what {@link matchTechSteps} needs. */ -export interface TechStepMappingRule { - techStepId: number; - /** - * Regex source, matched against the normalized description (see - * {@link normalizeText}) — may itself contain accented characters, - * normalized the same way before compiling. - */ - expression: string; - weight: number; -} - /** * Lowercases and strips diacritics (NFD decomposition + removal of - * combining marks, e.g. "Déglacer" -> "deglacer") — recipe step text and - * mapping expressions are both run through this before matching, so - * expressions can be authored with natural French accents in - * `reference-seed-data.ts` while matching stays accent/case-insensitive. + * combining marks, e.g. "Déglacer" -> "deglacer"). Still used by + * `ingredient-matcher.ts` for its own, unrelated free-text matching — kept + * here and exported rather than duplicated, this module owned it first. */ const COMBINING_DIACRITICS_PATTERN = /\p{Diacritic}/gu; @@ -48,155 +69,415 @@ export function normalizeText(text: string): string { return text.normalize("NFD").replace(COMBINING_DIACRITICS_PATTERN, "").toLowerCase(); } -/** Where in the (normalized) description one mapping matched, alongside the rule that matched — the raw material {@link matchTechStepSpans} resolves into a final sequence. */ -interface MatchCandidate extends TechStepMappingRule { - start: number; - end: number; -} - -/** Whether two candidates' matched spans share any character position — the case where two *different* techniques' expressions matched the same words (e.g. generic `cook`'s "cuire" inside specific `bake`'s "cuire au four"), meaning only one of them should survive. */ -function overlaps(a: MatchCandidate, b: MatchCandidate): boolean { - return a.start < b.end && b.start < a.end; -} - /** * One technique {@link matchTechStepSpans} found, alongside exactly where in - * `description` it matched — `[start, end)`, same convention as - * `String.prototype.slice`. Persisted as `StepTechStep.start`/`end` - * (`recipe.service.ts`) so the recipe detail view can highlight the exact - * matched words, not just know a technique was mentioned somewhere. + * `description` it matched — two nested spans, both `[start, end)` (same + * convention as `String.prototype.slice`): + * + * - `start`/`end` — the tight *keyword* span (e.g. "préchauffer") that + * directly triggered the match, or (when no NER anchor exists at all — + * see {@link splitIntoClauses}'s zero-candidate case) the whole clause, + * same as `contextStart`/`contextEnd` below. + * - `contextStart`/`contextEnd` — the wider *clause* the keyword was found + * in (e.g. "Dans une poêle chaude" for a `preheat` keyword of "poêle + * chaude") — what actually got fed to the classifier (see this file's + * doc comment, point 3), kept alongside the tight span so a caller can + * show *both*: the exact trigger word(s), and how much of the sentence + * is understood to be about that technique. Always contains `start`/`end` + * (`contextStart <= start`, `end <= contextEnd`). + * + * Persisted as `StepTechStep.start`/`end`/`contextStart`/`contextEnd` + * (`recipe.service.ts`) so the recipe detail view can highlight both spans, + * not just know a technique was mentioned somewhere. */ export interface TechStepMatch { techStepId: number; start: number; end: number; + contextStart: number; + contextEnd: number; +} + +/** A candidate technique mention found by NER — the raw material {@link splitIntoClauses} cuts a description around. */ +export interface TechniqueCandidate { + /** Training-data `uid` this candidate's synonym belongs to (e.g. `"melt"`) — not yet resolved to a DB id at this stage. */ + uid: string; + start: number; + end: number; +} + +/** One clause {@link splitIntoClauses} produced — `anchor` is `null` only for the single "whole description, no candidate found at all" fallback clause (see that function's doc comment). */ +export interface TechStepClause { + /** `[start, end)` into the original description — the text handed to the classifier for this clause, and (see {@link TechStepMatch}) what ends up as a match's `contextStart`/`contextEnd`. */ + start: number; + end: number; + /** The candidate this clause was cut around, if any — its own (tighter) span is what gets persisted as a match's `start`/`end` for the keyword highlight, the wider clause span is always its `contextStart`/`contextEnd`. */ + anchor: TechniqueCandidate | null; +} + +/** Matches a sentence-ending punctuation mark, for {@link findGapSplitPoint}'s preferred split points. */ +const SENTENCE_END_PATTERN = /[.!?]/; + +/** + * Picks where to cut the gap `[gapStart, gapEnd)` between two consecutive + * candidates — preferring a *sentence* boundary (right after `.`/`!`/`?`) + * nearest the gap's midpoint when one exists in the gap, otherwise any + * whitespace nearest the midpoint, so a clause boundary (surfaced to users + * as `contextStart`/`contextEnd`, unlike a keyword's own `[start, end)` + * which always lands on a real word by construction) never slices through + * the middle of a word — found while testing a context span that cut + * "poêle" into "poêl"/"e" across two clauses. + * + * The sentence-boundary preference matters beyond cosmetics: a description + * with two techniques in two different sentences ("Préchauffer le four à + * 180°C. Dans un saladier, mettre le beurre... et mélanger.") used to only + * get a plain nearest-midpoint whitespace split, which for a long first + * sentence lands *inside* the second one — handing the classifier a clause + * like "...(thermostat 6). Dans un saladier, mettre" that trails off + * mid-instruction with no object. That garbled, incomplete text is nothing + * like the short, complete training utterances, and was found to + * misclassify real recipe steps with high (>0.65) confidence in both + * halves — "Préchauffer..." scored as `mix`, its actual "mélanger" clause + * as `melt`. Splitting at the real sentence boundary instead hands the + * classifier two complete, grammatical clauses, each far closer to what it + * was trained on. + * + * Falls back to the raw midpoint when the gap has no whitespace at all + * (adjacent candidates, or a gap that's pure punctuation with no space) — + * same "some split point, however imperfect" fallback a plain midpoint + * always was. + */ +function findGapSplitPoint(description: string, gapStart: number, gapEnd: number): number { + if (gapStart >= gapEnd) return gapStart; + const midpoint = Math.floor((gapStart + gapEnd) / 2); + + let bestSentenceEnd: number | null = null; + let bestSentenceEndDistance = Number.POSITIVE_INFINITY; + let bestWhitespace: number | null = null; + let bestWhitespaceDistance = Number.POSITIVE_INFINITY; + for (let i = gapStart; i < gapEnd; i++) { + if (!/\s/.test(description[i] ?? "")) continue; + const distance = Math.abs(i - midpoint); + if (distance < bestWhitespaceDistance) { + bestWhitespace = i; + bestWhitespaceDistance = distance; + } + if ( + i > gapStart && + SENTENCE_END_PATTERN.test(description[i - 1] ?? "") && + distance < bestSentenceEndDistance + ) { + bestSentenceEnd = i; + bestSentenceEndDistance = distance; + } + } + return bestSentenceEnd ?? bestWhitespace ?? midpoint; } /** - * Detects every technique `description` mentions among `mappings`, as an - * ordered sequence of matches (each carrying *where* it matched) — empty if - * none match. The algorithm: + * Cuts `description` into clauses around `candidates` (NER's found + * technique mentions, already sorted or not — sorted internally), one + * clause per candidate, so each can be judged by the classifier on its own + * surrounding context rather than the whole (possibly multi-technique) + * description at once. * - * 1. Test every mapping against the normalized description; each one that - * matches becomes a candidate carrying *where* it matched (so - * overlapping matches can be compared). - * 2. Within a single technique, several of its own mappings might all - * match (different phrasings for the same `techStepId`) — keep only - * that technique's best candidate (highest weight, ties broken by - * earliest match), the same tie-break this function always used for a - * single winner. - * 3. Across *different* techniques, two candidates can still overlap (a - * generic pattern matching inside a more specific one's span, e.g. - * `cook` vs `bake` both matching "cuire au four") — resolve greedily by - * weight: take candidates highest-weight first, accept a candidate only - * if it doesn't overlap one already accepted. This is what keeps - * `bake` and drops the redundant `cook` for that phrase, while letting - * two genuinely distinct, non-overlapping techniques (e.g. `preheat` - * and `melt` in "Dans une poêle chaude, faire chauffer une noix de - * beurre") both survive. - * 4. Sort what's left by where it appears in the text — the sequence - * reads in the same order as the instruction itself. + * - **Zero candidates**: the whole description is one clause with no + * anchor — still worth classifying (a description mentioning no literal + * keyword at all can still *mean* a technique, the entire point of the + * classification step), just with no tight span to highlight, so callers + * fall back to highlighting the whole thing. + * - **One candidate**: the whole description is one clause too (nothing to + * cut around a single mention), but *with* that candidate as its anchor + * — callers get its tight span for highlighting. + * - **Two or more**: split points fall at the whitespace nearest the + * midpoint of each consecutive pair's `[end, nextStart]` gap (see + * {@link findGapSplitPoint} — never mid-word), producing that many + * contiguous, non-overlapping clauses covering the whole description — + * clause *i* is anchored on candidate *i*. * - * The returned `start`/`end` are offsets into `normalizeText(description)`, - * used as-is against the *original* `description` by callers that slice it - * for display (`highlight-tech-steps.ts`, apps/web) — `normalizeText` only - * strips diacritics/lowercases, which preserves character count for - * realistic French text (canonical NFD decomposition never turns one - * character into more than one base character), so this holds in practice. - * A pathological input where it doesn't (e.g. a bare standalone `^`, which - * `normalizeText` would strip as a diacritic) just produces a slightly - * misplaced highlight — degrades silently, doesn't crash. - * - * Pure — takes `mappings` as a plain argument rather than querying Prisma - * itself, so it's testable without a database (see - * `loadTechStepMappingRules` for the DB-backed loader). `mappings` should - * already be filtered to the locale the caller cares about — this function - * has no notion of locale, it just tests the rules it's given. + * Pure and DB/model-free — unit-tested directly (see + * `test/tech-step-matcher.test.ts`) without needing a trained classifier. */ -export function matchTechStepSpans( +export function splitIntoClauses( description: string, - mappings: TechStepMappingRule[], -): TechStepMatch[] { - const normalizedDescription = normalizeText(description); + candidates: TechniqueCandidate[], +): TechStepClause[] { + if (candidates.length === 0) { + return [{ start: 0, end: description.length, anchor: null }]; + } - const candidates: MatchCandidate[] = []; - for (const mapping of mappings) { - const pattern = new RegExp(normalizeText(mapping.expression), "i"); - const match = pattern.exec(normalizedDescription); - if (match === null) continue; - candidates.push({ - ...mapping, - start: match.index, - end: match.index + match[0].length, + const sorted = [...candidates].sort((a, b) => a.start - b.start); + const [first, ...rest] = sorted; + if (first === undefined) { + // Unreachable — `candidates.length === 0` already returned above, so + // `sorted` (same length) always has a first element here. Satisfies + // `noUncheckedIndexedAccess`, which can't see that from the length + // check alone. + return [{ start: 0, end: description.length, anchor: null }]; + } + + // Single pass, pairing each candidate with the next one as it goes — + // avoids re-indexing a separately-built `splitPoints` array afterward + // (also awkward under `noUncheckedIndexedAccess` for no real benefit, + // since every split point is only ever read once, right after it's + // computed). + const clauses: TechStepClause[] = []; + let clauseStart = 0; + let anchor = first; + for (const next of rest) { + const splitPoint = findGapSplitPoint(description, anchor.end, next.start); + clauses.push({ start: clauseStart, end: splitPoint, anchor }); + clauseStart = splitPoint; + anchor = next; + } + clauses.push({ start: clauseStart, end: description.length, anchor }); + return clauses; +} + +/** + * Below this confidence, a clause's classifier verdict isn't trusted on its + * own — falls back to its NER anchor's own technique instead (see this + * file's doc comment, point 3). Tuned empirically against + * `TECH_STEP_TRAINING_DATA` — see `test/tech-step-matcher.test.ts` for the + * cases this threshold was picked to pass. + * + * Raised from `0.65` after finding real (non-adversarial) misclassified + * clauses that scored just above the old threshold — e.g. English recipe + * text run through the French classifier (which must find *nothing*, + * confirmed by `recipe-translation.test.ts`'s own locale-isolation test) + * scored `0.69` for `boil`, essentially classifier noise on + * out-of-vocabulary input rather than a real, confident verdict. The + * clauses this threshold exists to actually trust score far higher in + * practice (`0.91`–`1.0` for the real corrected cases found this session) + * — `0.75` sits comfortably above the noise floor and below every genuine + * match seen so far. + */ +const CONFIDENCE_THRESHOLD = 0.75; + +/** + * Trains and owns the `node-nlp` model behind {@link matchTechStepSpans} — + * a real class (not a plain object of functions) per this repo's + * service-style-logic convention, even though it's only ever used as the + * one shared {@link techStepClassifier} singleton below: it holds real + * state (the trained model, the memoized training/lookup promises), not + * just grouped stateless helpers. + */ +export class TechStepClassifierService { + /** node-nlp's manager — both NER (enum entities) and NLP (intent classification) live on the same instance, trained together. */ + private readonly _manager: NlpManager; + /** Memoized training pass — `undefined` until the first call starts it, after which every caller (concurrent or not) awaits the same promise rather than retraining. */ + private _trained: Promise | undefined; + /** Memoized `TechStep.key -> id` lookup — training data only knows techniques by their stable `uid`/`key`, resolved to the real DB id once, alongside training. */ + private _techStepIdByUid: Map | undefined; + + public constructor() { + this._manager = new NlpManager({ + languages: ["fr", "en"], + forceNER: true, + nlu: { log: false }, + // node-nlp's enum-entity NER defaults to a fuzzy (Levenshtein-based) + // 0.8 accuracy threshold — loose enough that e.g. "faire" (the + // generic French helper verb in almost every recipe step) fuzzy- + // matches `fry`'s synonym "frire" at 0.80, a false positive found + // while tuning this against the real training corpus. `1` (exact, + // after node-nlp's own case/accent/stemming normalization — real + // conjugation variance is still covered by listing each form in + // `tech-step-training-data.ts`) removed it without losing any real + // match. Precision matters more than recall for this stage — NER + // only proposes candidate split points, `_classifyClause`'s trained + // model (not fuzzy string distance) is what actually has to be + // right. + ner: { threshold: 1 }, + // node-nlp defaults to `autoSave`/`autoLoad: true` — silently + // persisting the trained model to a `model.nlp` file in the process's + // cwd, and *loading from that file instead of retraining* the next + // time a manager is constructed, if the file already exists. Found + // this the hard way: a stray `model.nlp` appeared at the repo root + // after running this locally. That's the opposite of what this + // service wants — `TECH_STEP_TRAINING_DATA` in code is the single + // source of truth this always trains fresh from (see this file's own + // doc comment) — a stale on-disk model silently shadowing a + // corpus/threshold update would be a nasty, hard-to-notice class of + // bug. Both off; nothing here should ever touch disk. + autoSave: false, + autoLoad: false, }); } - // Step 2: one best candidate per techStepId. - const bestByTechStep = new Map(); - for (const candidate of candidates) { - const current = bestByTechStep.get(candidate.techStepId); - if ( - current === undefined || - candidate.weight > current.weight || - (candidate.weight === current.weight && candidate.start < current.start) - ) { - bestByTechStep.set(candidate.techStepId, candidate); + /** + * Forces training plus node-nlp's own one-time lazy setup (loading its + * bundled per-language stemmers/tokenizers on the *first* real + * `NlpManager.process()` call takes a few seconds by itself, separate + * from and much slower than the ~40ms `train()` pass — measured against + * this corpus while tuning the pipeline) to happen now, synchronously + * with server startup (see `server.ts`), rather than stalling whichever + * request happens to be first to save/preview a recipe. + */ + public async warmUp(): Promise { + try { + await this.matchTechStepSpans("faire cuire à feu doux", "fr"); + } catch (err) { + throw err; // see matchTechStepSpans()'s catch comment above } } - // Step 3: resolve cross-technique overlaps, highest weight first. - const byWeightDesc = [...bestByTechStep.values()].sort( - (a, b) => b.weight - a.weight || a.techStepId - b.techStepId, - ); - const accepted: MatchCandidate[] = []; - for (const candidate of byWeightDesc) { - if (accepted.some((other) => overlaps(candidate, other))) continue; - accepted.push(candidate); + /** + * Detects every technique `description` means, as an ordered sequence of + * matches (each carrying *where* it matched) — empty if none apply. See + * this file's doc comment for the full NER -> split -> classify + * pipeline. + * + * @param locale Which of `TECH_STEP_TRAINING_DATA`'s locales to match + * against — same "caller already knows/validated this" contract the + * old `matchTechStepSpans(description, mappings)` had via its + * pre-filtered `mappings` argument, just as an explicit parameter now + * that the training data isn't pre-filtered by the caller anymore. + */ + public async matchTechStepSpans(description: string, locale: string): Promise { + try { + await this._ensureTrained(); + if (description.trim().length === 0) return []; + + const nerResult = await this._manager.process(locale, description); + const candidates: TechniqueCandidate[] = nerResult.entities + // node-nlp's language plugins also auto-extract their own built-in + // entities (numbers, durations, dates…) alongside the enum + // entities `_train` registered from `TECH_STEP_TRAINING_DATA` — + // `type === "enum"` is what tells the two apart; without this + // filter a step like "10 minutes" would hand `splitIntoClauses` a + // bogus "duration" candidate that resolves to no real technique. + .filter((entity) => entity.type === "enum") + .map((entity) => ({ + uid: entity.entity, + start: entity.start, + // node-nlp's own `end` is inclusive (verified against a real + // trained model) — `+ 1` converts to this module's `[start, end)` + // convention, matching `String.prototype.slice`. + end: entity.end + 1, + })); + + const clauses = splitIntoClauses(description, candidates); + const matches: TechStepMatch[] = []; + for (const clause of clauses) { + const uid = await this._classifyClause(description, clause, locale); + if (uid === null) continue; + const techStepId = this._techStepIdByUid?.get(uid); + // A `uid` the classifier/NER was trained on but that no longer has + // a matching `TechStep` row (e.g. training data and + // `reference-seed-data.ts` drifted apart) — skip rather than + // persist a dangling id. + if (techStepId === undefined) continue; + const span = clause.anchor ?? { start: clause.start, end: clause.end }; + matches.push({ + techStepId, + start: span.start, + end: span.end, + contextStart: clause.start, + contextEnd: clause.end, + }); + } + + matches.sort((a, b) => a.start - b.start || a.techStepId - b.techStepId); + return matches; + } catch (err) { + // Rethrown as-is — `wrapAsyncHandler`/the error middleware (which + // already logs it, see `error-logger.ts`) is what actually handles + // it, this service layer just isn't allowed a bare `await` without a + // try/catch per the repo's convention. + throw err; + } } - // Step 4: reading order. - accepted.sort((a, b) => a.start - b.start || a.techStepId - b.techStepId); - return accepted.map(({ techStepId, start, end }) => ({ - techStepId, - start, - end, - })); -} + /** + * Convenience wrapper around {@link matchTechStepSpans} for callers that + * only care about *which* techniques matched, not where — e.g. + * `recipe-translation.ts`'s `translateRecipeSteps`, which declares a + * step's technique sequence for an imported recipe that isn't saved (and + * so has no `StepTechStep` row to persist a span into) yet. + */ + public async matchTechSteps(description: string, locale: string): Promise { + try { + return (await this.matchTechStepSpans(description, locale)).map((match) => match.techStepId); + } catch (err) { + throw err; // see matchTechStepSpans()'s catch comment above + } + } -/** - * Convenience wrapper around {@link matchTechStepSpans} for callers that - * only care about *which* techniques matched, not where — e.g. - * `recipe-translation.ts`'s `translateRecipeSteps`, which declares a step's - * technique sequence for an imported recipe that isn't saved (and so has no - * `StepTechStep` row to persist a span into) yet. - */ -export function matchTechSteps(description: string, mappings: TechStepMappingRule[]): number[] { - return matchTechStepSpans(description, mappings).map((match) => match.techStepId); -} + /** + * Classifies one clause, returning the technique `uid` it means (or + * `null` if none applies) — the classifier's own verdict when it's + * confident enough ({@link CONFIDENCE_THRESHOLD}), otherwise the + * clause's NER anchor (if it has one) as a floor: a clearly + * keyword-anchored clause a small model merely isn't sure how to + * classify shouldn't be dropped outright, only a genuinely + * anchor-less/low-confidence one should. + */ + private async _classifyClause( + description: string, + clause: TechStepClause, + locale: string, + ): Promise { + try { + const clauseText = description.slice(clause.start, clause.end).trim(); + if (clauseText.length === 0) return clause.anchor?.uid ?? null; -/** - * Loads every `TechStepMapping` row for `locale` as - * {@link TechStepMappingRule}s — meant to be fetched once per request by - * `recipe.service.ts`'s `createRecipe`/`updateRecipe` and reused across - * every step of the recipe being saved, not re-queried per step. - * - * No user-language preference exists anywhere in the app yet (a single - * `"fr"` translation file, no locale field on `User`/`UserProfile`) — - * callers pass a hardcoded locale for now; this parameter exists so that - * plugging in a real user preference later doesn't require touching this - * module. - */ -export async function loadTechStepMappingRules(locale: string): Promise { - try { - return await prisma.techStepMapping.findMany({ - where: { locale }, - select: { techStepId: true, expression: true, weight: true }, - }); - } catch (err) { - // Rethrown as-is — the caller (`recipe.service.ts`/`sources.service.ts`) - // already handles/logs failures centrally; this function just isn't - // allowed a bare `async` body without a try/catch per the repo's - // convention. - throw err; + const result = await this._manager.process(locale, clauseText); + if (result.intent !== "None" && result.score >= CONFIDENCE_THRESHOLD) { + return result.intent; + } + return clause.anchor?.uid ?? null; + } catch (err) { + throw err; // see matchTechStepSpans()'s catch comment above + } + } + + /** + * Trains `_manager` from {@link TECH_STEP_TRAINING_DATA} and resolves the + * `uid -> TechStep.id` lookup, both exactly once — memoized on + * `_trained` so a burst of concurrent calls (several steps of the same + * recipe save, awaited via the same event loop tick) all await the one + * in-flight training pass rather than each kicking off their own. + */ + private async _ensureTrained(): Promise { + if (this._trained === undefined) { + this._trained = this._train(); + } + try { + await this._trained; + } catch (err) { + // A failed training pass must be retried by the *next* call, not + // leave every future call permanently rejecting against a stale + // failed promise. + this._trained = undefined; + throw err; + } + } + + private async _train(): Promise { + try { + const techSteps = await prisma.techStep.findMany({ select: { id: true, key: true } }); + this._techStepIdByUid = new Map(techSteps.map((techStep) => [techStep.key, techStep.id])); + + for (const entry of TECH_STEP_TRAINING_DATA) { + for (const [locale, data] of [ + ["fr", entry.fr], + ["en", entry.en], + ] as const) { + if (data.synonyms.length > 0) { + this._manager.addNamedEntityText(entry.uid, entry.uid, [locale], data.synonyms); + } + for (const utterance of data.utterances) { + this._manager.addDocument(locale, utterance, entry.uid); + } + } + } + + await this._manager.train(); + } catch (err) { + throw err; // see matchTechStepSpans()'s catch comment above + } } } + +/** Single shared instance — training is expensive enough (a few hundred ms) that every caller must reuse the one already-trained model, never spin up their own. */ +export const techStepClassifier = new TechStepClassifierService(); diff --git a/apps/api/src/lib/recipe-matching/tech-step-training-data.ts b/apps/api/src/lib/recipe-matching/tech-step-training-data.ts new file mode 100644 index 0000000..b4e6019 --- /dev/null +++ b/apps/api/src/lib/recipe-matching/tech-step-training-data.ts @@ -0,0 +1,911 @@ +/** + * Training corpus for {@link TechStepClassifierService} (`tech-step-matcher.ts`) + * — one entry per `TechStep` (`uid` matches `reference-seed-data.ts`'s + * `TECH_STEPS`, which still owns the reference `TechStep` rows themselves; + * this file replaces `TECH_STEPS[].mappings`' regex expressions as the + * *matching* data source). + * + * Two distinct kinds of content per technique/locale, feeding two distinct + * mechanisms of the classifier (see that file's doc comment for why both + * are needed): + * + * - `synonyms` — short literal words/set phrases, fed to node-nlp's NER + * (enum entities). Mechanically equivalent to the old regexes' verb-form + * alternations, just spelled out as plain words instead of a pattern + * (node-nlp's own stemmer/fuzzy matching already covers minor + * conjugation/typo variance that the regexes had to enumerate by hand). + * Used only to find *candidate* technique mentions and cut a step into + * clauses around them — never the final answer on their own. + * - `utterances` — full example clauses, fed to node-nlp's NLP Manager as + * training documents for the intent classifier. Deliberately mixes + * keyword-anchored phrasings (reinforces the obvious case) with + * paraphrases that never use the technique's own verb at all (e.g. + * "jusqu'à ce que le beurre ait disparu" for `melt`) — this second kind + * is what actually delivers on "comprendre le sens, pas juste les mots + * clés" (see the PR this file was introduced in): a clause reaching the + * classifier gets labeled by what it's trained to recognize as *meaning* + * this technique, not by which literal word triggered its extraction. + * + * Kept as static in-code data (not DB rows, unlike the old + * `TechStepMapping` table) because nothing needs to query/edit it at + * runtime — it only ever feeds one thing, the classifier's one-time + * training pass (see `TechStepClassifierService._ensureTrained`) — same + * reasoning `INGREDIENT_LABELS_EN` (`packages/shared`) is a plain object, + * not a database table. + */ + +/** One technique's matching data for one locale — see this file's doc comment for what each list feeds. */ +export interface TechStepLocaleTrainingData { + synonyms: string[]; + utterances: string[]; +} + +/** One technique's full training entry — `uid` must match a `TECH_STEPS[].uid` in `reference-seed-data.ts`. */ +export interface TechStepTrainingEntry { + uid: string; + fr: TechStepLocaleTrainingData; + en: TechStepLocaleTrainingData; +} + +export const TECH_STEP_TRAINING_DATA: TechStepTrainingEntry[] = [ + { + uid: "cook", + fr: { + synonyms: [ + "cuire", + "cuisez", + "cuisant", + "cuisson", + "cuit", + "cuite", + "cuites", + "cuits", + "cuisiner", + "cuisinez", + "cuisiné", + "cuisinée", + "faire cuire", + "laisser cuire", + ], + utterances: [ + "faire cuire à feu moyen", + "laisser cuire jusqu'à ce que ce soit prêt", + "la cuisson dure environ dix minutes", + "jusqu'à ce que la viande ne soit plus rose au centre", + "poursuivre la cuisson à couvert", + // Two real recipe clauses found misclassified (as `preheat` and + // `panFry` respectively, both above the confidence threshold) once + // real, longer, comma-heavy sentences started reaching the + // classifier — neither error came from a missing keyword (both + // clauses' own NER anchor, "laisser cuire"/"faire cuire", was + // already right), just the classifier's low-heat/occasional- + // stirring phrasing not resembling anything short and clean-cut it + // had actually been trained on. + "baisser le feu et laisser cuire à découvert encore un quart d'heure", + "faire cuire à feu doux en remuant de temps en temps", + ], + }, + en: { + // NOT "cooked through"/"cooking through" — both are word-prefix + // extensions of "cooked"/"cooking" above, so any text containing them + // matches BOTH the short and long form as separate overlapping NER + // candidates, corrupting clause-splitting (confirmed via "It should + // be cooking through evenly", which spuriously grew a second, + // wrongly-classified `roast` candidate). See this pattern flagged + // throughout the file wherever it was found — the fix is always to + // drop the longer, redundant form rather than keep both. + synonyms: ["cook", "cooks", "cooked", "cooking"], + utterances: [ + "cook over medium heat", + "cook until done", + "cooking takes about ten minutes", + "until no longer pink in the middle", + "continue cooking covered", + ], + }, + }, + { + uid: "fry", + fr: { + synonyms: [ + "frire", + "frit", + "frite", + "frites", + "friture", + "faire frire", + "faites frire", + "bain de friture", + "huile de friture", + ], + utterances: [ + "faire frire dans l'huile chaude", + "plonger dans la friture", + "jusqu'à ce que ce soit doré et croustillant à l'extérieur", + "l'huile doit être bien chaude avant d'y plonger les morceaux", + ], + }, + en: { + // NOT "frying oil" — a word-prefix extension of "frying" above (see + // the `cook` entry's comment for why that duplicates/corrupts NER + // candidates; here it was even worse, misclassifying as `preheat`). + synonyms: ["fry", "fries", "fried", "frying", "deep fry", "deep-fried", "deep frying"], + utterances: [ + "fry in hot oil", + "deep fry until golden", + "until crisp and golden on the outside", + "the oil should be very hot before adding the pieces", + ], + }, + }, + { + uid: "melt", + fr: { + synonyms: [ + "fondre", + "fondu", + "fondue", + "fondues", + "faire fondre", + "faites fondre", + // Also a plausible way to say "melt" (heating something — usually + // a fat — until it liquefies), not just a `preheat` phrasing — + // restores what the regex-based system anchored on before this + // pipeline replaced it. + "faire chauffer", + "faites chauffer", + "liquéfier", + "liquéfiez", + "liquéfié", + "faire liquéfier", + ], + utterances: [ + "faire fondre le beurre", + "jusqu'à ce que le beurre ait disparu dans la poêle", + "le beurre doit être complètement liquide", + "laisser le fromage devenir tout liquide sur feu doux", + ], + }, + en: { + synonyms: ["melt", "melts", "melted", "melting", "liquefy", "liquefied"], + utterances: [ + "melt the butter", + "until the butter has completely disappeared into the pan", + "the butter should be fully liquid", + "let the cheese turn completely liquid over low heat", + ], + }, + }, + { + uid: "deglaze", + fr: { + // NOT "déglacer la poêle"/"déglacer le fond de cuisson" — both are + // word-prefix extensions of "déglacer" above (see `cook`'s comment + // for why that duplicates NER candidates). + synonyms: ["déglacer", "déglacez", "déglacé", "déglacée", "déglaçage"], + utterances: [ + "déglacer avec le vin blanc", + "verser le vin dans la poêle chaude pour décoller les sucs", + "gratter les sucs de cuisson au fond de la casserole avec un peu de bouillon", + ], + }, + en: { + // NOT "deglaze the pan" — a word-prefix extension of "deglaze" above + // (see `cook`'s comment for why that duplicates NER candidates). + synonyms: ["deglaze", "deglazes", "deglazed", "deglazing", "lift the browned bits"], + utterances: [ + "deglaze with white wine", + "pour the wine into the hot pan to lift the browned bits", + "scrape up the browned bits at the bottom of the pan with a splash of stock", + ], + }, + }, + { + uid: "simmer", + fr: { + synonyms: [ + "mijoter", + "mijotez", + "mijote", + "mijotant", + "mijoté", + "frémir", + "frémissant", + "frémissante", + "à petit feu", + ], + utterances: [ + "laisser mijoter à feu doux", + "faire mijoter pendant une heure", + "de petites bulles doivent remonter doucement à la surface", + "laisser cuire tout doucement à couvert pendant longtemps", + ], + }, + en: { + // NOT "simmering gently" — a word-prefix extension of "simmering" + // above (see `cook`'s comment for why that duplicates NER candidates). + synonyms: ["simmer", "simmers", "simmered", "simmering", "gentle simmer", "low simmer"], + utterances: [ + "let it simmer over low heat", + "simmer for one hour", + "small bubbles should gently rise to the surface", + "let it cook very gently, covered, for a long time", + ], + }, + }, + { + uid: "boil", + fr: { + synonyms: [ + "bouillir", + "bouillant", + "bouillie", + "bouillies", + "ébullition", + "porter à ébullition", + "gros bouillons", + ], + utterances: [ + "porter à ébullition", + "faire bouillir l'eau", + "de grosses bulles doivent agiter la surface avec force", + "jusqu'à ce que ça bouillonne franchement", + ], + }, + en: { + // NOT "boiling point" — a word-prefix extension of "boiling" above + // (see `cook`'s comment for why that duplicates NER candidates). + synonyms: ["boil", "boils", "boiled", "boiling", "rolling boil"], + utterances: [ + "bring to a boil", + "boil the water", + "large bubbles should be vigorously breaking the surface", + "until it's rolling vigorously", + ], + }, + }, + { + uid: "roast", + fr: { + // NOT "rôti au four" — a word-prefix extension of "rôti" above (see + // `cook`'s comment for why that duplicates NER candidates). + synonyms: ["rôtir", "rôti", "rôtie", "rôties", "rôtis", "rôtissage"], + utterances: [ + "faire rôtir la volaille entière", + "le rôti doit dorer uniformément de tous les côtés", + "cuire la pièce de viande entière au four à chaleur sèche", + ], + }, + en: { + synonyms: ["roast", "roasts", "roasted", "roasting", "oven-roast", "oven roasted"], + utterances: [ + "roast the whole bird", + "it should brown evenly on every side", + "cook the whole piece of meat in dry oven heat", + ], + }, + }, + { + uid: "grill", + fr: { + synonyms: [ + "griller", + "grillez", + "grillé", + "grillée", + "grillées", + "grillade", + "grillades", + "barbecue", + "au barbecue", + ], + utterances: [ + "faire griller sur la grille du barbecue", + "marquer les steaks sur une plaque brûlante", + "des traces de quadrillage doivent apparaître à la cuisson", + ], + }, + en: { + synonyms: ["grill", "grills", "grilled", "grilling", "barbecue", "char-grill", "charbroiled"], + utterances: [ + "grill on the barbecue rack", + "sear the steaks on a scorching-hot plate", + "char marks should appear as it cooks", + ], + }, + }, + { + uid: "panFry", + fr: { + // Deliberately NOT "poêlé"/"poêlée"/"poêlés" here, despite reading + // like natural panFry vocabulary: node-nlp's French stemmer reduces + // them to the same root as the bare noun "poêle" (a pan), so + // registering them made every plain mention of "poêle" — e.g. + // `preheat`'s own "la poêle" — a false-positive panFry candidate too. + // Found via the "jusqu'à ce que le beurre ait disparu dans la poêle" + // regression test, which unexpectedly grew a spurious panFry match. + synonyms: ["sauter", "sautez", "sauté", "sautée", "sautées", "sautant", "à la poêle"], + utterances: [ + "faire sauter les légumes à la poêle", + "saisir rapidement à feu vif en remuant sans cesse", + "faire revenir en remuant vivement dans une poêle très chaude", + ], + }, + en: { + synonyms: [ + "sauté", + "sauteed", + "sautéed", + "sauteing", + "pan-fry", + "pan fried", + "pan-fried", + "stir-fry", + "pan searing", + "seared in a pan", + ], + utterances: [ + "sauté the vegetables in a pan", + "quickly sear over high heat, stirring constantly", + "cook briskly, stirring, in a very hot pan", + ], + }, + }, + { + uid: "blanch", + fr: { + synonyms: ["blanchir", "blanchissez", "blanchi", "blanchie", "blanchies", "blanchiment"], + utterances: [ + "faire blanchir les légumes deux minutes dans l'eau bouillante", + "plonger brièvement dans l'eau bouillante puis directement dans l'eau glacée", + "cuire très rapidement à l'eau bouillante avant de stopper la cuisson au froid", + ], + }, + en: { + // "parboil" is folded in here rather than kept a separate technique — + // in home-cooking usage (as opposed to professional usage, where they + // can differ) it names the same "briefly pre-cook in boiling water" + // move blanching does. + synonyms: [ + "blanch", + "blanches", + "blanched", + "blanching", + "parboil", + "parboiled", + "parboiling", + ], + utterances: [ + "blanch the vegetables for two minutes in boiling water", + "briefly plunge into boiling water then straight into ice water", + "cook very quickly in boiling water before stopping it cold", + ], + }, + }, + { + uid: "marinate", + fr: { + synonyms: [ + "mariner", + "marinez", + "mariné", + "marinée", + "marinées", + "marinade", + "macérer", + "macérez", + "macération", + "faire mariner", + ], + utterances: [ + "laisser mariner la viande toute la nuit au réfrigérateur", + "faire tremper dans la sauce plusieurs heures avant cuisson pour parfumer", + "laisser reposer dans le mélange d'huile et d'épices avant de cuisiner", + ], + }, + en: { + // NOT "marinating for" — a word-prefix extension of "marinating" + // above (see `cook`'s comment for why that duplicates NER candidates + // — here it was even worse, misclassifying as `simmer`). + synonyms: [ + "marinate", + "marinates", + "marinated", + "marinating", + "marinade", + "soak in the marinade", + ], + utterances: [ + "let the meat marinate overnight in the fridge", + "soak in the sauce for several hours before cooking to flavor it", + "let it sit in the oil and spice mixture before cooking", + ], + }, + }, + { + uid: "chop", + fr: { + // NOT "hacher grossièrement" — a word-prefix extension of "hacher" + // above (see `cook`'s comment for why that duplicates NER candidates). + synonyms: [ + "hacher", + "hachez", + "haché", + "hachée", + "hachées", + "hachis", + "couper en morceaux", + "tailler en morceaux", + ], + utterances: [ + "hacher finement les oignons", + "couper en tout petits morceaux irréguliers au couteau", + "réduire les herbes en petits fragments avant de les ajouter", + ], + }, + en: { + // NOT "chop coarsely" — a word-prefix extension of "chop" above (see + // `cook`'s comment for why that duplicates NER candidates). + synonyms: ["chop", "chops", "chopped", "chopping", "roughly chop", "coarsely chopped"], + utterances: [ + "finely chop the onions", + "cut into small, uneven pieces with a knife", + "break the herbs down into small bits before adding them", + ], + }, + }, + { + uid: "peel", + fr: { + synonyms: [ + "éplucher", + "épluchez", + "épluché", + "épluchée", + "épluchées", + "épluchage", + "peler", + "pelez", + "pelé", + "pelée", + "pelées", + ], + utterances: [ + "éplucher les pommes de terre", + "retirer la peau des carottes avec un économe", + "ôter la pelure du fruit avant de le couper", + ], + }, + en: { + synonyms: ["peel", "peels", "peeled", "peeling", "pare", "pared", "paring"], + utterances: [ + "peel the potatoes", + "remove the skin from the carrots with a peeler", + "take the skin off the fruit before cutting it", + ], + }, + }, + { + uid: "mince", + fr: { + synonyms: [ + "émincer", + "émincez", + "émincé", + "émincée", + "émincées", + "ciseler", + "ciselez", + "ciselé", + "ciselée", + "ciselées", + ], + utterances: [ + "émincer l'oignon en fines lamelles", + "couper en très fines tranches régulières", + "détailler en lamelles aussi fines que possible", + // Without this, a short clause naming a different vegetable — + // "Émincer les tomates" — scored just above `melt`'s confidence + // threshold instead (a training-set-composition side effect of + // adding utterances elsewhere in this same pass, found by the full + // regression suite). A second example anchored on a different noun + // widens `mince`'s own region enough to reclaim it. + "émincer les tomates en fines rondelles", + ], + }, + en: { + // NOT "mince finely" — a word-prefix extension of "mince" above (see + // `cook`'s comment for why that duplicates NER candidates). + synonyms: ["mince", "minces", "minced", "mincing", "thinly slice", "finely mince"], + utterances: [ + "mince the onion into thin strips", + "cut into very thin, even slices", + "slice into strips as thin as possible", + ], + }, + }, + { + uid: "mix", + fr: { + synonyms: [ + "mélanger", + "mélangez", + "mélangé", + "mélangée", + "mélangées", + "mélange", + "brasser", + "brassez", + "amalgamer", + "amalgamez", + ], + utterances: [ + "mélanger tous les ingrédients dans un saladier", + "combiner le sucre et la farine ensemble", + "remuer jusqu'à obtenir une préparation homogène", + ], + }, + en: { + synonyms: [ + "mix", + "mixes", + "mixed", + "mixing", + "combine", + "combined", + "blend", + "blended", + "blending", + "stir together", + ], + utterances: [ + "mix all the ingredients in a bowl", + "combine the sugar and flour together", + "stir until the mixture is smooth and even", + ], + }, + }, + { + uid: "whisk", + fr: { + synonyms: [ + "fouetter", + "fouettez", + "fouetté", + "fouettée", + "fouettées", + "au fouet", + "battre au fouet", + "monter au fouet", + ], + utterances: [ + "fouetter les œufs et le sucre", + "battre vigoureusement au fouet jusqu'à ce que ça blanchisse", + "travailler énergiquement pour incorporer de l'air au mélange", + // Without these, "Fouetter les blancs en neige" misclassified as + // `foldIn` — its own training utterance below also happens to say + // "les blancs en neige", and node-nlp's intent classifier leaned on + // that shared noun phrase over the actual verb. The exact phrase + // itself is needed (not just a paraphrase of it) — a longer, + // differently-worded utterance alone wasn't enough to outweigh + // `foldIn`'s own close phrasing. + "fouetter les blancs en neige", + "fouetter les blancs en neige jusqu'à ce qu'ils soient fermes", + ], + }, + en: { + synonyms: ["whisk", "whisks", "whisked", "whisking", "beat", "whip", "whipped", "whipping"], + utterances: [ + "whisk the eggs and sugar", + "beat vigorously with a whisk until pale", + "work it briskly to whip air into the mixture", + "whisk the egg whites until stiff peaks form", + ], + }, + }, + { + uid: "foldIn", + fr: { + synonyms: [ + "incorporer", + "incorporez", + "incorporé", + "incorporée", + "incorporées", + // NOT "incorporer délicatement" — it's a superstring of "incorporer" + // above, so both would match the same text and hand + // `splitIntoClauses` two overlapping candidates for one mention + // (found via "Incorporer délicatement la farine" producing two + // duplicate matches instead of one). + "mélanger délicatement", + ], + utterances: [ + "incorporer délicatement les blancs en neige", + "ajouter en soulevant doucement la masse pour ne pas casser les bulles", + "mélanger tout doucement de bas en haut pour garder l'air emprisonné", + ], + }, + en: { + synonyms: ["fold in", "folds in", "folded in", "folding in", "gently fold", "fold gently"], + utterances: [ + "gently fold in the beaten egg whites", + "add by gently lifting the batter so you don't knock the air out", + "very gently stir from the bottom up to keep the air trapped in", + ], + }, + }, + { + uid: "setAside", + fr: { + synonyms: [ + "réserver", + "réservez", + "réservé", + "réservée", + "réservées", + "mettre de côté", + "laisser de côté", + ], + utterances: [ + "réserver au frais en attendant", + "mettre de côté pour plus tard", + "laisser attendre sur le plan de travail pendant la préparation du reste", + ], + }, + en: { + synonyms: ["set aside", "sets aside", "setting aside", "set it aside", "reserve", "reserved"], + utterances: [ + "set aside in the fridge for now", + "put it aside for later", + "let it wait on the counter while you prepare the rest", + ], + }, + }, + { + uid: "season", + fr: { + synonyms: [ + "assaisonner", + "assaisonnez", + "assaisonné", + "assaisonnée", + "assaisonnement", + "relever", + "relevez", + "épicer", + "épicez", + ], + utterances: [ + "assaisonner avec du sel et du poivre", + "rectifier le goût en ajoutant des épices", + "ajouter du sel selon votre goût avant de servir", + ], + }, + en: { + synonyms: ["season", "seasons", "seasoned", "seasoning", "spice it up", "add seasoning"], + utterances: [ + "season with salt and pepper", + "adjust the taste by adding spices", + "add salt to taste before serving", + ], + }, + }, + { + uid: "drain", + fr: { + synonyms: [ + "égoutter", + "égouttez", + "égoutté", + "égouttée", + "égouttées", + "essorer", + "essorez", + "essoré", + "essorée", + ], + utterances: [ + "égoutter les pâtes dans une passoire", + "verser dans une passoire pour retirer l'eau de cuisson", + "laisser l'excédent d'eau s'écouler avant de servir", + ], + }, + en: { + synonyms: ["drain", "drains", "drained", "draining", "strain", "strained", "straining"], + utterances: [ + "drain the pasta in a colander", + "pour into a colander to remove the cooking water", + "let the excess water run off before serving", + ], + }, + }, + { + uid: "brown", + fr: { + synonyms: [ + "faire revenir", + "faites revenir", + "faire dorer", + "faites dorer", + "colorer", + "colorez", + "faire colorer", + ], + utterances: [ + "faire revenir les oignons dans l'huile chaude", + "faire dorer la viande sur toutes les faces", + "saisir jusqu'à ce que la surface prenne une belle couleur caramel", + ], + }, + en: { + // Verb forms only (not bare "brown"), same reasoning the old regex + // doc comment gave — a bare "brown" false-positives on ingredient + // descriptions like "brown sugar"/"brown rice", which never get to + // the classifier since they're not step text, but keeping the + // synonym itself anchored costs nothing and stays consistent. + synonyms: ["browned", "browning"], + utterances: [ + "brown the onions in hot oil", + "brown the meat on every side", + "sear until the surface turns a deep caramel color", + ], + }, + }, + { + uid: "rest", + fr: { + synonyms: ["reposer", "laisser reposer", "laissez reposer", "temps de repos"], + utterances: [ + "laisser reposer la pâte trente minutes", + "laisser la viande se détendre hors du four avant de la découper", + "attendre quelques minutes avant de servir pour que les jus se répartissent", + ], + }, + en: { + // Anchored to "let ... rest"/"rest for" rather than bare "rest", + // same false-positive reasoning as `brown` above ("the rest of the"). + synonyms: ["let it rest", "let them rest", "resting for", "rested for", "resting time"], + utterances: [ + "let the dough rest for thirty minutes", + "let the meat relax outside the oven before carving it", + "wait a few minutes before serving so the juices redistribute", + ], + }, + }, + { + uid: "preheat", + fr: { + synonyms: [ + "préchauffer", + "préchauffez", + "préchauffé", + "préchauffée", + // A pan already described as hot ("poêle chaude") implies it's + // been preheated, without the verb itself — the classic "Dans une + // poêle chaude, faire chauffer une noix de beurre" case (both + // `preheat` and `melt` in one instruction). + "poêle chaude", + "préchauffage", + ], + utterances: [ + "préchauffer le four à 180 degrés", + "mettre le four à chauffer avant d'y placer le plat", + "allumer le four à l'avance pour qu'il soit à température", + // A pan gets preheated too, not just an oven — without an example + // like this, "poêle" (which also appears throughout `panFry`'s own + // training utterances) biased the classifier toward `panFry` for + // any preheating clause that happens to mention a pan, found while + // testing against the classic "Préchauffer la poêle, puis faire + // fondre le beurre" case. + "préchauffer la poêle avant d'y verser l'huile", + "faire chauffer la poêle à vide quelques minutes", + // "poêle" + "feu vif" together still read as `panFry` (the act of + // actually cooking something in it) rather than `preheat` (getting + // it hot beforehand, nothing in it yet) without an example this + // close to that exact wording — found via "mettre la poêle sur feu + // vif" (no food mentioned at all) still classifying as panFry. + "mettre la poêle vide sur feu vif avant d'ajouter quoi que ce soit", + "mettre la poêle sur feu vif", + ], + }, + en: { + // NOT "preheating time" — a word-prefix extension of "preheating" + // above (see `cook`'s comment for why that duplicates NER candidates). + synonyms: ["preheat", "preheats", "preheated", "preheating", "hot pan"], + utterances: [ + "preheat the oven to 180 degrees", + "turn the oven on to heat up before putting the dish in", + "switch the oven on ahead of time so it's up to temperature", + "preheat the pan before adding the oil", + "heat the empty pan for a few minutes first", + ], + }, + }, + { + uid: "bake", + fr: { + synonyms: [ + "cuire au four", + "cuisson au four", + "enfourner", + "enfournez", + "au four", + "enfourné", + "enfournée", + ], + utterances: [ + "enfourner pendant quarante-cinq minutes", + "mettre au four jusqu'à ce que ce soit doré", + "cuire dans le four préchauffé jusqu'à ce que la surface soit ferme", + ], + }, + en: { + // NOT "baked in the oven" — a word-prefix extension of "baked" above + // (see `cook`'s comment for why that duplicates NER candidates). + synonyms: ["bake", "bakes", "baked", "baking", "in the oven", "oven-baked"], + utterances: [ + "bake for forty-five minutes", + "put it in the oven until golden", + "cook in the preheated oven until the surface is firm", + ], + }, + }, + { + uid: "plate", + fr: { + // NOT "dressage de l'assiette" — a word-prefix extension of + // "dressage" above (see `cook`'s comment for why that duplicates NER + // candidates). + synonyms: ["dresser", "dressez", "dressage", "disposer dans l'assiette"], + utterances: [ + "dresser harmonieusement dans les assiettes", + "disposer joliment sur l'assiette avant de servir", + "présenter avec soin au centre de l'assiette", + ], + }, + en: { + // NOT "plate up"/"plated nicely" — both are word-prefix extensions of + // "plate"/"plated" above (see `cook`'s comment for why that + // duplicates NER candidates). + synonyms: ["plate", "plates", "plated", "plating"], + utterances: [ + "plate it up nicely", + "arrange it neatly on the plate before serving", + "present it carefully in the center of the plate", + ], + }, + }, + { + uid: "coat", + fr: { + synonyms: [ + "napper", + "nappez", + "nappé", + "nappée", + "nappées", + "nappage", + "enrober", + "enrobez", + "enrobé", + "enrobée", + "enrobées", + ], + utterances: [ + "napper le gâteau de chocolat fondu", + "recouvrir uniformément d'une fine couche de sauce", + "verser la sauce par-dessus pour bien enrober", + ], + }, + en: { + // NOT "coat evenly" — a word-prefix extension of "coat" above (see + // `cook`'s comment for why that duplicates NER candidates). + synonyms: ["coat", "coats", "coated", "coating", "dredge", "dredged", "dredging"], + utterances: [ + "coat the cake with melted chocolate", + "cover evenly with a thin layer of sauce", + "pour the sauce over it so it's well covered", + ], + }, + }, +]; diff --git a/apps/api/src/lib/recipe-sources/recipe-source-adapter.ts b/apps/api/src/lib/recipe-sources/recipe-source-adapter.ts index d663855..822e32b 100644 --- a/apps/api/src/lib/recipe-sources/recipe-source-adapter.ts +++ b/apps/api/src/lib/recipe-sources/recipe-source-adapter.ts @@ -16,9 +16,9 @@ * user has already brought in as if they were new. A separate, pure * step rather than something `list()` itself does: an adapter only * knows its source, never our database — same reasoning as - * `tech-step-matcher.ts`'s split between pure `matchTechStep` and its - * DB-touching `loadTechStepMappingRules`. Whichever future layer - * queries "which externalIds from this source do we already have" + * `tech-step-matcher.ts`'s split between pure `splitIntoClauses` and + * its DB/model-touching `TechStepClassifierService`. Whichever future + * layer queries "which externalIds from this source do we already have" * (not yet decided — it needs a place to persist that link, * see {@link RecipeSourceListItem.externalId}) calls this to annotate * the page before returning it. @@ -177,7 +177,7 @@ export interface RecipeSourceAdapter { * `steps[].description`/`ingredients[].name`) — e.g. `"en"` for * TheMealDB. Not a user preference: the language the source's own * content is actually written in, regardless of who's browsing it. - * Determines which `TechStepMapping`/ingredient-label locale + * Determines which trained-classifier/ingredient-label locale * `translateRecipe` (`recipe-translation.ts`) resolves this source's * recipes against when previewing/importing one. */ diff --git a/apps/api/src/modules/recipe/recipe.service.ts b/apps/api/src/modules/recipe/recipe.service.ts index 64a2e1b..ea6df3b 100644 --- a/apps/api/src/modules/recipe/recipe.service.ts +++ b/apps/api/src/modules/recipe/recipe.service.ts @@ -15,15 +15,15 @@ import { import type { Prisma } from "@prisma/client"; import { prisma } from "../../db/prisma.js"; import { - loadTechStepMappingRules, - matchTechStepSpans, + type TechStepMatch, + techStepClassifier, } from "../../lib/recipe-matching/tech-step-matcher.js"; // No user-language preference exists anywhere in the app yet (a single // "fr" translation file, no locale field on User/UserProfile) — steps are // matched against this hardcoded locale for now. See -// `tech-step-matcher.ts`'s `loadTechStepMappingRules` for why the locale is -// a parameter rather than baked into that module. +// `tech-step-matcher.ts`'s `TechStepClassifierService` for why the locale +// is a parameter rather than baked into that module. const DEFAULT_TECH_STEP_LOCALE = "fr"; /** Prisma `include` for every query that needs a full {@link RecipeView} — ingredients resolved to their reference data + allergens, steps in order, diet tags, and whether `viewerId` has favorited it. Parameterized by viewer since `favoritedBy` is per-viewer, not a static shape. */ @@ -123,24 +123,28 @@ function toRecipeSummaryView(recipe: RecipeWithDetails): RecipeSummaryView { /** * Shapes a step's `StepTechStep` rows into {@link StepTechStepView}s — a row - * whose `start`/`end` is still `null` (a pre-existing row saved before this - * column existed, not yet recomputed by a resave — see the schema doc + * whose `start`/`end` is still `null` (a pre-existing row saved before that + * column pair existed, not yet recomputed by a resave — see the schema doc * comment on `StepTechStep`) is dropped rather than surfaced with a null * span, so the frontend only ever deals with real, highlightable matches. + * `contextStart`/`contextEnd` are treated more leniently — a row with a + * real keyword span but no context (saved before *that* column pair + * existed) still has a perfectly good match to show, just without the + * wider highlight, so those two are included only when both are present + * rather than dropping the whole entry over a still-missing "nice to have". */ function toStepTechStepViews( techSteps: RecipeWithDetails["steps"][number]["techSteps"], ): StepTechStepView[] { const views: StepTechStepView[] = []; for (const stepTechStep of techSteps) { - if (stepTechStep.start === null || stepTechStep.end === null) continue; + const { start, end, contextStart, contextEnd, techStep } = stepTechStep; + if (start === null || end === null) continue; views.push({ - techStep: { - id: stepTechStep.techStep.id, - key: stepTechStep.techStep.key, - }, - start: stepTechStep.start, - end: stepTechStep.end, + techStep: { id: techStep.id, key: techStep.key }, + start, + end, + ...(contextStart !== null && contextEnd !== null ? { contextStart, contextEnd } : {}), }); } return views; @@ -454,6 +458,36 @@ export async function createImportedRecipe( } } +/** One input step, bundled with its own technique matches — see {@link matchStepsTechSteps}. */ +interface StepWithTechSteps { + step: T; + matches: TechStepMatch[]; +} + +/** + * Matches every one of `steps`' technique sequence against `locale`, in + * parallel, each bundled back with its own originating step (rather than + * returned as a same-length array callers would have to re-zip with + * `steps` by index — `noUncheckedIndexedAccess` makes that genuinely + * awkward for no benefit, since every match list is only ever read back + * once) — the shared prep step {@link createRecipeInternal}/ + * {@link updateRecipe} both need before building their (synchronous) + * Prisma `create` payload, now that matching itself is async + * (`techStepClassifier`, a trained model rather than a pure regex test — + * see `tech-step-matcher.ts`). + */ +async function matchStepsTechSteps( + steps: T[], + locale: string, +): Promise[]> { + return Promise.all( + steps.map(async (step) => ({ + step, + matches: await techStepClassifier.matchTechStepSpans(step.description, locale), + })), + ); +} + async function createRecipeInternal( input: CreateRecipeInput, authorId: number, @@ -464,7 +498,13 @@ async function createRecipeInternal( await assertIngredientsExist(input.ingredients.map((i) => i.ingredientId)); await assertUnitsExist(input.ingredients.map((i) => i.unitId)); await assertDietsExist(input.dietIds); - const techStepMappings = await loadTechStepMappingRules( + // Matched up front (one call per step, in parallel) rather than inline + // inside the `steps.create` map below — `techStepClassifier` is async + // (a trained model, not a pure regex test), so its result has to + // already be in hand by the time this synchronous Prisma payload is + // built. + const stepsWithTechSteps = await matchStepsTechSteps( + input.steps, source?.locale ?? DEFAULT_TECH_STEP_LOCALE, ); @@ -487,19 +527,19 @@ async function createRecipeInternal( })), }, steps: { - create: input.steps.map((step, index) => ({ + create: stepsWithTechSteps.map(({ step, matches }, index) => ({ description: step.description, picture: step.picture ?? null, order: index, techSteps: { - create: matchTechStepSpans(step.description, techStepMappings).map( - (match, order) => ({ - techStepId: match.techStepId, - start: match.start, - end: match.end, - order, - }), - ), + create: matches.map((match, order) => ({ + techStepId: match.techStepId, + start: match.start, + end: match.end, + contextStart: match.contextStart, + contextEnd: match.contextEnd, + order, + })), }, })), }, @@ -537,7 +577,7 @@ export async function updateRecipe( await assertIngredientsExist(input.ingredients.map((i) => i.ingredientId)); await assertUnitsExist(input.ingredients.map((i) => i.unitId)); await assertDietsExist(input.dietIds); - const techStepMappings = await loadTechStepMappingRules(DEFAULT_TECH_STEP_LOCALE); + const stepsWithTechSteps = await matchStepsTechSteps(input.steps, DEFAULT_TECH_STEP_LOCALE); await prisma.$transaction([ prisma.recipeIngredient.deleteMany({ where: { recipeId: id } }), @@ -559,19 +599,19 @@ export async function updateRecipe( })), }, steps: { - create: input.steps.map((step, index) => ({ + create: stepsWithTechSteps.map(({ step, matches }, index) => ({ description: step.description, picture: step.picture ?? null, order: index, techSteps: { - create: matchTechStepSpans(step.description, techStepMappings).map( - (match, order) => ({ - techStepId: match.techStepId, - start: match.start, - end: match.end, - order, - }), - ), + create: matches.map((match, order) => ({ + techStepId: match.techStepId, + start: match.start, + end: match.end, + contextStart: match.contextStart, + contextEnd: match.contextEnd, + order, + })), }, })), }, diff --git a/apps/api/src/modules/sources/sources.service.ts b/apps/api/src/modules/sources/sources.service.ts index 5ca4317..36e8a45 100644 --- a/apps/api/src/modules/sources/sources.service.ts +++ b/apps/api/src/modules/sources/sources.service.ts @@ -20,10 +20,7 @@ import { mergeDuplicateIngredients, translateRecipeIngredients, } from "../../lib/recipe-matching/recipe-translation.js"; -import { - loadTechStepMappingRules, - matchTechStepSpans, -} from "../../lib/recipe-matching/tech-step-matcher.js"; +import { techStepClassifier } from "../../lib/recipe-matching/tech-step-matcher.js"; import { markAlreadyImported, type RecipeSourceAdapter, @@ -172,14 +169,20 @@ export async function previewSourceItem( throw err; } - const [techStepMappings, ingredientCatalog, unitCatalog, techStepsByKey] = await Promise.all([ - loadTechStepMappingRules(adapter.locale), - adapter.locale === "en" - ? loadIngredientCatalog() - : Promise.resolve([]), - adapter.locale === "en" ? loadUnitCatalog() : Promise.resolve([]), - prisma.techStep.findMany({ select: { id: true, key: true } }), - ]); + const [stepsWithTechStepMatches, ingredientCatalog, unitCatalog, techStepsByKey] = + await Promise.all([ + Promise.all( + parsed.steps.map(async (step) => ({ + step, + matches: await techStepClassifier.matchTechStepSpans(step.description, adapter.locale), + })), + ), + adapter.locale === "en" + ? loadIngredientCatalog() + : Promise.resolve([]), + adapter.locale === "en" ? loadUnitCatalog() : Promise.resolve([]), + prisma.techStep.findMany({ select: { id: true, key: true } }), + ]); const techStepById = new Map(techStepsByKey.map((techStep) => [techStep.id, techStep])); const translatedIngredients = translateRecipeIngredients( @@ -209,12 +212,22 @@ export async function previewSourceItem( unit: ingredient.unitId !== null ? (unitById.get(ingredient.unitId) ?? null) : null, })); - const steps: DraftRecipeStepView[] = parsed.steps.map((step) => ({ + const steps: DraftRecipeStepView[] = stepsWithTechStepMatches.map(({ step, matches }) => ({ description: step.description, picture: step.picture, - techSteps: matchTechStepSpans(step.description, techStepMappings).flatMap((match) => { + techSteps: matches.flatMap((match) => { const techStep = techStepById.get(match.techStepId); - return techStep ? [{ techStep, start: match.start, end: match.end }] : []; + return techStep + ? [ + { + techStep, + start: match.start, + end: match.end, + contextStart: match.contextStart, + contextEnd: match.contextEnd, + }, + ] + : []; }), })); diff --git a/apps/api/src/scripts/backfill-tech-steps.ts b/apps/api/src/scripts/backfill-tech-steps.ts new file mode 100644 index 0000000..519f107 --- /dev/null +++ b/apps/api/src/scripts/backfill-tech-steps.ts @@ -0,0 +1,60 @@ +import { prisma } from "../db/prisma.js"; +import { techStepClassifier } from "../lib/recipe-matching/tech-step-matcher.js"; + +/** + * One-off maintenance script: recomputes every existing `Step`'s + * `StepTechStep` sequence against the *current* classifier + * (`tech-step-matcher.ts`/`tech-step-training-data.ts`), the same way + * `updateRecipe` does when a user resaves a recipe through the UI — + * always `"fr"` (`DEFAULT_TECH_STEP_LOCALE` in `recipe.service.ts`; there's + * no persisted per-recipe locale to recover for a step that already exists, + * so this matches real resave behavior exactly rather than guessing). + * + * Needed because tech-step detection only ever runs at create/update time + * (`recipe.service.ts`'s `matchStepsTechSteps`), never retroactively — a + * step saved before a classifier/corpus change (new vocabulary, or the + * `contextStart`/`contextEnd` columns this same session added) keeps + * whatever it was matched with at the time until it's next resaved. Run + * this after a corpus change to bring every existing step in sync without + * asking users to open and resave every recipe by hand: + * + * pnpm --filter api exec tsx src/scripts/backfill-tech-steps.ts + * + * Safe to re-run: each step's technique sequence is fully replaced (delete + * + recreate) from the classifier's current output, same as a real edit — + * running it twice in a row with no corpus change in between is a no-op. + */ +async function backfillTechSteps(): Promise { + const steps = await prisma.step.findMany({ select: { id: true, description: true } }); + console.info(`Recomputing tech steps for ${steps.length} step(s)...`); + + let changed = 0; + for (const step of steps) { + const matches = await techStepClassifier.matchTechStepSpans(step.description, "fr"); + await prisma.$transaction([ + prisma.stepTechStep.deleteMany({ where: { stepId: step.id } }), + prisma.stepTechStep.createMany({ + data: matches.map((match, order) => ({ + stepId: step.id, + techStepId: match.techStepId, + order, + start: match.start, + end: match.end, + contextStart: match.contextStart, + contextEnd: match.contextEnd, + })), + }), + ]); + changed += 1; + } + + console.info(`Done — ${changed} step(s) recomputed.`); +} + +backfillTechSteps() + .then(() => prisma.$disconnect()) + .catch(async (err) => { + console.error(err); + await prisma.$disconnect(); + process.exit(1); + }); diff --git a/apps/api/src/server.ts b/apps/api/src/server.ts index 85f2cf9..9c477a9 100644 --- a/apps/api/src/server.ts +++ b/apps/api/src/server.ts @@ -1,6 +1,7 @@ import { createServer } from "./app.js"; import { env } from "./config/env.js"; import { logger } from "./lib/logger.service.js"; +import { techStepClassifier } from "./lib/recipe-matching/tech-step-matcher.js"; import { registerAllRecipeSources } from "./sources/index.js"; // Populates the recipe-source registry (recipe-source-registry.ts) before @@ -8,6 +9,22 @@ import { registerAllRecipeSources } from "./sources/index.js"; // doesn't happen inside app.ts/createServer() itself. registerAllRecipeSources(); +// Trains the tech-step classifier (and pays node-nlp's own one-time lazy +// setup cost — see `TechStepClassifierService.warmUp`) before accepting +// any traffic, so the first real recipe save/preview isn't the one stuck +// waiting several seconds for it. +try { + await techStepClassifier.warmUp(); +} catch (err) { + // Not fatal to startup — a failed warm-up just means the *next* call + // retries training itself (see `_ensureTrained`'s own retry-on-failure + // comment), same graceful-degrade posture as everywhere else training + // failures surface. Still worth a loud log: this shouldn't normally fail. + logger.error("Tech-step classifier warm-up failed", { + error: err instanceof Error ? err.message : String(err), + }); +} + const server = createServer(); server.listen(env.PORT, () => { diff --git a/apps/api/src/types/node-nlp.d.ts b/apps/api/src/types/node-nlp.d.ts new file mode 100644 index 0000000..36d8fa3 --- /dev/null +++ b/apps/api/src/types/node-nlp.d.ts @@ -0,0 +1,50 @@ +/** + * Minimal ambient typing for `node-nlp` (no official/DefinitelyTyped types + * exist for it) — declares only the `NlpManager` surface + * `tech-step-matcher.ts` actually calls, verified against the real + * package (v4.27.0) rather than the library's full documented API, which + * this repo doesn't use the rest of. + */ +declare module "node-nlp" { + /** Constructor options this repo passes — `NlpManager` accepts more, only what's used here is typed. */ + export interface NlpManagerOptions { + languages?: string[]; + forceNER?: boolean; + nlu?: { log?: boolean }; + ner?: { threshold?: number }; + /** Defaults to `true` — persists the trained model to `modelFileName` (default `model.nlp`, in `process.cwd()`). See `tech-step-matcher.ts`'s own constructor comment for why this repo always sets it `false`. */ + autoSave?: boolean; + /** Defaults to `true` — loads from `modelFileName` instead of training fresh if that file already exists. Always `false` here, same reasoning as `autoSave`. */ + autoLoad?: boolean; + } + + /** One entity `NlpManager.process`'s result reports — see `tech-step-matcher.ts`'s own `NerEntity` for the subset this repo reads. */ + export interface NlpEntity { + entity: string; + start: number; + end: number; + type: string; + accuracy?: number; + sourceText?: string; + } + + /** `NlpManager.process`'s result — trimmed to the fields this repo reads (the real object carries many more). */ + export interface NlpProcessResult { + intent: string; + score: number; + entities: NlpEntity[]; + } + + export class NlpManager { + public constructor(options?: NlpManagerOptions); + public addNamedEntityText( + entityName: string, + optionName: string, + languages: string[], + texts: string[], + ): void; + public addDocument(locale: string, utterance: string, intent: string): void; + public train(): Promise; + public process(locale: string, text: string): Promise; + } +} diff --git a/apps/api/test-support/reset-db.ts b/apps/api/test-support/reset-db.ts index cdf8e04..c12b1ac 100644 --- a/apps/api/test-support/reset-db.ts +++ b/apps/api/test-support/reset-db.ts @@ -45,7 +45,7 @@ export async function resetDatabase() { TRUNCATE TABLE "user_profile_allergy", "user_preference", "allergy", "category", "planning_item", "planning", - "recipe_ingredient", "step_tech_step", "step", "tech_step_mapping", "tech_step", + "recipe_ingredient", "step_tech_step", "step", "tech_step", "recipe", "ingredients", "sources", "unit", "user_profiles", "diet", "house" RESTART IDENTITY CASCADE; diff --git a/apps/api/test/recipe-matching/recipe-translation.test.ts b/apps/api/test/recipe-matching/recipe-translation.test.ts index c74b98d..b06c51e 100644 --- a/apps/api/test/recipe-matching/recipe-translation.test.ts +++ b/apps/api/test/recipe-matching/recipe-translation.test.ts @@ -12,7 +12,6 @@ import { translateRecipeSteps, type UnitConversionEntry, } from "../../src/lib/recipe-matching/recipe-translation.js"; -import type { TechStepMappingRule } from "../../src/lib/recipe-matching/tech-step-matcher.js"; import type { ParsedRecipe, ParsedRecipeIngredient, @@ -36,56 +35,66 @@ function buildParsedRecipe(descriptions: string[]): ParsedRecipe { } describe("recipe-translation", () => { + // `translateRecipeSteps` now goes through `techStepClassifier` (a + // trained model, not a pure regex test against a caller-supplied + // mapping list — see `tech-step-matcher.ts`), so these tests exercise + // the real training corpus (`tech-step-training-data.ts`) against a real + // `TechStep` catalog rather than synthetic fixtures — same posture + // `tech-step-matcher.test.ts`'s own `techStepClassifier` describe block + // takes, for the same reason. describe("translateRecipeSteps", () => { - const simmer: TechStepMappingRule = { - techStepId: 1, - expression: "\\bmijot(er|ez|e|ant|é)\\b", - weight: 15, - }; - const preheat: TechStepMappingRule = { - techStepId: 2, - expression: "\\bpr[ée]chauff(er|ez|é|ée)\\b", - weight: 20, - }; - const melt: TechStepMappingRule = { - techStepId: 3, - expression: "\\bfaire fondre\\b|\\bfaites fondre\\b", - weight: 15, - }; + let simmerId: number; + let preheatId: number; + let meltId: number; - it("declares each step's technique sequence, preserving order", () => { + beforeEach(async () => { + await resetDatabase(); + simmerId = (await prisma.techStep.findFirstOrThrow({ where: { key: "simmer" } })).id; + preheatId = (await prisma.techStep.findFirstOrThrow({ where: { key: "preheat" } })).id; + meltId = (await prisma.techStep.findFirstOrThrow({ where: { key: "melt" } })).id; + }); + + after(async () => { + await prisma.$disconnect(); + }); + + it("declares each step's technique sequence, preserving order", async () => { const recipe = buildParsedRecipe([ "Préchauffer la poêle, puis faire fondre le beurre", "Servir immédiatement", "Faire mijoter à feu doux", ]); - const translated = translateRecipeSteps(recipe, [simmer, preheat, melt]); + const translated = await translateRecipeSteps(recipe, "fr"); - expect(translated.steps.map((step) => step.techStepIds)).to.deep.equal([[2, 3], [], [1]]); + expect(translated.steps.map((step) => step.techStepIds)).to.deep.equal([ + [preheatId, meltId], + [], + [simmerId], + ]); }); - it("leaves description/picture untouched on each step", () => { - const recipe = buildParsedRecipe(["Faire mijoter à feu doux", "Servir"]); + it("leaves description/picture untouched on each step", async () => { + const recipe = buildParsedRecipe(["Faire mijoter à feu doux", "Servir immédiatement"]); - const translated = translateRecipeSteps(recipe, [simmer]); + const translated = await translateRecipeSteps(recipe, "fr"); expect(translated.steps[0]).to.deep.equal({ description: "Faire mijoter à feu doux", picture: "https://example.test/step1.jpg", - techStepIds: [1], + techStepIds: [simmerId], }); expect(translated.steps[1]).to.deep.equal({ - description: "Servir", + description: "Servir immédiatement", picture: null, techStepIds: [], }); }); - it("passes every other field through unchanged", () => { - const recipe = buildParsedRecipe(["Servir"]); + it("passes every other field through unchanged", async () => { + const recipe = buildParsedRecipe(["Servir immédiatement"]); - const translated = translateRecipeSteps(recipe, []); + const translated = await translateRecipeSteps(recipe, "fr"); expect(translated.name).to.equal(recipe.name); expect(translated.description).to.equal(recipe.description); @@ -94,28 +103,31 @@ describe("recipe-translation", () => { expect(translated.sourceUrl).to.equal(recipe.sourceUrl); }); - it("stubs every ingredient's ingredientId/unitId to null, leaving the rest of it untouched — actual ingredient matching is translateRecipeIngredients' job", () => { - const recipe = buildParsedRecipe(["Servir"]); + it("stubs every ingredient's ingredientId/unitId to null, leaving the rest of it untouched — actual ingredient matching is translateRecipeIngredients' job", async () => { + const recipe = buildParsedRecipe(["Servir immédiatement"]); - const translated = translateRecipeSteps(recipe, []); + const translated = await translateRecipeSteps(recipe, "fr"); expect(translated.ingredients).to.deep.equal([ { ...recipe.ingredients[0], ingredientId: null, unitId: null }, ]); }); - it("gives every step an empty sequence when there are no mappings at all", () => { - const recipe = buildParsedRecipe(["Faire mijoter à feu doux", "Préchauffer le four"]); + it("gives every step an empty sequence when nothing in it means a known technique", async () => { + const recipe = buildParsedRecipe([ + "Servir immédiatement", + "Ranger les couverts dans le tiroir", + ]); - const translated = translateRecipeSteps(recipe, []); + const translated = await translateRecipeSteps(recipe, "fr"); expect(translated.steps.map((step) => step.techStepIds)).to.deep.equal([[], []]); }); - it("handles a recipe with no steps without error", () => { + it("handles a recipe with no steps without error", async () => { const recipe = buildParsedRecipe([]); - const translated = translateRecipeSteps(recipe, [simmer]); + const translated = await translateRecipeSteps(recipe, "fr"); expect(translated.steps).to.deep.equal([]); }); diff --git a/apps/api/test/recipe-matching/tech-step-matcher.test.ts b/apps/api/test/recipe-matching/tech-step-matcher.test.ts index 6f3a7b8..a2fc55b 100644 --- a/apps/api/test/recipe-matching/tech-step-matcher.test.ts +++ b/apps/api/test/recipe-matching/tech-step-matcher.test.ts @@ -1,11 +1,10 @@ import { expect } from "chai"; import { prisma } from "../../src/db/prisma.js"; import { - loadTechStepMappingRules, - matchTechStepSpans, - matchTechSteps, normalizeText, - type TechStepMappingRule, + splitIntoClauses, + type TechniqueCandidate, + techStepClassifier, } from "../../src/lib/recipe-matching/tech-step-matcher.js"; import { resetDatabase } from "../../test-support/reset-db.js"; @@ -28,227 +27,321 @@ describe("tech-step-matcher", () => { }); }); - describe("matchTechSteps", () => { - const simmer: TechStepMappingRule = { - techStepId: 1, - expression: "\\bmijot(er|ez|e|ant|é)\\b", - weight: 15, - }; - const cook: TechStepMappingRule = { - techStepId: 2, - expression: "\\bcui(re|sez|sant|sson)\\b|\\bcuit(e|es|s)?\\b", - weight: 10, - }; - const bake: TechStepMappingRule = { - techStepId: 3, - expression: - "\\bcuire au four\\b|\\bcuisson au four\\b|\\benfourn(er|ez|é|ée|ées)\\b|\\bau four\\b", - weight: 25, - }; - const preheat: TechStepMappingRule = { - techStepId: 4, - expression: "\\bpr[ée]chauff(er|ez|é|ée)\\b", - weight: 20, - }; - const melt: TechStepMappingRule = { - techStepId: 5, - expression: "\\bfondre\\b|\\bfaire fondre\\b|\\bfaites fondre\\b", - weight: 15, - }; + describe("splitIntoClauses", () => { + // A candidate's own `uid` doesn't matter to the splitting logic itself + // (it's opaque, carried through as `anchor`) — kept short and + // arbitrary across these fixtures. + function candidate(uid: string, start: number, end: number): TechniqueCandidate { + return { uid, start, end }; + } - it("matches an exact expression", () => { - expect(matchTechSteps("Faire mijoter à feu doux", [simmer])).to.deep.equal([1]); + it("returns the whole description as one anchor-less clause when there are no candidates", () => { + const text = "Servir immédiatement"; + const result = splitIntoClauses(text, []); + expect(result).to.deep.equal([{ start: 0, end: text.length, anchor: null }]); }); - it("is case- and accent-insensitive, on both the description and the expression itself", () => { - // `simmer`'s own expression source contains a literal "é" — exercises - // normalizeText being applied to the expression, not just the description. - expect(matchTechSteps("FAIRE MIJOTER", [simmer])).to.deep.equal([1]); - expect(matchTechSteps("faire mijote", [simmer])).to.deep.equal([1]); + it("returns the whole description as one clause anchored on the single candidate", () => { + const melt = candidate("melt", 6, 13); + const text = "Faire fondre le beurre"; + const result = splitIntoClauses(text, [melt]); + expect(result).to.deep.equal([{ start: 0, end: text.length, anchor: melt }]); }); - it("returns an empty sequence when nothing matches", () => { - expect(matchTechSteps("Servir immédiatement", [simmer, cook, bake])).to.deep.equal([]); - }); - - it("returns an empty sequence for an empty mappings list", () => { - expect(matchTechSteps("Faire mijoter à feu doux", [])).to.deep.equal([]); - }); - - it("returns an empty sequence for an empty description", () => { - expect(matchTechSteps("", [simmer, cook, bake])).to.deep.equal([]); - }); - - it("detects several distinct, non-overlapping techniques as an ordered sequence", () => { - // The motivating case: "Dans une poêle chaude, faire chauffer une noix - // de beurre" involves both preheating and melting — a step can name - // more than one technique, in the order they're mentioned. - expect( - matchTechSteps("Préchauffer la poêle, puis faire fondre le beurre", [preheat, melt]), - ).to.deep.equal([4, 5]); - // Order in the output follows order of mention in the text, not - // argument order. - expect( - matchTechSteps("Préchauffer la poêle, puis faire fondre le beurre", [melt, preheat]), - ).to.deep.equal([4, 5]); - }); - - it("reverses the sequence when the techniques are mentioned in the opposite order", () => { - expect( - matchTechSteps("Faire fondre le beurre puis préchauffer le four", [preheat, melt]), - ).to.deep.equal([5, 4]); - }); - - it("keeps only the highest-weight technique when two different techniques' expressions overlap the same words", () => { - // "Cuire au four" matches both `cook` (weight 10) and `bake` (weight - // 25) at essentially the same span — only the more specific `bake` - // should survive, not both. - expect(matchTechSteps("Cuire au four pendant 30 minutes", [cook, bake])).to.deep.equal([3]); - // Order-independent. - expect(matchTechSteps("Cuire au four pendant 30 minutes", [bake, cook])).to.deep.equal([3]); - }); - - it("still keeps a non-overlapping technique alongside an overlap-resolved one", () => { - // `bake` wins over `cook` for "cuire au four" (overlap), but `melt` - // matches an entirely different, non-overlapping span and survives. - const result = matchTechSteps("Faire fondre le beurre, puis cuire au four", [ - cook, - bake, - melt, - ]); - expect(result).to.deep.equal([5, 3]); - }); - - it("breaks a same-span weight tie by lowest techStepId", () => { - const a: TechStepMappingRule = { techStepId: 5, expression: "\\bmelanger\\b", weight: 10 }; - const b: TechStepMappingRule = { techStepId: 2, expression: "\\bmelanger\\b", weight: 10 }; - expect(matchTechSteps("Mélanger les ingrédients", [a, b])).to.deep.equal([2]); - }); - - it("still resolves to one techStep when two of its own mappings both match", () => { - const wholeWord: TechStepMappingRule = { - techStepId: 7, - expression: "\\bmijoter\\b", - weight: 15, - }; - const withAdverb: TechStepMappingRule = { - techStepId: 7, - expression: "\\bmijoter à feu doux\\b", - weight: 15, - }; - expect(matchTechSteps("Faire mijoter à feu doux", [wholeWord, withAdverb])).to.deep.equal([ - 7, - ]); - }); - - it("respects word boundaries — a technique's verb embedded in a longer word doesn't false-positive", () => { - // "recuire"/"précuit" contain "cuire"/"cuit" as a substring, but not as - // a standalone word — the \b-anchored expression must not match them. - expect(matchTechSteps("Faire recuire la sauce", [cook])).to.deep.equal([]); - expect(matchTechSteps("Un plat précuit", [cook])).to.deep.equal([]); - // The standalone forms still match. - expect(matchTechSteps("Faire cuire la sauce", [cook])).to.deep.equal([2]); - expect(matchTechSteps("Le riz est cuit", [cook])).to.deep.equal([2]); - }); - }); - - describe("matchTechStepSpans", () => { - // Same fixtures as `matchTechSteps` above (kept local to this describe - // block rather than shared — each block's fixtures should be readable - // on their own). - const simmer: TechStepMappingRule = { - techStepId: 1, - expression: "\\bmijot(er|ez|e|ant|é)\\b", - weight: 15, - }; - const cook: TechStepMappingRule = { - techStepId: 2, - expression: "\\bcui(re|sez|sant|sson)\\b|\\bcuit(e|es|s)?\\b", - weight: 10, - }; - const bake: TechStepMappingRule = { - techStepId: 3, - expression: - "\\bcuire au four\\b|\\bcuisson au four\\b|\\benfourn(er|ez|é|ée|ées)\\b|\\bau four\\b", - weight: 25, - }; - const preheat: TechStepMappingRule = { - techStepId: 4, - expression: "\\bpr[ée]chauff(er|ez|é|ée)\\b", - weight: 20, - }; - const melt: TechStepMappingRule = { - techStepId: 5, - expression: "\\bfondre\\b|\\bfaire fondre\\b|\\bfaites fondre\\b", - weight: 15, - }; - - it("returns the matched span alongside the techStepId for a simple match", () => { - // "Faire mijoter à feu doux" — "mijoter" starts right after "Faire ". - expect(matchTechStepSpans("Faire mijoter à feu doux", [simmer])).to.deep.equal([ - { techStepId: 1, start: 6, end: 13 }, - ]); - }); - - it("returns an empty list when nothing matches", () => { - expect(matchTechStepSpans("Servir immédiatement", [simmer, cook, bake])).to.deep.equal([]); - }); - - it("returns each distinct technique's own span, in reading order", () => { + it("splits into two clauses at the whitespace nearest the gap's midpoint between two candidates", () => { + // "Préchauffer la poêle, puis faire fondre le beurre" + // 0 1 2 3 4 + // 0123456789012345678901234567890123456789012345678901 + const preheat = candidate("preheat", 0, 11); // "Préchauffer" + const melt = candidate("melt", 27, 39); // "faire fondre" const text = "Préchauffer la poêle, puis faire fondre le beurre"; - const result = matchTechStepSpans(text, [preheat, melt]); + + const result = splitIntoClauses(text, [preheat, melt]); + expect(result).to.have.length(2); - expect(result[0].techStepId).to.equal(4); - expect(result[1].techStepId).to.equal(5); - // Each span, sliced back out of the original text, is exactly the - // word(s) that triggered that match — what the frontend needs to - // highlight the right characters. - expect(text.slice(result[0].start, result[0].end).toLowerCase()).to.equal("préchauffer"); - expect(text.slice(result[1].start, result[1].end).toLowerCase()).to.equal("faire fondre"); + // The gap between the two candidates is [11, 27) — its raw midpoint + // (19) falls inside "poêle" (see findGapSplitPoint's doc comment for + // why that's specifically what this snaps away from); the nearest + // actual whitespace to that midpoint is the space at 21, right after + // the comma. + expect(result[0]).to.deep.equal({ start: 0, end: 21, anchor: preheat }); + expect(result[1]).to.deep.equal({ start: 21, end: text.length, anchor: melt }); + // The two clauses are contiguous and cover the whole text. + expect( + text.slice(result[0].start, result[0].end) + text.slice(result[1].start, result[1].end), + ).to.equal(text); }); - it("keeps only the winning span when two techniques' expressions overlap", () => { - // `bake` (weight 25) wins over `cook` (weight 10) for "cuire au four" - // — only bake's span survives, not two overlapping entries. - const text = "Cuire au four pendant 30 minutes"; - const result = matchTechStepSpans(text, [cook, bake]); - expect(result).to.deep.equal([{ techStepId: 3, start: 0, end: 13 }]); - expect(text.slice(0, 13)).to.equal("Cuire au four"); + it("sorts out-of-order candidates before splitting, and anchors each clause on the matching one", () => { + const preheat = candidate("preheat", 0, 11); + const melt = candidate("melt", 27, 39); + // Passed in reverse — the function must still produce clauses in + // reading order, each anchored on the right candidate. + const result = splitIntoClauses("Préchauffer la poêle, puis faire fondre le beurre", [ + melt, + preheat, + ]); + expect(result.map((clause) => clause.anchor?.uid)).to.deep.equal(["preheat", "melt"]); + }); + + it("produces N contiguous clauses for N candidates, each anchored on its own", () => { + const a = candidate("a", 0, 3); + const b = candidate("b", 10, 13); + const c = candidate("c", 20, 23); + const text = "x".repeat(30); + + const result = splitIntoClauses(text, [a, b, c]); + + expect(result).to.have.length(3); + expect(result.map((clause) => clause.anchor?.uid)).to.deep.equal(["a", "b", "c"]); + // Contiguous: each clause's end is the next one's start. + expect(result[0].start).to.equal(0); + expect(result[0].end).to.equal(result[1].start); + expect(result[1].end).to.equal(result[2].start); + expect(result[2].end).to.equal(text.length); + }); + + it("clamps the split point to the earlier candidate's own end when two candidates are adjacent/overlapping", () => { + // Gap midpoint would fall *before* `a`'s own end here — must not + // produce a clause that cuts into `a`'s own anchor span. + const a = candidate("a", 0, 10); + const b = candidate("b", 8, 15); + + const result = splitIntoClauses("x".repeat(20), [a, b]); + + expect(result[0].end).to.be.at.least(a.end); + expect(result[1].start).to.equal(result[0].end); }); }); - describe("loadTechStepMappingRules", () => { + describe("techStepClassifier", () => { + // `techStepClassifier` is the one shared singleton (see + // tech-step-matcher.ts's own doc comment on why) — these tests + // exercise it against the real training corpus + // (`tech-step-training-data.ts`) and the real seeded `TechStep` + // catalog, rather than synthetic injectable fixtures the old + // regex-based `matchTechStepSpans(description, mappings)` allowed. + // Training + node-nlp's own one-time per-language setup can take a + // few seconds on the very first call in the whole suite (subsequent + // calls reuse the same trained model and are fast) — comfortably + // inside this suite's default 10s timeout (.mocharc.json). + let simmerId: number; + let cookId: number; + let bakeId: number; + let preheatId: number; + let meltId: number; + let boilId: number; + let chopId: number; + beforeEach(async () => { await resetDatabase(); + const [simmer, cook, bake, preheat, melt, boil, chop] = await Promise.all([ + prisma.techStep.findFirstOrThrow({ where: { key: "simmer" } }), + prisma.techStep.findFirstOrThrow({ where: { key: "cook" } }), + prisma.techStep.findFirstOrThrow({ where: { key: "bake" } }), + prisma.techStep.findFirstOrThrow({ where: { key: "preheat" } }), + prisma.techStep.findFirstOrThrow({ where: { key: "melt" } }), + prisma.techStep.findFirstOrThrow({ where: { key: "boil" } }), + prisma.techStep.findFirstOrThrow({ where: { key: "chop" } }), + ]); + simmerId = simmer.id; + cookId = cook.id; + bakeId = bake.id; + preheatId = preheat.id; + meltId = melt.id; + boilId = boil.id; + chopId = chop.id; }); after(async () => { await prisma.$disconnect(); }); - it("only returns mappings for the requested locale", async () => { - const simmer = await prisma.techStep.findFirstOrThrow({ where: { key: "simmer" } }); - // "de" has no seeded mappings at all (unlike "fr"/"en", which the - // real catalog now both populate) — a clean locale to attach one - // synthetic row to without conflating it with real seed data. - await prisma.techStepMapping.create({ - data: { techStepId: simmer.id, locale: "de", expression: "\\bsimmer\\b", weight: 15 }, + describe("matchTechSteps", () => { + it("matches an exact expression", async () => { + expect( + await techStepClassifier.matchTechSteps("Faire mijoter à feu doux", "fr"), + ).to.deep.equal([simmerId]); }); - // The seeded catalog (26 "fr" mappings) must be untouched by the extra - // "de" row — same count, and none of them carry its expression. - const frRules = await loadTechStepMappingRules("fr"); - expect(frRules).to.have.length(26); - expect(frRules.map((rule) => rule.expression)).to.not.include("\\bsimmer\\b"); + it("is case- and accent-insensitive", async () => { + expect(await techStepClassifier.matchTechSteps("FAIRE MIJOTER", "fr")).to.deep.equal([ + simmerId, + ]); + }); - const deRules = await loadTechStepMappingRules("de"); - expect(deRules).to.deep.equal([ - { techStepId: simmer.id, expression: "\\bsimmer\\b", weight: 15 }, - ]); + it("returns an empty sequence when nothing matches", async () => { + expect( + await techStepClassifier.matchTechSteps("Ranger les couverts dans le tiroir", "fr"), + ).to.deep.equal([]); + }); + + it("returns an empty sequence for an empty description", async () => { + expect(await techStepClassifier.matchTechSteps("", "fr")).to.deep.equal([]); + }); + + it("returns an empty sequence for a locale nothing was trained on", async () => { + expect( + await techStepClassifier.matchTechSteps("Faire mijoter à feu doux", "de"), + ).to.deep.equal([]); + }); + + it("detects several distinct techniques in one step, in reading order", async () => { + expect( + await techStepClassifier.matchTechSteps( + "Préchauffer la poêle, puis faire fondre le beurre", + "fr", + ), + ).to.deep.equal([preheatId, meltId]); + }); + + it("reverses the sequence when the techniques are mentioned in the opposite order", async () => { + expect( + await techStepClassifier.matchTechSteps( + "Faire fondre le beurre puis préchauffer le four", + "fr", + ), + ).to.deep.equal([meltId, preheatId]); + }); + + it("still matches the generic technique on its own when the more specific one isn't implied", async () => { + expect( + await techStepClassifier.matchTechSteps("Faire cuire à feu moyen", "fr"), + ).to.deep.equal([cookId]); + }); + + it("resolves the more specific technique when a generic one's own vocabulary is embedded in it", async () => { + // "Cuire au four" literally contains "cuire" (the generic `cook` + // verb) but means the more specific `bake` — the classifier (not + // a weight table) is what has to get this right now. + expect( + await techStepClassifier.matchTechSteps("Cuire au four pendant 30 minutes", "fr"), + ).to.deep.equal([bakeId]); + }); + + it("understands a technique described without ever naming it — the whole point of moving off pure keyword matching", async () => { + // No literal "fondre"/"fondu" anywhere in this sentence, yet it + // unambiguously means `melt` — this is the exact motivating case + // (see this module's own doc comment) a regex could never catch. + expect( + await techStepClassifier.matchTechSteps( + "jusqu'à ce que le beurre ait disparu dans la poêle", + "fr", + ), + ).to.deep.equal([meltId]); + }); + + it("understands preheating described without the verb 'préchauffer'", async () => { + expect( + await techStepClassifier.matchTechSteps("mettre la poêle sur feu vif", "fr"), + ).to.deep.equal([preheatId]); + }); + + it("matches English text against the English-trained vocabulary", async () => { + expect( + await techStepClassifier.matchTechSteps( + "Bring a large saucepan of salted water to the boil", + "en", + ), + ).to.deep.equal([boilId]); + }); }); - it("returns an empty list for a locale with no mappings at all", async () => { - expect(await loadTechStepMappingRules("de")).to.deep.equal([]); + describe("matchTechStepSpans", () => { + it("returns a tight keyword span, and a wider context span that's the whole description when there's only one candidate", async () => { + const text = "Faire mijoter à feu doux"; + const result = await techStepClassifier.matchTechStepSpans(text, "fr"); + expect(result).to.deep.equal([ + { techStepId: simmerId, start: 6, end: 13, contextStart: 0, contextEnd: text.length }, + ]); + expect(text.slice(6, 13).toLowerCase()).to.equal("mijoter"); + }); + + it("returns an empty list when nothing matches", async () => { + expect( + await techStepClassifier.matchTechStepSpans("Ranger les couverts dans le tiroir", "fr"), + ).to.deep.equal([]); + }); + + it("returns each distinct technique's own tight keyword span and its own wider context span, in reading order", async () => { + const text = "Préchauffer la poêle, puis faire fondre le beurre"; + const result = await techStepClassifier.matchTechStepSpans(text, "fr"); + + expect(result).to.have.length(2); + expect(result[0].techStepId).to.equal(preheatId); + expect(result[1].techStepId).to.equal(meltId); + // Each keyword span, sliced back out of the original text, is + // exactly the word(s) that anchored that match — what the frontend + // needs to highlight the exact right characters. + expect(text.slice(result[0].start, result[0].end).toLowerCase()).to.equal("préchauffer"); + expect(text.slice(result[1].start, result[1].end).toLowerCase()).to.equal("faire fondre"); + // Each context span is the wider clause the keyword was found in — + // the two are contiguous and cover the whole description between + // them (see splitIntoClauses, which computed these). + expect(text.slice(result[0].contextStart, result[0].contextEnd)).to.equal( + "Préchauffer la poêle,", + ); + expect(text.slice(result[1].contextStart, result[1].contextEnd)).to.equal( + " puis faire fondre le beurre", + ); + expect(result[0].contextEnd).to.equal(result[1].contextStart); + }); + + it("understands both techniques in the classic 'Dans une poêle chaude, faire chauffer une noix de beurre' example, each with its own keyword and context", async () => { + // The motivating example for context spans in the first place: + // `preheat`'s keyword is a noun phrase ("poêle chaude"), not a + // verb — its context ("Dans une poêle chaude") is what actually + // shows this is about preparing the pan, not (say) deglazing one. + const text = "Dans une poêle chaude, faire chauffer une noix de beurre"; + const result = await techStepClassifier.matchTechStepSpans(text, "fr"); + + expect(result).to.have.length(2); + expect(result[0]).to.deep.equal({ + techStepId: preheatId, + start: 9, + end: 21, + contextStart: 0, + contextEnd: 22, + }); + expect(result[1]).to.deep.equal({ + techStepId: meltId, + start: 23, + end: 37, + contextStart: 22, + contextEnd: text.length, + }); + expect(text.slice(result[0].start, result[0].end)).to.equal("poêle chaude"); + expect(text.slice(result[0].contextStart, result[0].contextEnd)).to.equal( + "Dans une poêle chaude,", + ); + expect(text.slice(result[1].start, result[1].end)).to.equal("faire chauffer"); + expect(text.slice(result[1].contextStart, result[1].contextEnd)).to.equal( + " faire chauffer une noix de beurre", + ); + }); + + it("falls back to highlighting the whole clause for both spans when a technique was found with no literal anchor word", async () => { + const text = "jusqu'à ce que le beurre ait disparu dans la poêle"; + const result = await techStepClassifier.matchTechStepSpans(text, "fr"); + expect(result).to.deep.equal([ + { + techStepId: meltId, + start: 0, + end: text.length, + contextStart: 0, + contextEnd: text.length, + }, + ]); + }); + + it("chop matches English text against the English-trained vocabulary, tight keyword span", async () => { + const text = "Chop the onions finely"; + const result = await techStepClassifier.matchTechStepSpans(text, "en"); + expect(result).to.deep.equal([ + { techStepId: chopId, start: 0, end: 4, contextStart: 0, contextEnd: text.length }, + ]); + expect(text.slice(0, 4)).to.equal("Chop"); + }); }); }); }); diff --git a/apps/api/test/reference.test.ts b/apps/api/test/reference.test.ts index 05de918..a2a1427 100644 --- a/apps/api/test/reference.test.ts +++ b/apps/api/test/reference.test.ts @@ -149,15 +149,13 @@ describe("Reference data", () => { expect(keys).to.deep.equal([...keys].sort()); }); - it("reseeding is idempotent — no duplicate techniques or mappings", async () => { + it("reseeding is idempotent — no duplicate techniques", async () => { // resetDatabase already seeded once in beforeEach; seed a second time // on top of that without truncating, the way a redeploy would. await seedReferenceData(prisma); const res = await request(app).get("/reference/tech-steps"); expect(res.body).to.have.length(26); - // 26 techniques × one "fr" + one "en" mapping each. - expect(await prisma.techStepMapping.count()).to.equal(52); }); }); diff --git a/apps/web/cypress/component/highlight-tech-steps.cy.tsx b/apps/web/cypress/component/highlight-tech-steps.cy.tsx index b859738..a581538 100644 --- a/apps/web/cypress/component/highlight-tech-steps.cy.tsx +++ b/apps/web/cypress/component/highlight-tech-steps.cy.tsx @@ -6,42 +6,56 @@ import { splitDescriptionByTechSteps } from "../../src/features/recipes/steps/hi // `expect`, not for rendering. `.cy.tsx` (not `.cy.ts`) only because that's // what `cypress.config.ts`'s component `specPattern` looks for. -function techStep(key: string, id: number, start: number, end: number): StepTechStepView { - return { techStep: { id, key }, start, end }; +/** Builds a `StepTechStepView` — `context` omitted entirely (not just undefined) when absent, matching what the API actually sends for an older, not-yet-recomputed match (see `StepTechStepView`'s own doc comment). */ +function techStep( + key: string, + id: number, + start: number, + end: number, + context?: { start: number; end: number }, +): StepTechStepView { + return { + techStep: { id, key }, + start, + end, + ...(context ? { contextStart: context.start, contextEnd: context.end } : {}), + }; } describe("splitDescriptionByTechSteps", () => { it("returns the whole description as one plain segment when there are no matches", () => { expect(splitDescriptionByTechSteps("Servir immédiatement", [])).to.deep.equal([ - { text: "Servir immédiatement", techStep: null }, + { text: "Servir immédiatement", techStep: null, isKeyword: false }, ]); }); - it("splits a single match into before/match/after segments", () => { - // "Faire mijoter à feu doux" — "mijoter" is [6, 13). + it("splits a single keyword-only match (no context) into before/match/after segments", () => { + // "Faire mijoter à feu doux" — "mijoter" is [6, 13). Same shape as + // before context spans existed at all — the common case for a short, + // already-imperative clause where the keyword and its context coincide. const result = splitDescriptionByTechSteps("Faire mijoter à feu doux", [ techStep("simmer", 1, 6, 13), ]); expect(result).to.deep.equal([ - { text: "Faire ", techStep: null }, - { text: "mijoter", techStep: { id: 1, key: "simmer" } }, - { text: " à feu doux", techStep: null }, + { text: "Faire ", techStep: null, isKeyword: false }, + { text: "mijoter", techStep: { id: 1, key: "simmer" }, isKeyword: true }, + { text: " à feu doux", techStep: null, isKeyword: false }, ]); }); it("handles a match at the very start, with nothing before it", () => { const result = splitDescriptionByTechSteps("Hacher les oignons", [techStep("chop", 2, 0, 6)]); expect(result).to.deep.equal([ - { text: "Hacher", techStep: { id: 2, key: "chop" } }, - { text: " les oignons", techStep: null }, + { text: "Hacher", techStep: { id: 2, key: "chop" }, isKeyword: true }, + { text: " les oignons", techStep: null, isKeyword: false }, ]); }); it("handles a match at the very end, with nothing after it", () => { const result = splitDescriptionByTechSteps("Faire cuire", [techStep("cook", 3, 6, 11)]); expect(result).to.deep.equal([ - { text: "Faire ", techStep: null }, - { text: "cuire", techStep: { id: 3, key: "cook" } }, + { text: "Faire ", techStep: null, isKeyword: false }, + { text: "cuire", techStep: { id: 3, key: "cook" }, isKeyword: true }, ]); }); @@ -53,34 +67,38 @@ describe("splitDescriptionByTechSteps", () => { ]); expect(result.map((s) => s.text).join("")).to.equal(text); expect(result.filter((s) => s.techStep !== null)).to.have.length(2); - expect(result[0]).to.deep.equal({ text: "Préchauffer", techStep: { id: 4, key: "preheat" } }); + expect(result[0]).to.deep.equal({ + text: "Préchauffer", + techStep: { id: 4, key: "preheat" }, + isKeyword: true, + }); }); it("re-sorts entries that aren't already in start order", () => { const text = "Faire fondre le beurre puis préchauffer le four"; // Passed in techStepId order, not text order — the function must sort - // by `start`, not trust the input order. + // by position, not trust the input order. const result = splitDescriptionByTechSteps(text, [ techStep("preheat", 4, 28, 39), techStep("melt", 5, 0, 12), ]); - const matches = result.filter((s) => s.techStep !== null); + const matches = result.filter((s) => s.techStep !== null && s.isKeyword); expect(matches.map((s) => s.techStep?.key)).to.deep.equal(["melt", "preheat"]); }); it("drops a match whose end is past the end of the description", () => { const result = splitDescriptionByTechSteps("Cuire", [techStep("cook", 3, 0, 999)]); - expect(result).to.deep.equal([{ text: "Cuire", techStep: null }]); + expect(result).to.deep.equal([{ text: "Cuire", techStep: null, isKeyword: false }]); }); it("drops a match with a negative start", () => { const result = splitDescriptionByTechSteps("Cuire", [techStep("cook", 3, -1, 5)]); - expect(result).to.deep.equal([{ text: "Cuire", techStep: null }]); + expect(result).to.deep.equal([{ text: "Cuire", techStep: null, isKeyword: false }]); }); it("drops a match whose start isn't before its end", () => { const result = splitDescriptionByTechSteps("Cuire", [techStep("cook", 3, 3, 3)]); - expect(result).to.deep.equal([{ text: "Cuire", techStep: null }]); + expect(result).to.deep.equal([{ text: "Cuire", techStep: null, isKeyword: false }]); }); it("drops a later match that overlaps one already accepted", () => { @@ -91,10 +109,69 @@ describe("splitDescriptionByTechSteps", () => { techStep("bake", 3, 0, 13), techStep("cook", 2, 0, 5), ]); - expect(result).to.deep.equal([{ text: "Cuire au four", techStep: { id: 3, key: "bake" } }]); + expect(result).to.deep.equal([ + { text: "Cuire au four", techStep: { id: 3, key: "bake" }, isKeyword: true }, + ]); }); it("returns a single empty-ish segment for an empty description with no matches", () => { expect(splitDescriptionByTechSteps("", [])).to.deep.equal([]); }); + + describe("with a context span wider than the keyword", () => { + it("splits into context-before / keyword / context-after around a keyword in the middle of its clause", () => { + // The motivating example: "Dans une poêle chaude, faire chauffer une + // noix de beurre" — `preheat`'s keyword is "poêle chaude", its + // context is the whole "Dans une poêle chaude" clause around it. + const text = "Dans une poêle chaude, faire chauffer une noix de beurre"; + const result = splitDescriptionByTechSteps(text, [ + techStep("preheat", 4, 9, 21, { start: 0, end: 21 }), + ]); + expect(result).to.deep.equal([ + { text: "Dans une ", techStep: { id: 4, key: "preheat" }, isKeyword: false }, + { text: "poêle chaude", techStep: { id: 4, key: "preheat" }, isKeyword: true }, + { + text: ", faire chauffer une noix de beurre", + techStep: null, + isKeyword: false, + }, + ]); + }); + + it("omits the context-before segment when the keyword starts right at the context's own start", () => { + const result = splitDescriptionByTechSteps("préchauffer le four", [ + techStep("preheat", 4, 0, 11, { start: 0, end: 19 }), + ]); + expect(result).to.deep.equal([ + { text: "préchauffer", techStep: { id: 4, key: "preheat" }, isKeyword: true }, + { text: " le four", techStep: { id: 4, key: "preheat" }, isKeyword: false }, + ]); + }); + + it("omits the context-after segment when the keyword ends right at the context's own end", () => { + const result = splitDescriptionByTechSteps("mettre le four à préchauffer", [ + techStep("preheat", 4, 17, 28, { start: 7, end: 28 }), + ]); + expect(result).to.deep.equal([ + { text: "mettre ", techStep: null, isKeyword: false }, + { text: "le four à ", techStep: { id: 4, key: "preheat" }, isKeyword: false }, + { text: "préchauffer", techStep: { id: 4, key: "preheat" }, isKeyword: true }, + ]); + }); + + it("falls back to a keyword-only segment when context is absent (an older, not-yet-recomputed match)", () => { + const result = splitDescriptionByTechSteps("Faire mijoter à feu doux", [ + techStep("simmer", 1, 6, 13), + ]); + expect(result.some((s) => s.techStep !== null && !s.isKeyword)).to.equal(false); + }); + + it("drops an entry whose context doesn't actually contain its own keyword span", () => { + const result = splitDescriptionByTechSteps("Cuire au four", [ + // contextEnd (5) is before the keyword's own end (13) — malformed. + techStep("bake", 3, 0, 13, { start: 0, end: 5 }), + ]); + expect(result).to.deep.equal([{ text: "Cuire au four", techStep: null, isKeyword: false }]); + }); + }); }); diff --git a/apps/web/src/features/recipes/recipes.scss b/apps/web/src/features/recipes/recipes.scss index 26d4e96..ed923a0 100644 --- a/apps/web/src/features/recipes/recipes.scss +++ b/apps/web/src/features/recipes/recipes.scss @@ -627,6 +627,12 @@ } } +// `.step-tech-step-context` (the wider clause a `.step-tech-step` keyword +// was found in) used to be highlighted here too, more subtly — turned back +// off (see `StepDescription.tsx`'s doc comment): the backend still +// computes and persists `contextStart`/`contextEnd`, this file just no +// longer gives that class any styling to render with. + // --- Favorite star toggle (detail panel header) ----------------------------- .favorite-star-button { position: absolute; diff --git a/apps/web/src/features/recipes/steps/StepDescription.tsx b/apps/web/src/features/recipes/steps/StepDescription.tsx index cbbd317..7e5641e 100644 --- a/apps/web/src/features/recipes/steps/StepDescription.tsx +++ b/apps/web/src/features/recipes/steps/StepDescription.tsx @@ -10,6 +10,17 @@ import { splitDescriptionByTechSteps } from "./highlight-tech-steps"; * technique (e.g. hovering/focusing "hacher" in "Hacher les oignons" shows * "Hacher") — `RecipeDetailPanel`'s replacement for a bare `

{description}

`. * + * A match's wider `contextStart`/`contextEnd` clause (see + * `StepTechStepView`) is deliberately *not* visualized here — only the + * tight keyword span is highlighted. The backend still computes and + * persists it (`tech-step-matcher.ts`/`StepTechStep`), and + * `splitDescriptionByTechSteps` still splits the description around it + * (`isKeyword: false` context segments), but this component now renders + * those non-keyword segments as plain text, same as a segment with no + * technique at all — the visual "wider clause, subtler highlight" + * treatment (`.step-tech-step-context`) turned out to be more visual noise + * than useful signal in practice and was turned back off. + * * `techStep.key` resolves its tooltip label through `catalog.techSteps.` * i18n, the same pattern every other reference catalog (diets, units, …) * uses for its display text. @@ -34,6 +45,13 @@ export function StepDescription({ // spliced in place), so using it as part of the key is safe here. const key = `${index}-${segment.text}`; if (!segment.techStep) return {segment.text}; + + if (!segment.isKeyword) { + // Context-only run — rendered as plain text, same as a segment + // with no technique at all (see this component's doc comment for + // why the wider-clause highlight was turned back off). + return {segment.text}; + } return ( {/* A real