diff --git a/.gitignore b/.gitignore index 6d92b7c..93666d2 100644 --- a/.gitignore +++ b/.gitignore @@ -71,6 +71,12 @@ web_modules/ !.env.example !.env.test.example +# node-nlp's default auto-save file (apps/api/src/lib/recipe-matching/ +# tech-step-matcher.ts explicitly disables autoSave/autoLoad, but this is a +# belt-and-suspenders guard against it ever reappearing — a stale trained +# model on disk must never silently shadow TECH_STEP_TRAINING_DATA). +model.nlp + # parcel-bundler cache (https://parceljs.org/) .cache .parcel-cache diff --git a/apps/api/package.json b/apps/api/package.json index 8ccf511..1147e02 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -26,6 +26,7 @@ "dotenv": "^16.4.5", "express": "^4.21.1", "jsonwebtoken": "^9.0.3", + "node-nlp": "4.27.0", "prisma": "^5.22.0", "zod": "^3.23.8" }, diff --git a/apps/api/prisma/migrations/20260821130000_drop_tech_step_mapping/migration.sql b/apps/api/prisma/migrations/20260821130000_drop_tech_step_mapping/migration.sql new file mode 100644 index 0000000..3e3f219 --- /dev/null +++ b/apps/api/prisma/migrations/20260821130000_drop_tech_step_mapping/migration.sql @@ -0,0 +1,6 @@ +-- DropForeignKey +ALTER TABLE "tech_step_mapping" DROP CONSTRAINT "tech_step_mapping_tech_step_id_fkey"; + +-- DropTable +DROP TABLE "tech_step_mapping"; + diff --git a/apps/api/prisma/schema.prisma b/apps/api/prisma/schema.prisma index 9cc9ad8..318dc1a 100644 --- a/apps/api/prisma/schema.prisma +++ b/apps/api/prisma/schema.prisma @@ -619,36 +619,26 @@ model RecipeIngredient { /// camelCase uid (e.g. `"simmer"`), not the display label — the French /// label lives in `apps/web`'s `locales/fr/translation.json` under /// `catalog.techSteps.` (see `reference-seed-data.ts`'s `TECH_STEPS`). +/// +/// Matching a step's free text against these (`tech-step-matcher.ts`'s +/// `TechStepClassifierService`) used to go through a DB-backed +/// `TechStepMapping` table of per-locale regex expressions — replaced with +/// a node-nlp model trained from in-code data +/// (`tech-step-training-data.ts`) once regexes turned out unable to +/// generalize past their own literal vocabulary. Nothing queries/edits +/// that matching data at runtime anymore (it only ever feeds the +/// classifier's one-time training pass), so it no longer needs a table of +/// its own — this row now only exists to be a stable id/key other tables +/// (`StepTechStep`) reference. model TechStep { id Int @id @default(autoincrement()) key String @unique - steps StepTechStep[] - mappings TechStepMapping[] + steps StepTechStep[] @@map("tech_step") } -/// Used by `tech-step-matcher.ts` to auto-detect which technique a recipe -/// step's description corresponds to (expression = regex pattern tested -/// against the description, weight = tie-break score when several -/// mappings match, or overlap-resolution score when two mappings match the -/// same span of text — see `matchTechSteps`). `locale` (e.g. `"fr"`) lets -/// the same TechStep carry one matching rule set per language — the -/// matcher is always called with a target locale and only considers -/// mappings for that locale. -model TechStepMapping { - id Int @id @default(autoincrement()) - techStepId Int @map("tech_step_id") - locale String - expression String - weight Int - - techStep TechStep @relation(fields: [techStepId], references: [id], onDelete: Cascade) - - @@map("tech_step_mapping") -} - /// Modeled as one-to-many (a step belongs to exactly one recipe), not the /// many-to-many noted in the spec doc: `order` only makes sense scoped to a /// single recipe, which isn't reconcilable with steps being shared across diff --git a/apps/api/src/db/prisma.ts b/apps/api/src/db/prisma.ts index f54b78c..7792956 100644 --- a/apps/api/src/db/prisma.ts +++ b/apps/api/src/db/prisma.ts @@ -1,3 +1,22 @@ +// Imported for its side effect only (loading `.env`/`.env.test` via +// dotenv) — must run *before* `new PrismaClient()` below. The generated +// Prisma Client bakes in its own fallback `.env` path (always +// `apps/api/.env`, the dev one — resolved once at `prisma generate` time) +// and loads it internally the first time a `PrismaClient` is constructed, +// unless `DATABASE_URL` is already set in `process.env` by then — dotenv +// never overrides an already-set variable, so whichever of these two env +// loads runs first "wins" for the rest of the process. Without this +// import, that race depended entirely on which test file some *other* +// module happened to import first, which normally worked out only by +// coincidence (whatever file mocha's `test/**/*.test.ts` glob happens to +// resolve first) — running a single test file in isolation (e.g. `mocha +// test/some-file.test.ts` directly, bypassing that glob) could silently +// resolve `DATABASE_URL` to the real dev database instead of +// `.env.test`'s. `resetDatabase()`'s own `assertRunningAgainstTestDatabase` +// guard (test-support/reset-db.ts) is what actually caught this in +// practice — it throws rather than truncating the wrong database — but +// the fix belongs here, at the source, not just at that one call site. +import "../config/env.js"; import { PrismaClient } from "@prisma/client"; /** diff --git a/apps/api/src/db/reference-seed-data.ts b/apps/api/src/db/reference-seed-data.ts index cee9030..ac5cf75 100644 --- a/apps/api/src/db/reference-seed-data.ts +++ b/apps/api/src/db/reference-seed-data.ts @@ -51,245 +51,49 @@ export const UNITS: Array<{ uid: string; type: UnitType; toBaseFactor: number }> { uid: "pound", type: "MASS", toBaseFactor: 453.5924 }, ]; -// Cooking-technique catalog (French recipe-step normalization) — a static -// list of common instructions, each carrying one or more text-matching -// rules used by `tech-step-matcher.ts` to auto-detect which technique(s) a -// free-text `Step.description` corresponds to (a step can mention several, -// e.g. "faire chauffer une poêle puis y faire fondre le beurre" is both -// `preheat` and `melt` — see `Step.techSteps`/`StepTechStep` in -// schema.prisma). Same "English camelCase uid, no French label" authoring -// as DIETS/UNITS — the label lives in apps/web's -// locales/fr/translation.json under `catalog.techSteps.`. -// `expression` is a regex source matched (case/accent-insensitive, via -// `normalizeText`) against the step description; `weight` breaks ties when -// two *different* techniques' expressions match the same span of text -// (highest weight wins) — see `tech-step-matcher.ts`'s `matchTechSteps`. -// Specific, multi-word phrases ("cuire au four", "faire revenir") are -// weighted higher than the generic single-verb forms they overlap with -// ("cuire", "sauter") so the more specific technique wins when both match -// the same words. `locale` lets the same technique carry one matching rule -// set per language — `"fr"` and `"en"` today (the latter mainly for -// English-language sources like TheMealDB), more can be added later -// without a schema change. The two locales are independent rule sets, not -// translations of each other — an English recipe is matched only against -// the `"en"` mappings, never a mix of both. -export const TECH_STEPS: Array<{ - uid: string; - mappings: Array<{ locale: string; expression: string; weight: number }>; -}> = [ - { - uid: "cook", - mappings: [ - { locale: "fr", expression: "\\bcui(re|sez|sant|sson)\\b|\\bcuit(e|es|s)?\\b", weight: 10 }, - { locale: "en", expression: "\\bcook(s|ed|ing)?\\b", weight: 10 }, - ], - }, - { - uid: "fry", - mappings: [ - { locale: "fr", expression: "\\bfri(re|t|te|ts|tes|ture)\\b", weight: 15 }, - { locale: "en", expression: "\\bfr(y|ies|ied|ying)\\b", weight: 15 }, - ], - }, - { - uid: "melt", - mappings: [ - { - locale: "fr", - expression: - "\\bfondre\\b|\\bfondu(e|es|s)?\\b|\\bfaire fondre\\b|\\bfaites fondre\\b|\\bfaire chauffer\\b|\\bfaites chauffer\\b", - weight: 15, - }, - { locale: "en", expression: "\\bmelt(s|ed|ing)?\\b", weight: 15 }, - ], - }, - { - uid: "deglaze", - mappings: [ - { locale: "fr", expression: "\\bd[ée]glac(er|ez|é|ée|age)\\b", weight: 20 }, - { locale: "en", expression: "\\bdeglaz(e|es|ed|ing)\\b", weight: 20 }, - ], - }, - { - uid: "simmer", - mappings: [ - { locale: "fr", expression: "\\bmijot(er|ez|e|ant|é)\\b", weight: 15 }, - { locale: "en", expression: "\\bsimmer(s|ed|ing)?\\b", weight: 15 }, - ], - }, - { - uid: "boil", - mappings: [ - { locale: "fr", expression: "\\bbouill(ir|ant|ie|ies)\\b|\\b[ée]bullition\\b", weight: 12 }, - { locale: "en", expression: "\\bboil(s|ed|ing)?\\b", weight: 12 }, - ], - }, - { - uid: "roast", - mappings: [ - { locale: "fr", expression: "\\br[ôo]tir\\b|\\br[ôo]ti(e|es|s)?\\b", weight: 15 }, - { locale: "en", expression: "\\broast(s|ed|ing)?\\b", weight: 15 }, - ], - }, - { - uid: "grill", - mappings: [ - { locale: "fr", expression: "\\bgrill(er|ez|é|ée|ées|ade)\\b", weight: 15 }, - { locale: "en", expression: "\\bgrill(s|ed|ing)?\\b", weight: 15 }, - ], - }, - { - uid: "panFry", - mappings: [ - { locale: "fr", expression: "\\bsaut(er|ez|é|ée|ées|ant)\\b", weight: 12 }, - { - locale: "en", - expression: "\\bsaut[ée](s|ed|ing)?\\b|\\bpan[- ]?fr(y|ies|ied|ying)\\b", - weight: 12, - }, - ], - }, - { - uid: "blanch", - mappings: [ - { locale: "fr", expression: "\\bblanch(ir|issez|i|ie|ies|iment)\\b", weight: 18 }, - { locale: "en", expression: "\\bblanch(es|ed|ing)?\\b", weight: 18 }, - ], - }, - { - uid: "marinate", - mappings: [ - { locale: "fr", expression: "\\bmarin(er|ez|é|ée|ées|ade)\\b", weight: 18 }, - { locale: "en", expression: "\\bmarinat(e|es|ed|ing)\\b|\\bmarinad(e|es)\\b", weight: 18 }, - ], - }, - { - uid: "chop", - mappings: [ - { locale: "fr", expression: "\\bhach(er|ez|é|ée|ées|is)\\b", weight: 15 }, - { locale: "en", expression: "\\bchop(s|ped|ping)?\\b", weight: 15 }, - ], - }, - { - uid: "peel", - mappings: [ - { locale: "fr", expression: "\\b[ée]pluch(er|ez|é|ée|ées|age)\\b", weight: 15 }, - { locale: "en", expression: "\\bpeel(s|ed|ing)?\\b", weight: 15 }, - ], - }, - { - uid: "mince", - mappings: [ - { locale: "fr", expression: "\\b[ée]minc(er|ez|é|ée|ées)\\b", weight: 18 }, - { locale: "en", expression: "\\bminc(e|es|ed|ing)\\b", weight: 18 }, - ], - }, - { - uid: "mix", - mappings: [ - { locale: "fr", expression: "\\bm[ée]lang(er|ez|é|ée|ées|e|es)\\b", weight: 10 }, - { locale: "en", expression: "\\bmix(es|ed|ing)?\\b|\\bcombine(s|d)?\\b", weight: 10 }, - ], - }, - { - uid: "whisk", - mappings: [ - { locale: "fr", expression: "\\bfouett(er|ez|é|ée|ées)\\b|\\bau fouet\\b", weight: 15 }, - { locale: "en", expression: "\\bwhisk(s|ed|ing)?\\b", weight: 15 }, - ], - }, - { - uid: "foldIn", - mappings: [ - { locale: "fr", expression: "\\bincorpor(er|ez|é|ée|ées|ant)\\b", weight: 15 }, - { locale: "en", expression: "\\bfold(s|ed|ing)? in\\b", weight: 15 }, - ], - }, - { - uid: "setAside", - mappings: [ - { locale: "fr", expression: "\\br[ée]serv(er|ez|é|ée|ées)\\b", weight: 15 }, - { locale: "en", expression: "\\bset(s)? aside\\b|\\bsetting aside\\b", weight: 15 }, - ], - }, - { - uid: "season", - mappings: [ - { locale: "fr", expression: "\\bassaisonn(er|ez|é|ée|ées|ement)\\b", weight: 15 }, - { locale: "en", expression: "\\bseason(s|ed|ing)?\\b", weight: 15 }, - ], - }, - { - uid: "drain", - mappings: [ - { locale: "fr", expression: "\\b[ée]goutt(er|ez|é|ée|ées)\\b", weight: 15 }, - { locale: "en", expression: "\\bdrain(s|ed|ing)?\\b", weight: 15 }, - ], - }, - { - uid: "brown", - mappings: [ - { - locale: "fr", - expression: - "\\bfaire revenir\\b|\\bfaites revenir\\b|\\bfais revenir\\b|\\bfaire dorer\\b|\\bfaites dorer\\b", - weight: 25, - }, - // Verb forms only (not bare "brown"), which would false-positive on - // ingredient descriptions like "brown sugar"/"brown rice". - { locale: "en", expression: "\\bbrown(ed|ing)\\b", weight: 25 }, - ], - }, - { - uid: "rest", - mappings: [ - { locale: "fr", expression: "\\blaiss(er|ez|e) reposer\\b|\\breposer\\b", weight: 20 }, - // Anchored to "let ... rest"/"rest for" rather than bare "rest", - // which would false-positive on phrases like "the rest of the". - { - locale: "en", - expression: "\\blet (it |them )?rest\\b|\\brest(s|ed|ing)? for\\b", - weight: 20, - }, - ], - }, - { - uid: "preheat", - mappings: [ - { locale: "fr", expression: "\\bpr[ée]chauff(er|ez|é|ée)\\b", weight: 20 }, - { locale: "en", expression: "\\bpreheat(s|ed|ing)?\\b", weight: 20 }, - ], - }, - { - uid: "bake", - mappings: [ - { - locale: "fr", - expression: - "\\bcuire au four\\b|\\bcuisson au four\\b|\\benfourn(er|ez|é|ée|ées)\\b|\\bau four\\b", - weight: 25, - }, - { - locale: "en", - expression: "\\bbak(e|es|ed|ing)\\b|\\bin (a|the) (preheated )?oven\\b", - weight: 25, - }, - ], - }, - { - uid: "plate", - mappings: [ - { locale: "fr", expression: "\\bdress(er|ez|age)\\b", weight: 15 }, - { locale: "en", expression: "\\bplat(e|es|ed|ing)\\b", weight: 15 }, - ], - }, - { - uid: "coat", - mappings: [ - { locale: "fr", expression: "\\bnapp(er|ez|é|ée|ées|age)\\b", weight: 15 }, - { locale: "en", expression: "\\bcoat(s|ed|ing)?\\b", weight: 15 }, - ], - }, +// Cooking-technique catalog (French recipe-step normalization) — the +// stable `key`s `tech-step-matcher.ts` auto-detects in a free-text +// `Step.description` (a step can mention several, e.g. "faire chauffer une +// poêle puis y faire fondre le beurre" is both `preheat` and `melt` — see +// `Step.techSteps`/`StepTechStep` in schema.prisma). Same "English +// camelCase uid, no French label" authoring as DIETS/UNITS — the label +// lives in apps/web's locales/fr/translation.json under +// `catalog.techSteps.`. +// +// Just a flat list of stable ids here — the actual matching data (per- +// locale synonym lists + example phrasings the classifier trains on) lives +// in `lib/recipe-matching/tech-step-training-data.ts`'s +// `TECH_STEP_TRAINING_DATA`, not here: unlike this list, it's read by +// `TechStepClassifierService`'s training pass, not the seed script, so it +// doesn't belong alongside the rest of this file's DB-seeded reference +// data. Every entry here must have a matching entry there. +export const TECH_STEPS: string[] = [ + "cook", + "fry", + "melt", + "deglaze", + "simmer", + "boil", + "roast", + "grill", + "panFry", + "blanch", + "marinate", + "chop", + "peel", + "mince", + "mix", + "whisk", + "foldIn", + "setAside", + "season", + "drain", + "brown", + "rest", + "preheat", + "bake", + "plate", + "coat", ]; // The 14 allergens EU Regulation 1169/2011 (Annex II) requires food @@ -1389,37 +1193,11 @@ export async function seedReferenceData(prisma: PrismaClient): Promise { } // TechStep: upsert by key (same idempotent-seed reasoning as everything - // above), then fully replace its mappings on every reseed. Mappings carry - // no natural per-row identity to upsert against, and expressions/weights - // are expected to be tuned over time — a straight "delete all, recreate - // from source" keeps the table an exact mirror of `TECH_STEPS` rather - // than accumulating stale/duplicate rows from earlier edits. Nothing else - // references `TechStepMapping.id` (`Step` only points at `TechStep`, not - // at a specific mapping), so this replace is safe. - for (const { uid: key } of TECH_STEPS) { + // above) — just the stable id/key rows themselves now, no matching data + // to replace alongside them (see `TECH_STEPS`' own comment for why). + for (const key of TECH_STEPS) { await prisma.techStep.upsert({ where: { key }, update: {}, create: { key } }); } - const techSteps = await prisma.techStep.findMany({ - where: { key: { in: TECH_STEPS.map((t) => t.uid) } }, - }); - const techStepIdByKey = new Map(techSteps.map((t) => [t.key, t.id])); - - await prisma.techStepMapping.deleteMany({ - where: { techStepId: { in: [...techStepIdByKey.values()] } }, - }); - const techStepMappingRows = TECH_STEPS.flatMap(({ uid, mappings }) => { - const techStepId = techStepIdByKey.get(uid); - if (techStepId === undefined) return []; - return mappings.map(({ locale, expression, weight }) => ({ - techStepId, - locale, - expression, - weight, - })); - }); - if (techStepMappingRows.length > 0) { - await prisma.techStepMapping.createMany({ data: techStepMappingRows }); - } // `Allergy` itself carries no `key` — it's the selectable instance of a // keyed `Category` (see schema.prisma) — so seeding an allergen means one diff --git a/apps/api/src/lib/recipe-matching/ingredient-matcher.ts b/apps/api/src/lib/recipe-matching/ingredient-matcher.ts index 61e7327..cc47781 100644 --- a/apps/api/src/lib/recipe-matching/ingredient-matcher.ts +++ b/apps/api/src/lib/recipe-matching/ingredient-matcher.ts @@ -31,8 +31,9 @@ import { normalizeText } from "./tech-step-matcher.js"; * unit-testable without a database (see `test/ingredient-matcher.test.ts`); * `loadIngredientCatalog`/`loadUnitCatalog` are the only DB-touching pieces, * meant to be fetched once per request and reused across every ingredient - * line, the same "don't requery per item" convention as - * `loadTechStepMappingRules`. + * line, the same "don't requery per item" convention + * `tech-step-matcher.ts`'s `TechStepClassifierService` follows for its own + * one-time training pass. */ /** One `Ingredient` row trimmed to what {@link matchIngredientName} needs, alongside its English matching label. */ diff --git a/apps/api/src/lib/recipe-matching/recipe-translation.ts b/apps/api/src/lib/recipe-matching/recipe-translation.ts index adcf398..dd328b8 100644 --- a/apps/api/src/lib/recipe-matching/recipe-translation.ts +++ b/apps/api/src/lib/recipe-matching/recipe-translation.ts @@ -13,11 +13,7 @@ import { matchUnit, type UnitMatchEntry, } from "./ingredient-matcher.js"; -import { - loadTechStepMappingRules, - matchTechSteps, - type TechStepMappingRule, -} from "./tech-step-matcher.js"; +import { techStepClassifier } from "./tech-step-matcher.js"; /** * The "Traduction en étapes" stage of the import pipeline described in @@ -36,17 +32,18 @@ import { * can't know those) — this is one step of the pipeline, not the whole * thing. * - * `translateRecipeSteps`/`translateRecipeIngredients` are pure (take their - * matching data as plain arguments, same convention as `matchTechSteps`/ - * `matchIngredientName` themselves) so they're unit-testable without a - * database; `translateRecipe` is the DB-backed convenience wrapper a caller - * reaches for in practice, mirroring `tech-step-matcher.ts`'s own - * pure/DB-touching split. + * `translateRecipeIngredients` stays pure (takes its matching data as plain + * arguments, same convention `matchIngredientName` itself has) so it's + * unit-testable without a database. `translateRecipeSteps` no longer is — + * technique detection now goes through `techStepClassifier`'s trained + * model (`tech-step-matcher.ts`), which needs an async call — but is still + * exported separately from `translateRecipe` for callers/tests that only + * care about step translation, not ingredients too. */ /** A {@link ParsedRecipeStep}, after tech-step detection — declares its technique sequence alongside the description/picture it already had. */ export interface TranslatedRecipeStep extends ParsedRecipeStep { - /** Ordered sequence of detected `TechStep` ids (see `matchTechSteps`) — empty if this step doesn't mention any known technique. */ + /** Ordered sequence of detected `TechStep` ids (see `TechStepClassifierService.matchTechSteps`) — empty if this step doesn't mention any known technique. */ techStepIds: number[]; } @@ -63,34 +60,43 @@ export interface TranslatedRecipe extends Omit ({ - ...ingredient, - ingredientId: null, - unitId: null, - })), - steps: recipe.steps.map((step) => ({ - ...step, - techStepIds: matchTechSteps(step.description, techStepMappings), - })), - }; + locale: string, +): Promise { + try { + const steps = await Promise.all( + recipe.steps.map(async (step) => ({ + ...step, + techStepIds: await techStepClassifier.matchTechSteps(step.description, locale), + })), + ); + return { + ...recipe, + ingredients: recipe.ingredients.map((ingredient) => ({ + ...ingredient, + ingredientId: null, + unitId: null, + })), + steps, + }; + } catch (err) { + // Rethrown as-is — the caller (`sources.service.ts`) already + // handles/logs failures centrally; this function just isn't allowed a + // bare `await` per the repo's async/try-catch convention. + throw err; + } } /** @@ -259,12 +265,13 @@ export function mergeDuplicateIngredients( * manually-authored recipes. * * No user- or recipe-level language preference exists anywhere in the app - * yet (see `tech-step-matcher.ts`'s `loadTechStepMappingRules`) — callers + * yet (see `tech-step-matcher.ts`'s `TechStepClassifierService`) — callers * pass a locale explicitly rather than this module guessing one. Note that * an English-language source (e.g. TheMealDB) translated against `"fr"` - * mappings will currently get an empty `techStepIds` sequence on every - * step — matching-language mappings for that source's language don't exist - * yet, this stage doesn't invent them. + * will currently get an empty (or nonsensical) `techStepIds` sequence on + * every step — the classifier is trained per-locale, so calling it with a + * locale that doesn't match the actual text's language doesn't degrade + * gracefully, it just gets things wrong. * * Ingredient/unit matching only has English data today * (`INGREDIENT_LABELS_EN`/`UNIT_LABELS_EN`, `packages/shared`) — for any @@ -279,8 +286,7 @@ export async function translateRecipe( locale: string, ): Promise { try { - const techStepMappings = await loadTechStepMappingRules(locale); - const translated = translateRecipeSteps(recipe, techStepMappings); + const translated = await translateRecipeSteps(recipe, locale); if (locale !== "en") return translated; diff --git a/apps/api/src/lib/recipe-matching/tech-step-matcher.ts b/apps/api/src/lib/recipe-matching/tech-step-matcher.ts index b37dd2a..01d015b 100644 --- a/apps/api/src/lib/recipe-matching/tech-step-matcher.ts +++ b/apps/api/src/lib/recipe-matching/tech-step-matcher.ts @@ -1,46 +1,67 @@ +import { NlpManager } from "node-nlp"; import { prisma } from "../../db/prisma.js"; +import { TECH_STEP_TRAINING_DATA } from "./tech-step-training-data.js"; /** * Auto-detects which cooking techniques (`TechStep`) a free-text recipe - * step description corresponds to, using the static `TechStepMapping` - * catalog (see `reference-seed-data.ts`'s `TECH_STEPS`) — groundwork for a - * future batch-cooking optimization algorithm, and (via `matchTechStepSpans`) - * what `recipe.service.ts` persists as `StepTechStep.start`/`end` so the - * recipe UI can highlight the exact matched words (see `StepView` in + * step description corresponds to — groundwork for a future batch-cooking + * optimization algorithm, and (via `matchTechStepSpans`) what + * `recipe.service.ts` persists as `StepTechStep.start`/`end` so the recipe + * UI can highlight the exact matched words (see `StepView` in * `packages/shared`). * - * A single instruction can genuinely involve more than one technique (e.g. - * "Dans une poêle chaude, faire chauffer une noix de beurre" is both - * `preheat` and `melt`) — both `matchTechSteps`/`matchTechStepSpans` return - * the whole *ordered sequence* they find, not a single winner, matching + * Regex-only matching used to live here (matching literal verb-form + * patterns from a DB-backed `TechStepMapping` table) but couldn't + * generalize past its own vocabulary — a step describing melting butter as + * "jusqu'à ce que le beurre ait disparu dans la poêle" mentions no verb any + * regex could anchor on, yet unmistakably *means* `melt`. Replaced with a + * small hybrid pipeline built on `node-nlp` ({@link TechStepClassifierService}): + * + * 1. **NER** (node-nlp enum entities, `synonyms` in `TECH_STEP_TRAINING_DATA`) + * finds every *candidate* technique mention in the whole + * description, each with its exact character span — mechanically the + * same job the old regexes did, just as flat synonym lists instead of + * hand-written patterns (node-nlp's own stemmer/fuzzy matching already + * covers minor conjugation/typo variance the regexes had to enumerate + * by hand). This step alone is *not* the final answer — see step 3. + * 2. The description is cut into clauses around those candidate spans + * ({@link splitIntoClauses}) — a step naming two techniques ("Dans une + * poêle chaude, faire chauffer une noix de beurre" is both `preheat` + * and `melt`) needs each judged on its own surrounding context, not the + * whole step lumped into one classification. + * 3. **NLP intent classification** (node-nlp's `NlpManager`, trained on + * `TECH_STEP_TRAINING_DATA`'s `utterances`) then classifies each clause + * on its own — this is what actually delivers "meaning, not keywords": + * the classifier was deliberately trained on paraphrases that never use + * the technique's own verb (e.g. "jusqu'à ce que le beurre ait + * disparu" for `melt`), so a clause reaching it gets labeled by what it + * was trained to recognize as *meaning* a technique, not by which + * literal word the NER step happened to anchor on. The NER-implied + * technique is kept only as a fallback for a clause the classifier + * isn't confident about (see `CONFIDENCE_THRESHOLD`) — a clearly + * keyword-anchored clause a small model merely isn't sure how to + * classify shouldn't be dropped outright. + * + * A single instruction can genuinely involve more than one technique — see + * point 2 above — so `matchTechSteps`/`matchTechStepSpans` both return the + * whole *ordered sequence* they find, not a single winner, matching * `Step.techSteps` (schema.prisma's `StepTechStep`, an ordered join table). * - * `normalizeText`/`matchTechStepSpans`/`matchTechSteps` are pure (no DB - * access) so they can be unit-tested in isolation (see - * `test/tech-step-matcher.test.ts`). `loadTechStepMappingRules` is the only - * DB-touching piece, kept separate so callers (`recipe.service.ts`) fetch - * the whole mapping list once per request and pass it to - * `matchTechStepSpans` per step, rather than querying once per step. + * `normalizeText` and {@link splitIntoClauses} are pure (no DB/model + * access) so they stay unit-testable in isolation (see + * `test/tech-step-matcher.test.ts`); the classifier itself needs a one-time + * training pass (`_ensureTrained`, node-nlp's `NlpManager.train()`) plus a + * `TechStep.key -> id` lookup from the DB, both memoized on the shared + * {@link techStepClassifier} singleton rather than repeated per call — + * training is the expensive part (a few hundred ms for this corpus), never + * worth redoing per request let alone per step. */ -/** One `TechStepMapping` row, trimmed to what {@link matchTechSteps} needs. */ -export interface TechStepMappingRule { - techStepId: number; - /** - * Regex source, matched against the normalized description (see - * {@link normalizeText}) — may itself contain accented characters, - * normalized the same way before compiling. - */ - expression: string; - weight: number; -} - /** * Lowercases and strips diacritics (NFD decomposition + removal of - * combining marks, e.g. "Déglacer" -> "deglacer") — recipe step text and - * mapping expressions are both run through this before matching, so - * expressions can be authored with natural French accents in - * `reference-seed-data.ts` while matching stays accent/case-insensitive. + * combining marks, e.g. "Déglacer" -> "deglacer"). Still used by + * `ingredient-matcher.ts` for its own, unrelated free-text matching — kept + * here and exported rather than duplicated, this module owned it first. */ const COMBINING_DIACRITICS_PATTERN = /\p{Diacritic}/gu; @@ -48,17 +69,6 @@ export function normalizeText(text: string): string { return text.normalize("NFD").replace(COMBINING_DIACRITICS_PATTERN, "").toLowerCase(); } -/** Where in the (normalized) description one mapping matched, alongside the rule that matched — the raw material {@link matchTechStepSpans} resolves into a final sequence. */ -interface MatchCandidate extends TechStepMappingRule { - start: number; - end: number; -} - -/** Whether two candidates' matched spans share any character position — the case where two *different* techniques' expressions matched the same words (e.g. generic `cook`'s "cuire" inside specific `bake`'s "cuire au four"), meaning only one of them should survive. */ -function overlaps(a: MatchCandidate, b: MatchCandidate): boolean { - return a.start < b.end && b.start < a.end; -} - /** * One technique {@link matchTechStepSpans} found, alongside exactly where in * `description` it matched — `[start, end)`, same convention as @@ -72,131 +82,307 @@ export interface TechStepMatch { end: number; } -/** - * Detects every technique `description` mentions among `mappings`, as an - * ordered sequence of matches (each carrying *where* it matched) — empty if - * none match. The algorithm: - * - * 1. Test every mapping against the normalized description; each one that - * matches becomes a candidate carrying *where* it matched (so - * overlapping matches can be compared). - * 2. Within a single technique, several of its own mappings might all - * match (different phrasings for the same `techStepId`) — keep only - * that technique's best candidate (highest weight, ties broken by - * earliest match), the same tie-break this function always used for a - * single winner. - * 3. Across *different* techniques, two candidates can still overlap (a - * generic pattern matching inside a more specific one's span, e.g. - * `cook` vs `bake` both matching "cuire au four") — resolve greedily by - * weight: take candidates highest-weight first, accept a candidate only - * if it doesn't overlap one already accepted. This is what keeps - * `bake` and drops the redundant `cook` for that phrase, while letting - * two genuinely distinct, non-overlapping techniques (e.g. `preheat` - * and `melt` in "Dans une poêle chaude, faire chauffer une noix de - * beurre") both survive. - * 4. Sort what's left by where it appears in the text — the sequence - * reads in the same order as the instruction itself. - * - * The returned `start`/`end` are offsets into `normalizeText(description)`, - * used as-is against the *original* `description` by callers that slice it - * for display (`highlight-tech-steps.ts`, apps/web) — `normalizeText` only - * strips diacritics/lowercases, which preserves character count for - * realistic French text (canonical NFD decomposition never turns one - * character into more than one base character), so this holds in practice. - * A pathological input where it doesn't (e.g. a bare standalone `^`, which - * `normalizeText` would strip as a diacritic) just produces a slightly - * misplaced highlight — degrades silently, doesn't crash. - * - * Pure — takes `mappings` as a plain argument rather than querying Prisma - * itself, so it's testable without a database (see - * `loadTechStepMappingRules` for the DB-backed loader). `mappings` should - * already be filtered to the locale the caller cares about — this function - * has no notion of locale, it just tests the rules it's given. - */ -export function matchTechStepSpans( - description: string, - mappings: TechStepMappingRule[], -): TechStepMatch[] { - const normalizedDescription = normalizeText(description); +/** A candidate technique mention found by NER — the raw material {@link splitIntoClauses} cuts a description around. */ +export interface TechniqueCandidate { + /** Training-data `uid` this candidate's synonym belongs to (e.g. `"melt"`) — not yet resolved to a DB id at this stage. */ + uid: string; + start: number; + end: number; +} - const candidates: MatchCandidate[] = []; - for (const mapping of mappings) { - const pattern = new RegExp(normalizeText(mapping.expression), "i"); - const match = pattern.exec(normalizedDescription); - if (match === null) continue; - candidates.push({ - ...mapping, - start: match.index, - end: match.index + match[0].length, +/** One clause {@link splitIntoClauses} produced — `anchor` is `null` only for the single "whole description, no candidate found at all" fallback clause (see that function's doc comment). */ +export interface TechStepClause { + /** `[start, end)` into the original description — the text handed to the classifier for this clause. */ + start: number; + end: number; + /** The candidate this clause was cut around, if any — its own (tighter) span is what gets persisted for highlighting, the wider clause span is only ever classifier input. */ + anchor: TechniqueCandidate | null; +} + +/** + * Cuts `description` into clauses around `candidates` (NER's found + * technique mentions, already sorted or not — sorted internally), one + * clause per candidate, so each can be judged by the classifier on its own + * surrounding context rather than the whole (possibly multi-technique) + * description at once. + * + * - **Zero candidates**: the whole description is one clause with no + * anchor — still worth classifying (a description mentioning no literal + * keyword at all can still *mean* a technique, the entire point of the + * classification step), just with no tight span to highlight, so callers + * fall back to highlighting the whole thing. + * - **One candidate**: the whole description is one clause too (nothing to + * cut around a single mention), but *with* that candidate as its anchor + * — callers get its tight span for highlighting. + * - **Two or more**: split points fall halfway between each consecutive + * pair's `[end, nextStart]` gap, producing that many contiguous, + * non-overlapping clauses covering the whole description — clause *i* is + * anchored on candidate *i*. + * + * Pure and DB/model-free — unit-tested directly (see + * `test/tech-step-matcher.test.ts`) without needing a trained classifier. + */ +export function splitIntoClauses( + description: string, + candidates: TechniqueCandidate[], +): TechStepClause[] { + if (candidates.length === 0) { + return [{ start: 0, end: description.length, anchor: null }]; + } + + const sorted = [...candidates].sort((a, b) => a.start - b.start); + const [first, ...rest] = sorted; + if (first === undefined) { + // Unreachable — `candidates.length === 0` already returned above, so + // `sorted` (same length) always has a first element here. Satisfies + // `noUncheckedIndexedAccess`, which can't see that from the length + // check alone. + return [{ start: 0, end: description.length, anchor: null }]; + } + + // Single pass, pairing each candidate with the next one as it goes — + // avoids re-indexing a separately-built `splitPoints` array afterward + // (also awkward under `noUncheckedIndexedAccess` for no real benefit, + // since every split point is only ever read once, right after it's + // computed). + const clauses: TechStepClause[] = []; + let clauseStart = 0; + let anchor = first; + for (const next of rest) { + // Midpoint of the gap between this candidate's end and the next one's + // start — if they're adjacent/overlapping (gap <= 0), falls back to + // the boundary right at the next candidate's start, still non-overlapping. + const gapMidpoint = Math.floor((anchor.end + next.start) / 2); + const splitPoint = Math.max(gapMidpoint, anchor.end); + clauses.push({ start: clauseStart, end: splitPoint, anchor }); + clauseStart = splitPoint; + anchor = next; + } + clauses.push({ start: clauseStart, end: description.length, anchor }); + return clauses; +} + +/** Below this confidence, a clause's classifier verdict isn't trusted on its own — falls back to its NER anchor's own technique instead (see this file's doc comment, point 3). Tuned empirically against `TECH_STEP_TRAINING_DATA` — see `test/tech-step-matcher.test.ts` for the cases this threshold was picked to pass. */ +const CONFIDENCE_THRESHOLD = 0.65; + +/** + * Trains and owns the `node-nlp` model behind {@link matchTechStepSpans} — + * a real class (not a plain object of functions) per this repo's + * service-style-logic convention, even though it's only ever used as the + * one shared {@link techStepClassifier} singleton below: it holds real + * state (the trained model, the memoized training/lookup promises), not + * just grouped stateless helpers. + */ +export class TechStepClassifierService { + /** node-nlp's manager — both NER (enum entities) and NLP (intent classification) live on the same instance, trained together. */ + private readonly _manager: NlpManager; + /** Memoized training pass — `undefined` until the first call starts it, after which every caller (concurrent or not) awaits the same promise rather than retraining. */ + private _trained: Promise | undefined; + /** Memoized `TechStep.key -> id` lookup — training data only knows techniques by their stable `uid`/`key`, resolved to the real DB id once, alongside training. */ + private _techStepIdByUid: Map | undefined; + + public constructor() { + this._manager = new NlpManager({ + languages: ["fr", "en"], + forceNER: true, + nlu: { log: false }, + // node-nlp's enum-entity NER defaults to a fuzzy (Levenshtein-based) + // 0.8 accuracy threshold — loose enough that e.g. "faire" (the + // generic French helper verb in almost every recipe step) fuzzy- + // matches `fry`'s synonym "frire" at 0.80, a false positive found + // while tuning this against the real training corpus. `1` (exact, + // after node-nlp's own case/accent/stemming normalization — real + // conjugation variance is still covered by listing each form in + // `tech-step-training-data.ts`) removed it without losing any real + // match. Precision matters more than recall for this stage — NER + // only proposes candidate split points, `_classifyClause`'s trained + // model (not fuzzy string distance) is what actually has to be + // right. + ner: { threshold: 1 }, + // node-nlp defaults to `autoSave`/`autoLoad: true` — silently + // persisting the trained model to a `model.nlp` file in the process's + // cwd, and *loading from that file instead of retraining* the next + // time a manager is constructed, if the file already exists. Found + // this the hard way: a stray `model.nlp` appeared at the repo root + // after running this locally. That's the opposite of what this + // service wants — `TECH_STEP_TRAINING_DATA` in code is the single + // source of truth this always trains fresh from (see this file's own + // doc comment) — a stale on-disk model silently shadowing a + // corpus/threshold update would be a nasty, hard-to-notice class of + // bug. Both off; nothing here should ever touch disk. + autoSave: false, + autoLoad: false, }); } - // Step 2: one best candidate per techStepId. - const bestByTechStep = new Map(); - for (const candidate of candidates) { - const current = bestByTechStep.get(candidate.techStepId); - if ( - current === undefined || - candidate.weight > current.weight || - (candidate.weight === current.weight && candidate.start < current.start) - ) { - bestByTechStep.set(candidate.techStepId, candidate); + /** + * Forces training plus node-nlp's own one-time lazy setup (loading its + * bundled per-language stemmers/tokenizers on the *first* real + * `NlpManager.process()` call takes a few seconds by itself, separate + * from and much slower than the ~40ms `train()` pass — measured against + * this corpus while tuning the pipeline) to happen now, synchronously + * with server startup (see `server.ts`), rather than stalling whichever + * request happens to be first to save/preview a recipe. + */ + public async warmUp(): Promise { + try { + await this.matchTechStepSpans("faire cuire à feu doux", "fr"); + } catch (err) { + throw err; // see matchTechStepSpans()'s catch comment above } } - // Step 3: resolve cross-technique overlaps, highest weight first. - const byWeightDesc = [...bestByTechStep.values()].sort( - (a, b) => b.weight - a.weight || a.techStepId - b.techStepId, - ); - const accepted: MatchCandidate[] = []; - for (const candidate of byWeightDesc) { - if (accepted.some((other) => overlaps(candidate, other))) continue; - accepted.push(candidate); + /** + * Detects every technique `description` means, as an ordered sequence of + * matches (each carrying *where* it matched) — empty if none apply. See + * this file's doc comment for the full NER -> split -> classify + * pipeline. + * + * @param locale Which of `TECH_STEP_TRAINING_DATA`'s locales to match + * against — same "caller already knows/validated this" contract the + * old `matchTechStepSpans(description, mappings)` had via its + * pre-filtered `mappings` argument, just as an explicit parameter now + * that the training data isn't pre-filtered by the caller anymore. + */ + public async matchTechStepSpans(description: string, locale: string): Promise { + try { + await this._ensureTrained(); + if (description.trim().length === 0) return []; + + const nerResult = await this._manager.process(locale, description); + const candidates: TechniqueCandidate[] = nerResult.entities + // node-nlp's language plugins also auto-extract their own built-in + // entities (numbers, durations, dates…) alongside the enum + // entities `_train` registered from `TECH_STEP_TRAINING_DATA` — + // `type === "enum"` is what tells the two apart; without this + // filter a step like "10 minutes" would hand `splitIntoClauses` a + // bogus "duration" candidate that resolves to no real technique. + .filter((entity) => entity.type === "enum") + .map((entity) => ({ + uid: entity.entity, + start: entity.start, + // node-nlp's own `end` is inclusive (verified against a real + // trained model) — `+ 1` converts to this module's `[start, end)` + // convention, matching `String.prototype.slice`. + end: entity.end + 1, + })); + + const clauses = splitIntoClauses(description, candidates); + const matches: TechStepMatch[] = []; + for (const clause of clauses) { + const uid = await this._classifyClause(description, clause, locale); + if (uid === null) continue; + const techStepId = this._techStepIdByUid?.get(uid); + // A `uid` the classifier/NER was trained on but that no longer has + // a matching `TechStep` row (e.g. training data and + // `reference-seed-data.ts` drifted apart) — skip rather than + // persist a dangling id. + if (techStepId === undefined) continue; + const span = clause.anchor ?? { start: clause.start, end: clause.end }; + matches.push({ techStepId, start: span.start, end: span.end }); + } + + matches.sort((a, b) => a.start - b.start || a.techStepId - b.techStepId); + return matches; + } catch (err) { + // Rethrown as-is — `wrapAsyncHandler`/the error middleware (which + // already logs it, see `error-logger.ts`) is what actually handles + // it, this service layer just isn't allowed a bare `await` without a + // try/catch per the repo's convention. + throw err; + } } - // Step 4: reading order. - accepted.sort((a, b) => a.start - b.start || a.techStepId - b.techStepId); - return accepted.map(({ techStepId, start, end }) => ({ - techStepId, - start, - end, - })); -} + /** + * Convenience wrapper around {@link matchTechStepSpans} for callers that + * only care about *which* techniques matched, not where — e.g. + * `recipe-translation.ts`'s `translateRecipeSteps`, which declares a + * step's technique sequence for an imported recipe that isn't saved (and + * so has no `StepTechStep` row to persist a span into) yet. + */ + public async matchTechSteps(description: string, locale: string): Promise { + try { + return (await this.matchTechStepSpans(description, locale)).map((match) => match.techStepId); + } catch (err) { + throw err; // see matchTechStepSpans()'s catch comment above + } + } -/** - * Convenience wrapper around {@link matchTechStepSpans} for callers that - * only care about *which* techniques matched, not where — e.g. - * `recipe-translation.ts`'s `translateRecipeSteps`, which declares a step's - * technique sequence for an imported recipe that isn't saved (and so has no - * `StepTechStep` row to persist a span into) yet. - */ -export function matchTechSteps(description: string, mappings: TechStepMappingRule[]): number[] { - return matchTechStepSpans(description, mappings).map((match) => match.techStepId); -} + /** + * Classifies one clause, returning the technique `uid` it means (or + * `null` if none applies) — the classifier's own verdict when it's + * confident enough ({@link CONFIDENCE_THRESHOLD}), otherwise the + * clause's NER anchor (if it has one) as a floor: a clearly + * keyword-anchored clause a small model merely isn't sure how to + * classify shouldn't be dropped outright, only a genuinely + * anchor-less/low-confidence one should. + */ + private async _classifyClause( + description: string, + clause: TechStepClause, + locale: string, + ): Promise { + try { + const clauseText = description.slice(clause.start, clause.end).trim(); + if (clauseText.length === 0) return clause.anchor?.uid ?? null; -/** - * Loads every `TechStepMapping` row for `locale` as - * {@link TechStepMappingRule}s — meant to be fetched once per request by - * `recipe.service.ts`'s `createRecipe`/`updateRecipe` and reused across - * every step of the recipe being saved, not re-queried per step. - * - * No user-language preference exists anywhere in the app yet (a single - * `"fr"` translation file, no locale field on `User`/`UserProfile`) — - * callers pass a hardcoded locale for now; this parameter exists so that - * plugging in a real user preference later doesn't require touching this - * module. - */ -export async function loadTechStepMappingRules(locale: string): Promise { - try { - return await prisma.techStepMapping.findMany({ - where: { locale }, - select: { techStepId: true, expression: true, weight: true }, - }); - } catch (err) { - // Rethrown as-is — the caller (`recipe.service.ts`/`sources.service.ts`) - // already handles/logs failures centrally; this function just isn't - // allowed a bare `async` body without a try/catch per the repo's - // convention. - throw err; + const result = await this._manager.process(locale, clauseText); + if (result.intent !== "None" && result.score >= CONFIDENCE_THRESHOLD) { + return result.intent; + } + return clause.anchor?.uid ?? null; + } catch (err) { + throw err; // see matchTechStepSpans()'s catch comment above + } + } + + /** + * Trains `_manager` from {@link TECH_STEP_TRAINING_DATA} and resolves the + * `uid -> TechStep.id` lookup, both exactly once — memoized on + * `_trained` so a burst of concurrent calls (several steps of the same + * recipe save, awaited via the same event loop tick) all await the one + * in-flight training pass rather than each kicking off their own. + */ + private async _ensureTrained(): Promise { + if (this._trained === undefined) { + this._trained = this._train(); + } + try { + await this._trained; + } catch (err) { + // A failed training pass must be retried by the *next* call, not + // leave every future call permanently rejecting against a stale + // failed promise. + this._trained = undefined; + throw err; + } + } + + private async _train(): Promise { + try { + const techSteps = await prisma.techStep.findMany({ select: { id: true, key: true } }); + this._techStepIdByUid = new Map(techSteps.map((techStep) => [techStep.key, techStep.id])); + + for (const entry of TECH_STEP_TRAINING_DATA) { + for (const [locale, data] of [ + ["fr", entry.fr], + ["en", entry.en], + ] as const) { + if (data.synonyms.length > 0) { + this._manager.addNamedEntityText(entry.uid, entry.uid, [locale], data.synonyms); + } + for (const utterance of data.utterances) { + this._manager.addDocument(locale, utterance, entry.uid); + } + } + } + + await this._manager.train(); + } catch (err) { + throw err; // see matchTechStepSpans()'s catch comment above + } } } + +/** Single shared instance — training is expensive enough (a few hundred ms) that every caller must reuse the one already-trained model, never spin up their own. */ +export const techStepClassifier = new TechStepClassifierService(); diff --git a/apps/api/src/lib/recipe-matching/tech-step-training-data.ts b/apps/api/src/lib/recipe-matching/tech-step-training-data.ts new file mode 100644 index 0000000..7eeba37 --- /dev/null +++ b/apps/api/src/lib/recipe-matching/tech-step-training-data.ts @@ -0,0 +1,581 @@ +/** + * Training corpus for {@link TechStepClassifierService} (`tech-step-matcher.ts`) + * — one entry per `TechStep` (`uid` matches `reference-seed-data.ts`'s + * `TECH_STEPS`, which still owns the reference `TechStep` rows themselves; + * this file replaces `TECH_STEPS[].mappings`' regex expressions as the + * *matching* data source). + * + * Two distinct kinds of content per technique/locale, feeding two distinct + * mechanisms of the classifier (see that file's doc comment for why both + * are needed): + * + * - `synonyms` — short literal words/set phrases, fed to node-nlp's NER + * (enum entities). Mechanically equivalent to the old regexes' verb-form + * alternations, just spelled out as plain words instead of a pattern + * (node-nlp's own stemmer/fuzzy matching already covers minor + * conjugation/typo variance that the regexes had to enumerate by hand). + * Used only to find *candidate* technique mentions and cut a step into + * clauses around them — never the final answer on their own. + * - `utterances` — full example clauses, fed to node-nlp's NLP Manager as + * training documents for the intent classifier. Deliberately mixes + * keyword-anchored phrasings (reinforces the obvious case) with + * paraphrases that never use the technique's own verb at all (e.g. + * "jusqu'à ce que le beurre ait disparu" for `melt`) — this second kind + * is what actually delivers on "comprendre le sens, pas juste les mots + * clés" (see the PR this file was introduced in): a clause reaching the + * classifier gets labeled by what it's trained to recognize as *meaning* + * this technique, not by which literal word triggered its extraction. + * + * Kept as static in-code data (not DB rows, unlike the old + * `TechStepMapping` table) because nothing needs to query/edit it at + * runtime — it only ever feeds one thing, the classifier's one-time + * training pass (see `TechStepClassifierService._ensureTrained`) — same + * reasoning `INGREDIENT_LABELS_EN` (`packages/shared`) is a plain object, + * not a database table. + */ + +/** One technique's matching data for one locale — see this file's doc comment for what each list feeds. */ +export interface TechStepLocaleTrainingData { + synonyms: string[]; + utterances: string[]; +} + +/** One technique's full training entry — `uid` must match a `TECH_STEPS[].uid` in `reference-seed-data.ts`. */ +export interface TechStepTrainingEntry { + uid: string; + fr: TechStepLocaleTrainingData; + en: TechStepLocaleTrainingData; +} + +export const TECH_STEP_TRAINING_DATA: TechStepTrainingEntry[] = [ + { + uid: "cook", + fr: { + synonyms: ["cuire", "cuisez", "cuisant", "cuisson", "cuit", "cuite", "cuites", "cuits"], + utterances: [ + "faire cuire à feu moyen", + "laisser cuire jusqu'à ce que ce soit prêt", + "la cuisson dure environ dix minutes", + "jusqu'à ce que la viande ne soit plus rose au centre", + "poursuivre la cuisson à couvert", + ], + }, + en: { + synonyms: ["cook", "cooks", "cooked", "cooking"], + utterances: [ + "cook over medium heat", + "cook until done", + "cooking takes about ten minutes", + "until no longer pink in the middle", + "continue cooking covered", + ], + }, + }, + { + uid: "fry", + fr: { + synonyms: ["frire", "frit", "frite", "frites", "friture"], + utterances: [ + "faire frire dans l'huile chaude", + "plonger dans la friture", + "jusqu'à ce que ce soit doré et croustillant à l'extérieur", + "l'huile doit être bien chaude avant d'y plonger les morceaux", + ], + }, + en: { + synonyms: ["fry", "fries", "fried", "frying", "deep fry", "deep-fried"], + utterances: [ + "fry in hot oil", + "deep fry until golden", + "until crisp and golden on the outside", + "the oil should be very hot before adding the pieces", + ], + }, + }, + { + uid: "melt", + fr: { + synonyms: ["fondre", "fondu", "fondue", "fondues", "faire fondre", "faites fondre"], + utterances: [ + "faire fondre le beurre", + "jusqu'à ce que le beurre ait disparu dans la poêle", + "le beurre doit être complètement liquide", + "laisser le fromage devenir tout liquide sur feu doux", + ], + }, + en: { + synonyms: ["melt", "melts", "melted", "melting"], + utterances: [ + "melt the butter", + "until the butter has completely disappeared into the pan", + "the butter should be fully liquid", + "let the cheese turn completely liquid over low heat", + ], + }, + }, + { + uid: "deglaze", + fr: { + synonyms: ["déglacer", "déglacez", "déglacé", "déglacée", "déglaçage"], + utterances: [ + "déglacer avec le vin blanc", + "verser le vin dans la poêle chaude pour décoller les sucs", + "gratter les sucs de cuisson au fond de la casserole avec un peu de bouillon", + ], + }, + en: { + synonyms: ["deglaze", "deglazes", "deglazed", "deglazing"], + utterances: [ + "deglaze with white wine", + "pour the wine into the hot pan to lift the browned bits", + "scrape up the browned bits at the bottom of the pan with a splash of stock", + ], + }, + }, + { + uid: "simmer", + fr: { + synonyms: ["mijoter", "mijotez", "mijote", "mijotant", "mijoté"], + utterances: [ + "laisser mijoter à feu doux", + "faire mijoter pendant une heure", + "de petites bulles doivent remonter doucement à la surface", + "laisser cuire tout doucement à couvert pendant longtemps", + ], + }, + en: { + synonyms: ["simmer", "simmers", "simmered", "simmering"], + utterances: [ + "let it simmer over low heat", + "simmer for one hour", + "small bubbles should gently rise to the surface", + "let it cook very gently, covered, for a long time", + ], + }, + }, + { + uid: "boil", + fr: { + synonyms: ["bouillir", "bouillant", "bouillie", "bouillies", "ébullition"], + utterances: [ + "porter à ébullition", + "faire bouillir l'eau", + "de grosses bulles doivent agiter la surface avec force", + "jusqu'à ce que ça bouillonne franchement", + ], + }, + en: { + synonyms: ["boil", "boils", "boiled", "boiling"], + utterances: [ + "bring to a boil", + "boil the water", + "large bubbles should be vigorously breaking the surface", + "until it's rolling vigorously", + ], + }, + }, + { + uid: "roast", + fr: { + synonyms: ["rôtir", "rôti", "rôtie", "rôties", "rôtis"], + utterances: [ + "faire rôtir la volaille entière", + "le rôti doit dorer uniformément de tous les côtés", + "cuire la pièce de viande entière au four à chaleur sèche", + ], + }, + en: { + synonyms: ["roast", "roasts", "roasted", "roasting"], + utterances: [ + "roast the whole bird", + "it should brown evenly on every side", + "cook the whole piece of meat in dry oven heat", + ], + }, + }, + { + uid: "grill", + fr: { + synonyms: ["griller", "grillez", "grillé", "grillée", "grillées", "grillade"], + utterances: [ + "faire griller sur la grille du barbecue", + "marquer les steaks sur une plaque brûlante", + "des traces de quadrillage doivent apparaître à la cuisson", + ], + }, + en: { + synonyms: ["grill", "grills", "grilled", "grilling"], + utterances: [ + "grill on the barbecue rack", + "sear the steaks on a scorching-hot plate", + "char marks should appear as it cooks", + ], + }, + }, + { + uid: "panFry", + fr: { + synonyms: ["sauter", "sautez", "sauté", "sautée", "sautées", "sautant", "à la poêle"], + utterances: [ + "faire sauter les légumes à la poêle", + "saisir rapidement à feu vif en remuant sans cesse", + "faire revenir en remuant vivement dans une poêle très chaude", + ], + }, + en: { + synonyms: ["sauté", "sauteed", "sautéed", "sauteing", "pan-fry", "pan fried", "stir-fry"], + utterances: [ + "sauté the vegetables in a pan", + "quickly sear over high heat, stirring constantly", + "cook briskly, stirring, in a very hot pan", + ], + }, + }, + { + uid: "blanch", + fr: { + synonyms: ["blanchir", "blanchissez", "blanchi", "blanchie", "blanchies"], + utterances: [ + "faire blanchir les légumes deux minutes dans l'eau bouillante", + "plonger brièvement dans l'eau bouillante puis directement dans l'eau glacée", + "cuire très rapidement à l'eau bouillante avant de stopper la cuisson au froid", + ], + }, + en: { + synonyms: ["blanch", "blanches", "blanched", "blanching"], + utterances: [ + "blanch the vegetables for two minutes in boiling water", + "briefly plunge into boiling water then straight into ice water", + "cook very quickly in boiling water before stopping it cold", + ], + }, + }, + { + uid: "marinate", + fr: { + synonyms: ["mariner", "marinez", "mariné", "marinée", "marinées", "marinade"], + utterances: [ + "laisser mariner la viande toute la nuit au réfrigérateur", + "faire tremper dans la sauce plusieurs heures avant cuisson pour parfumer", + "laisser reposer dans le mélange d'huile et d'épices avant de cuisiner", + ], + }, + en: { + synonyms: ["marinate", "marinates", "marinated", "marinating", "marinade"], + utterances: [ + "let the meat marinate overnight in the fridge", + "soak in the sauce for several hours before cooking to flavor it", + "let it sit in the oil and spice mixture before cooking", + ], + }, + }, + { + uid: "chop", + fr: { + synonyms: ["hacher", "hachez", "haché", "hachée", "hachées", "hachis"], + utterances: [ + "hacher finement les oignons", + "couper en tout petits morceaux irréguliers au couteau", + "réduire les herbes en petits fragments avant de les ajouter", + ], + }, + en: { + synonyms: ["chop", "chops", "chopped", "chopping"], + utterances: [ + "finely chop the onions", + "cut into small, uneven pieces with a knife", + "break the herbs down into small bits before adding them", + ], + }, + }, + { + uid: "peel", + fr: { + synonyms: ["éplucher", "épluchez", "épluché", "épluchée", "épluchées", "épluchage"], + utterances: [ + "éplucher les pommes de terre", + "retirer la peau des carottes avec un économe", + "ôter la pelure du fruit avant de le couper", + ], + }, + en: { + synonyms: ["peel", "peels", "peeled", "peeling"], + utterances: [ + "peel the potatoes", + "remove the skin from the carrots with a peeler", + "take the skin off the fruit before cutting it", + ], + }, + }, + { + uid: "mince", + fr: { + synonyms: ["émincer", "émincez", "émincé", "émincée", "émincées"], + utterances: [ + "émincer l'oignon en fines lamelles", + "couper en très fines tranches régulières", + "détailler en lamelles aussi fines que possible", + ], + }, + en: { + synonyms: ["mince", "minces", "minced", "mincing", "thinly slice"], + utterances: [ + "mince the onion into thin strips", + "cut into very thin, even slices", + "slice into strips as thin as possible", + ], + }, + }, + { + uid: "mix", + fr: { + synonyms: ["mélanger", "mélangez", "mélangé", "mélangée", "mélangées", "mélange"], + utterances: [ + "mélanger tous les ingrédients dans un saladier", + "combiner le sucre et la farine ensemble", + "remuer jusqu'à obtenir une préparation homogène", + ], + }, + en: { + synonyms: ["mix", "mixes", "mixed", "mixing", "combine", "combined"], + utterances: [ + "mix all the ingredients in a bowl", + "combine the sugar and flour together", + "stir until the mixture is smooth and even", + ], + }, + }, + { + uid: "whisk", + fr: { + synonyms: ["fouetter", "fouettez", "fouetté", "fouettée", "fouettées", "au fouet"], + utterances: [ + "fouetter les œufs et le sucre", + "battre vigoureusement au fouet jusqu'à ce que ça blanchisse", + "travailler énergiquement pour incorporer de l'air au mélange", + ], + }, + en: { + synonyms: ["whisk", "whisks", "whisked", "whisking", "beat"], + utterances: [ + "whisk the eggs and sugar", + "beat vigorously with a whisk until pale", + "work it briskly to whip air into the mixture", + ], + }, + }, + { + uid: "foldIn", + fr: { + synonyms: ["incorporer", "incorporez", "incorporé", "incorporée", "incorporées"], + utterances: [ + "incorporer délicatement les blancs en neige", + "ajouter en soulevant doucement la masse pour ne pas casser les bulles", + "mélanger tout doucement de bas en haut pour garder l'air emprisonné", + ], + }, + en: { + synonyms: ["fold in", "folds in", "folded in", "folding in"], + utterances: [ + "gently fold in the beaten egg whites", + "add by gently lifting the batter so you don't knock the air out", + "very gently stir from the bottom up to keep the air trapped in", + ], + }, + }, + { + uid: "setAside", + fr: { + synonyms: ["réserver", "réservez", "réservé", "réservée", "réservées"], + utterances: [ + "réserver au frais en attendant", + "mettre de côté pour plus tard", + "laisser attendre sur le plan de travail pendant la préparation du reste", + ], + }, + en: { + synonyms: ["set aside", "sets aside", "setting aside", "set it aside"], + utterances: [ + "set aside in the fridge for now", + "put it aside for later", + "let it wait on the counter while you prepare the rest", + ], + }, + }, + { + uid: "season", + fr: { + synonyms: ["assaisonner", "assaisonnez", "assaisonné", "assaisonnée", "assaisonnement"], + utterances: [ + "assaisonner avec du sel et du poivre", + "rectifier le goût en ajoutant des épices", + "ajouter du sel selon votre goût avant de servir", + ], + }, + en: { + synonyms: ["season", "seasons", "seasoned", "seasoning"], + utterances: [ + "season with salt and pepper", + "adjust the taste by adding spices", + "add salt to taste before serving", + ], + }, + }, + { + uid: "drain", + fr: { + synonyms: ["égoutter", "égouttez", "égoutté", "égouttée", "égouttées"], + utterances: [ + "égoutter les pâtes dans une passoire", + "verser dans une passoire pour retirer l'eau de cuisson", + "laisser l'excédent d'eau s'écouler avant de servir", + ], + }, + en: { + synonyms: ["drain", "drains", "drained", "draining"], + utterances: [ + "drain the pasta in a colander", + "pour into a colander to remove the cooking water", + "let the excess water run off before serving", + ], + }, + }, + { + uid: "brown", + fr: { + synonyms: ["faire revenir", "faites revenir", "faire dorer", "faites dorer"], + utterances: [ + "faire revenir les oignons dans l'huile chaude", + "faire dorer la viande sur toutes les faces", + "saisir jusqu'à ce que la surface prenne une belle couleur caramel", + ], + }, + en: { + // Verb forms only (not bare "brown"), same reasoning the old regex + // doc comment gave — a bare "brown" false-positives on ingredient + // descriptions like "brown sugar"/"brown rice", which never get to + // the classifier since they're not step text, but keeping the + // synonym itself anchored costs nothing and stays consistent. + synonyms: ["browned", "browning"], + utterances: [ + "brown the onions in hot oil", + "brown the meat on every side", + "sear until the surface turns a deep caramel color", + ], + }, + }, + { + uid: "rest", + fr: { + synonyms: ["reposer", "laisser reposer", "laissez reposer"], + utterances: [ + "laisser reposer la pâte trente minutes", + "laisser la viande se détendre hors du four avant de la découper", + "attendre quelques minutes avant de servir pour que les jus se répartissent", + ], + }, + en: { + // Anchored to "let ... rest"/"rest for" rather than bare "rest", + // same false-positive reasoning as `brown` above ("the rest of the"). + synonyms: ["let it rest", "let them rest", "resting for", "rested for"], + utterances: [ + "let the dough rest for thirty minutes", + "let the meat relax outside the oven before carving it", + "wait a few minutes before serving so the juices redistribute", + ], + }, + }, + { + uid: "preheat", + fr: { + synonyms: ["préchauffer", "préchauffez", "préchauffé", "préchauffée"], + utterances: [ + "préchauffer le four à 180 degrés", + "mettre le four à chauffer avant d'y placer le plat", + "allumer le four à l'avance pour qu'il soit à température", + // A pan gets preheated too, not just an oven — without an example + // like this, "poêle" (which also appears throughout `panFry`'s own + // training utterances) biased the classifier toward `panFry` for + // any preheating clause that happens to mention a pan, found while + // testing against the classic "Préchauffer la poêle, puis faire + // fondre le beurre" case. + "préchauffer la poêle avant d'y verser l'huile", + "faire chauffer la poêle à vide quelques minutes", + // "poêle" + "feu vif" together still read as `panFry` (the act of + // actually cooking something in it) rather than `preheat` (getting + // it hot beforehand, nothing in it yet) without an example this + // close to that exact wording — found via "mettre la poêle sur feu + // vif" (no food mentioned at all) still classifying as panFry. + "mettre la poêle vide sur feu vif avant d'ajouter quoi que ce soit", + "mettre la poêle sur feu vif", + ], + }, + en: { + synonyms: ["preheat", "preheats", "preheated", "preheating"], + utterances: [ + "preheat the oven to 180 degrees", + "turn the oven on to heat up before putting the dish in", + "switch the oven on ahead of time so it's up to temperature", + "preheat the pan before adding the oil", + "heat the empty pan for a few minutes first", + ], + }, + }, + { + uid: "bake", + fr: { + synonyms: ["cuire au four", "cuisson au four", "enfourner", "enfournez", "au four"], + utterances: [ + "enfourner pendant quarante-cinq minutes", + "mettre au four jusqu'à ce que ce soit doré", + "cuire dans le four préchauffé jusqu'à ce que la surface soit ferme", + ], + }, + en: { + synonyms: ["bake", "bakes", "baked", "baking", "in the oven"], + utterances: [ + "bake for forty-five minutes", + "put it in the oven until golden", + "cook in the preheated oven until the surface is firm", + ], + }, + }, + { + uid: "plate", + fr: { + synonyms: ["dresser", "dressez", "dressage"], + utterances: [ + "dresser harmonieusement dans les assiettes", + "disposer joliment sur l'assiette avant de servir", + "présenter avec soin au centre de l'assiette", + ], + }, + en: { + synonyms: ["plate", "plates", "plated", "plating"], + utterances: [ + "plate it up nicely", + "arrange it neatly on the plate before serving", + "present it carefully in the center of the plate", + ], + }, + }, + { + uid: "coat", + fr: { + synonyms: ["napper", "nappez", "nappé", "nappée", "nappées", "nappage"], + utterances: [ + "napper le gâteau de chocolat fondu", + "recouvrir uniformément d'une fine couche de sauce", + "verser la sauce par-dessus pour bien enrober", + ], + }, + en: { + synonyms: ["coat", "coats", "coated", "coating"], + utterances: [ + "coat the cake with melted chocolate", + "cover evenly with a thin layer of sauce", + "pour the sauce over it so it's well covered", + ], + }, + }, +]; diff --git a/apps/api/src/lib/recipe-sources/recipe-source-adapter.ts b/apps/api/src/lib/recipe-sources/recipe-source-adapter.ts index d663855..822e32b 100644 --- a/apps/api/src/lib/recipe-sources/recipe-source-adapter.ts +++ b/apps/api/src/lib/recipe-sources/recipe-source-adapter.ts @@ -16,9 +16,9 @@ * user has already brought in as if they were new. A separate, pure * step rather than something `list()` itself does: an adapter only * knows its source, never our database — same reasoning as - * `tech-step-matcher.ts`'s split between pure `matchTechStep` and its - * DB-touching `loadTechStepMappingRules`. Whichever future layer - * queries "which externalIds from this source do we already have" + * `tech-step-matcher.ts`'s split between pure `splitIntoClauses` and + * its DB/model-touching `TechStepClassifierService`. Whichever future + * layer queries "which externalIds from this source do we already have" * (not yet decided — it needs a place to persist that link, * see {@link RecipeSourceListItem.externalId}) calls this to annotate * the page before returning it. @@ -177,7 +177,7 @@ export interface RecipeSourceAdapter { * `steps[].description`/`ingredients[].name`) — e.g. `"en"` for * TheMealDB. Not a user preference: the language the source's own * content is actually written in, regardless of who's browsing it. - * Determines which `TechStepMapping`/ingredient-label locale + * Determines which trained-classifier/ingredient-label locale * `translateRecipe` (`recipe-translation.ts`) resolves this source's * recipes against when previewing/importing one. */ diff --git a/apps/api/src/modules/recipe/recipe.service.ts b/apps/api/src/modules/recipe/recipe.service.ts index 64a2e1b..d59423b 100644 --- a/apps/api/src/modules/recipe/recipe.service.ts +++ b/apps/api/src/modules/recipe/recipe.service.ts @@ -15,15 +15,15 @@ import { import type { Prisma } from "@prisma/client"; import { prisma } from "../../db/prisma.js"; import { - loadTechStepMappingRules, - matchTechStepSpans, + type TechStepMatch, + techStepClassifier, } from "../../lib/recipe-matching/tech-step-matcher.js"; // No user-language preference exists anywhere in the app yet (a single // "fr" translation file, no locale field on User/UserProfile) — steps are // matched against this hardcoded locale for now. See -// `tech-step-matcher.ts`'s `loadTechStepMappingRules` for why the locale is -// a parameter rather than baked into that module. +// `tech-step-matcher.ts`'s `TechStepClassifierService` for why the locale +// is a parameter rather than baked into that module. const DEFAULT_TECH_STEP_LOCALE = "fr"; /** Prisma `include` for every query that needs a full {@link RecipeView} — ingredients resolved to their reference data + allergens, steps in order, diet tags, and whether `viewerId` has favorited it. Parameterized by viewer since `favoritedBy` is per-viewer, not a static shape. */ @@ -454,6 +454,36 @@ export async function createImportedRecipe( } } +/** One input step, bundled with its own technique matches — see {@link matchStepsTechSteps}. */ +interface StepWithTechSteps { + step: T; + matches: TechStepMatch[]; +} + +/** + * Matches every one of `steps`' technique sequence against `locale`, in + * parallel, each bundled back with its own originating step (rather than + * returned as a same-length array callers would have to re-zip with + * `steps` by index — `noUncheckedIndexedAccess` makes that genuinely + * awkward for no benefit, since every match list is only ever read back + * once) — the shared prep step {@link createRecipeInternal}/ + * {@link updateRecipe} both need before building their (synchronous) + * Prisma `create` payload, now that matching itself is async + * (`techStepClassifier`, a trained model rather than a pure regex test — + * see `tech-step-matcher.ts`). + */ +async function matchStepsTechSteps( + steps: T[], + locale: string, +): Promise[]> { + return Promise.all( + steps.map(async (step) => ({ + step, + matches: await techStepClassifier.matchTechStepSpans(step.description, locale), + })), + ); +} + async function createRecipeInternal( input: CreateRecipeInput, authorId: number, @@ -464,7 +494,13 @@ async function createRecipeInternal( await assertIngredientsExist(input.ingredients.map((i) => i.ingredientId)); await assertUnitsExist(input.ingredients.map((i) => i.unitId)); await assertDietsExist(input.dietIds); - const techStepMappings = await loadTechStepMappingRules( + // Matched up front (one call per step, in parallel) rather than inline + // inside the `steps.create` map below — `techStepClassifier` is async + // (a trained model, not a pure regex test), so its result has to + // already be in hand by the time this synchronous Prisma payload is + // built. + const stepsWithTechSteps = await matchStepsTechSteps( + input.steps, source?.locale ?? DEFAULT_TECH_STEP_LOCALE, ); @@ -487,19 +523,17 @@ async function createRecipeInternal( })), }, steps: { - create: input.steps.map((step, index) => ({ + create: stepsWithTechSteps.map(({ step, matches }, index) => ({ description: step.description, picture: step.picture ?? null, order: index, techSteps: { - create: matchTechStepSpans(step.description, techStepMappings).map( - (match, order) => ({ - techStepId: match.techStepId, - start: match.start, - end: match.end, - order, - }), - ), + create: matches.map((match, order) => ({ + techStepId: match.techStepId, + start: match.start, + end: match.end, + order, + })), }, })), }, @@ -537,7 +571,7 @@ export async function updateRecipe( await assertIngredientsExist(input.ingredients.map((i) => i.ingredientId)); await assertUnitsExist(input.ingredients.map((i) => i.unitId)); await assertDietsExist(input.dietIds); - const techStepMappings = await loadTechStepMappingRules(DEFAULT_TECH_STEP_LOCALE); + const stepsWithTechSteps = await matchStepsTechSteps(input.steps, DEFAULT_TECH_STEP_LOCALE); await prisma.$transaction([ prisma.recipeIngredient.deleteMany({ where: { recipeId: id } }), @@ -559,19 +593,17 @@ export async function updateRecipe( })), }, steps: { - create: input.steps.map((step, index) => ({ + create: stepsWithTechSteps.map(({ step, matches }, index) => ({ description: step.description, picture: step.picture ?? null, order: index, techSteps: { - create: matchTechStepSpans(step.description, techStepMappings).map( - (match, order) => ({ - techStepId: match.techStepId, - start: match.start, - end: match.end, - order, - }), - ), + create: matches.map((match, order) => ({ + techStepId: match.techStepId, + start: match.start, + end: match.end, + order, + })), }, })), }, diff --git a/apps/api/src/modules/sources/sources.service.ts b/apps/api/src/modules/sources/sources.service.ts index 5ca4317..7413f99 100644 --- a/apps/api/src/modules/sources/sources.service.ts +++ b/apps/api/src/modules/sources/sources.service.ts @@ -20,10 +20,7 @@ import { mergeDuplicateIngredients, translateRecipeIngredients, } from "../../lib/recipe-matching/recipe-translation.js"; -import { - loadTechStepMappingRules, - matchTechStepSpans, -} from "../../lib/recipe-matching/tech-step-matcher.js"; +import { techStepClassifier } from "../../lib/recipe-matching/tech-step-matcher.js"; import { markAlreadyImported, type RecipeSourceAdapter, @@ -172,14 +169,20 @@ export async function previewSourceItem( throw err; } - const [techStepMappings, ingredientCatalog, unitCatalog, techStepsByKey] = await Promise.all([ - loadTechStepMappingRules(adapter.locale), - adapter.locale === "en" - ? loadIngredientCatalog() - : Promise.resolve([]), - adapter.locale === "en" ? loadUnitCatalog() : Promise.resolve([]), - prisma.techStep.findMany({ select: { id: true, key: true } }), - ]); + const [stepsWithTechStepMatches, ingredientCatalog, unitCatalog, techStepsByKey] = + await Promise.all([ + Promise.all( + parsed.steps.map(async (step) => ({ + step, + matches: await techStepClassifier.matchTechStepSpans(step.description, adapter.locale), + })), + ), + adapter.locale === "en" + ? loadIngredientCatalog() + : Promise.resolve([]), + adapter.locale === "en" ? loadUnitCatalog() : Promise.resolve([]), + prisma.techStep.findMany({ select: { id: true, key: true } }), + ]); const techStepById = new Map(techStepsByKey.map((techStep) => [techStep.id, techStep])); const translatedIngredients = translateRecipeIngredients( @@ -209,10 +212,10 @@ export async function previewSourceItem( unit: ingredient.unitId !== null ? (unitById.get(ingredient.unitId) ?? null) : null, })); - const steps: DraftRecipeStepView[] = parsed.steps.map((step) => ({ + const steps: DraftRecipeStepView[] = stepsWithTechStepMatches.map(({ step, matches }) => ({ description: step.description, picture: step.picture, - techSteps: matchTechStepSpans(step.description, techStepMappings).flatMap((match) => { + techSteps: matches.flatMap((match) => { const techStep = techStepById.get(match.techStepId); return techStep ? [{ techStep, start: match.start, end: match.end }] : []; }), diff --git a/apps/api/src/server.ts b/apps/api/src/server.ts index 85f2cf9..9c477a9 100644 --- a/apps/api/src/server.ts +++ b/apps/api/src/server.ts @@ -1,6 +1,7 @@ import { createServer } from "./app.js"; import { env } from "./config/env.js"; import { logger } from "./lib/logger.service.js"; +import { techStepClassifier } from "./lib/recipe-matching/tech-step-matcher.js"; import { registerAllRecipeSources } from "./sources/index.js"; // Populates the recipe-source registry (recipe-source-registry.ts) before @@ -8,6 +9,22 @@ import { registerAllRecipeSources } from "./sources/index.js"; // doesn't happen inside app.ts/createServer() itself. registerAllRecipeSources(); +// Trains the tech-step classifier (and pays node-nlp's own one-time lazy +// setup cost — see `TechStepClassifierService.warmUp`) before accepting +// any traffic, so the first real recipe save/preview isn't the one stuck +// waiting several seconds for it. +try { + await techStepClassifier.warmUp(); +} catch (err) { + // Not fatal to startup — a failed warm-up just means the *next* call + // retries training itself (see `_ensureTrained`'s own retry-on-failure + // comment), same graceful-degrade posture as everywhere else training + // failures surface. Still worth a loud log: this shouldn't normally fail. + logger.error("Tech-step classifier warm-up failed", { + error: err instanceof Error ? err.message : String(err), + }); +} + const server = createServer(); server.listen(env.PORT, () => { diff --git a/apps/api/src/types/node-nlp.d.ts b/apps/api/src/types/node-nlp.d.ts new file mode 100644 index 0000000..36d8fa3 --- /dev/null +++ b/apps/api/src/types/node-nlp.d.ts @@ -0,0 +1,50 @@ +/** + * Minimal ambient typing for `node-nlp` (no official/DefinitelyTyped types + * exist for it) — declares only the `NlpManager` surface + * `tech-step-matcher.ts` actually calls, verified against the real + * package (v4.27.0) rather than the library's full documented API, which + * this repo doesn't use the rest of. + */ +declare module "node-nlp" { + /** Constructor options this repo passes — `NlpManager` accepts more, only what's used here is typed. */ + export interface NlpManagerOptions { + languages?: string[]; + forceNER?: boolean; + nlu?: { log?: boolean }; + ner?: { threshold?: number }; + /** Defaults to `true` — persists the trained model to `modelFileName` (default `model.nlp`, in `process.cwd()`). See `tech-step-matcher.ts`'s own constructor comment for why this repo always sets it `false`. */ + autoSave?: boolean; + /** Defaults to `true` — loads from `modelFileName` instead of training fresh if that file already exists. Always `false` here, same reasoning as `autoSave`. */ + autoLoad?: boolean; + } + + /** One entity `NlpManager.process`'s result reports — see `tech-step-matcher.ts`'s own `NerEntity` for the subset this repo reads. */ + export interface NlpEntity { + entity: string; + start: number; + end: number; + type: string; + accuracy?: number; + sourceText?: string; + } + + /** `NlpManager.process`'s result — trimmed to the fields this repo reads (the real object carries many more). */ + export interface NlpProcessResult { + intent: string; + score: number; + entities: NlpEntity[]; + } + + export class NlpManager { + public constructor(options?: NlpManagerOptions); + public addNamedEntityText( + entityName: string, + optionName: string, + languages: string[], + texts: string[], + ): void; + public addDocument(locale: string, utterance: string, intent: string): void; + public train(): Promise; + public process(locale: string, text: string): Promise; + } +} diff --git a/apps/api/test-support/reset-db.ts b/apps/api/test-support/reset-db.ts index cdf8e04..c12b1ac 100644 --- a/apps/api/test-support/reset-db.ts +++ b/apps/api/test-support/reset-db.ts @@ -45,7 +45,7 @@ export async function resetDatabase() { TRUNCATE TABLE "user_profile_allergy", "user_preference", "allergy", "category", "planning_item", "planning", - "recipe_ingredient", "step_tech_step", "step", "tech_step_mapping", "tech_step", + "recipe_ingredient", "step_tech_step", "step", "tech_step", "recipe", "ingredients", "sources", "unit", "user_profiles", "diet", "house" RESTART IDENTITY CASCADE; diff --git a/apps/api/test/recipe-matching/recipe-translation.test.ts b/apps/api/test/recipe-matching/recipe-translation.test.ts index c74b98d..b06c51e 100644 --- a/apps/api/test/recipe-matching/recipe-translation.test.ts +++ b/apps/api/test/recipe-matching/recipe-translation.test.ts @@ -12,7 +12,6 @@ import { translateRecipeSteps, type UnitConversionEntry, } from "../../src/lib/recipe-matching/recipe-translation.js"; -import type { TechStepMappingRule } from "../../src/lib/recipe-matching/tech-step-matcher.js"; import type { ParsedRecipe, ParsedRecipeIngredient, @@ -36,56 +35,66 @@ function buildParsedRecipe(descriptions: string[]): ParsedRecipe { } describe("recipe-translation", () => { + // `translateRecipeSteps` now goes through `techStepClassifier` (a + // trained model, not a pure regex test against a caller-supplied + // mapping list — see `tech-step-matcher.ts`), so these tests exercise + // the real training corpus (`tech-step-training-data.ts`) against a real + // `TechStep` catalog rather than synthetic fixtures — same posture + // `tech-step-matcher.test.ts`'s own `techStepClassifier` describe block + // takes, for the same reason. describe("translateRecipeSteps", () => { - const simmer: TechStepMappingRule = { - techStepId: 1, - expression: "\\bmijot(er|ez|e|ant|é)\\b", - weight: 15, - }; - const preheat: TechStepMappingRule = { - techStepId: 2, - expression: "\\bpr[ée]chauff(er|ez|é|ée)\\b", - weight: 20, - }; - const melt: TechStepMappingRule = { - techStepId: 3, - expression: "\\bfaire fondre\\b|\\bfaites fondre\\b", - weight: 15, - }; + let simmerId: number; + let preheatId: number; + let meltId: number; - it("declares each step's technique sequence, preserving order", () => { + beforeEach(async () => { + await resetDatabase(); + simmerId = (await prisma.techStep.findFirstOrThrow({ where: { key: "simmer" } })).id; + preheatId = (await prisma.techStep.findFirstOrThrow({ where: { key: "preheat" } })).id; + meltId = (await prisma.techStep.findFirstOrThrow({ where: { key: "melt" } })).id; + }); + + after(async () => { + await prisma.$disconnect(); + }); + + it("declares each step's technique sequence, preserving order", async () => { const recipe = buildParsedRecipe([ "Préchauffer la poêle, puis faire fondre le beurre", "Servir immédiatement", "Faire mijoter à feu doux", ]); - const translated = translateRecipeSteps(recipe, [simmer, preheat, melt]); + const translated = await translateRecipeSteps(recipe, "fr"); - expect(translated.steps.map((step) => step.techStepIds)).to.deep.equal([[2, 3], [], [1]]); + expect(translated.steps.map((step) => step.techStepIds)).to.deep.equal([ + [preheatId, meltId], + [], + [simmerId], + ]); }); - it("leaves description/picture untouched on each step", () => { - const recipe = buildParsedRecipe(["Faire mijoter à feu doux", "Servir"]); + it("leaves description/picture untouched on each step", async () => { + const recipe = buildParsedRecipe(["Faire mijoter à feu doux", "Servir immédiatement"]); - const translated = translateRecipeSteps(recipe, [simmer]); + const translated = await translateRecipeSteps(recipe, "fr"); expect(translated.steps[0]).to.deep.equal({ description: "Faire mijoter à feu doux", picture: "https://example.test/step1.jpg", - techStepIds: [1], + techStepIds: [simmerId], }); expect(translated.steps[1]).to.deep.equal({ - description: "Servir", + description: "Servir immédiatement", picture: null, techStepIds: [], }); }); - it("passes every other field through unchanged", () => { - const recipe = buildParsedRecipe(["Servir"]); + it("passes every other field through unchanged", async () => { + const recipe = buildParsedRecipe(["Servir immédiatement"]); - const translated = translateRecipeSteps(recipe, []); + const translated = await translateRecipeSteps(recipe, "fr"); expect(translated.name).to.equal(recipe.name); expect(translated.description).to.equal(recipe.description); @@ -94,28 +103,31 @@ describe("recipe-translation", () => { expect(translated.sourceUrl).to.equal(recipe.sourceUrl); }); - it("stubs every ingredient's ingredientId/unitId to null, leaving the rest of it untouched — actual ingredient matching is translateRecipeIngredients' job", () => { - const recipe = buildParsedRecipe(["Servir"]); + it("stubs every ingredient's ingredientId/unitId to null, leaving the rest of it untouched — actual ingredient matching is translateRecipeIngredients' job", async () => { + const recipe = buildParsedRecipe(["Servir immédiatement"]); - const translated = translateRecipeSteps(recipe, []); + const translated = await translateRecipeSteps(recipe, "fr"); expect(translated.ingredients).to.deep.equal([ { ...recipe.ingredients[0], ingredientId: null, unitId: null }, ]); }); - it("gives every step an empty sequence when there are no mappings at all", () => { - const recipe = buildParsedRecipe(["Faire mijoter à feu doux", "Préchauffer le four"]); + it("gives every step an empty sequence when nothing in it means a known technique", async () => { + const recipe = buildParsedRecipe([ + "Servir immédiatement", + "Ranger les couverts dans le tiroir", + ]); - const translated = translateRecipeSteps(recipe, []); + const translated = await translateRecipeSteps(recipe, "fr"); expect(translated.steps.map((step) => step.techStepIds)).to.deep.equal([[], []]); }); - it("handles a recipe with no steps without error", () => { + it("handles a recipe with no steps without error", async () => { const recipe = buildParsedRecipe([]); - const translated = translateRecipeSteps(recipe, [simmer]); + const translated = await translateRecipeSteps(recipe, "fr"); expect(translated.steps).to.deep.equal([]); }); diff --git a/apps/api/test/recipe-matching/tech-step-matcher.test.ts b/apps/api/test/recipe-matching/tech-step-matcher.test.ts index 6f3a7b8..9f8b89f 100644 --- a/apps/api/test/recipe-matching/tech-step-matcher.test.ts +++ b/apps/api/test/recipe-matching/tech-step-matcher.test.ts @@ -1,11 +1,10 @@ import { expect } from "chai"; import { prisma } from "../../src/db/prisma.js"; import { - loadTechStepMappingRules, - matchTechStepSpans, - matchTechSteps, normalizeText, - type TechStepMappingRule, + splitIntoClauses, + type TechniqueCandidate, + techStepClassifier, } from "../../src/lib/recipe-matching/tech-step-matcher.js"; import { resetDatabase } from "../../test-support/reset-db.js"; @@ -28,227 +27,261 @@ describe("tech-step-matcher", () => { }); }); - describe("matchTechSteps", () => { - const simmer: TechStepMappingRule = { - techStepId: 1, - expression: "\\bmijot(er|ez|e|ant|é)\\b", - weight: 15, - }; - const cook: TechStepMappingRule = { - techStepId: 2, - expression: "\\bcui(re|sez|sant|sson)\\b|\\bcuit(e|es|s)?\\b", - weight: 10, - }; - const bake: TechStepMappingRule = { - techStepId: 3, - expression: - "\\bcuire au four\\b|\\bcuisson au four\\b|\\benfourn(er|ez|é|ée|ées)\\b|\\bau four\\b", - weight: 25, - }; - const preheat: TechStepMappingRule = { - techStepId: 4, - expression: "\\bpr[ée]chauff(er|ez|é|ée)\\b", - weight: 20, - }; - const melt: TechStepMappingRule = { - techStepId: 5, - expression: "\\bfondre\\b|\\bfaire fondre\\b|\\bfaites fondre\\b", - weight: 15, - }; + describe("splitIntoClauses", () => { + // A candidate's own `uid` doesn't matter to the splitting logic itself + // (it's opaque, carried through as `anchor`) — kept short and + // arbitrary across these fixtures. + function candidate(uid: string, start: number, end: number): TechniqueCandidate { + return { uid, start, end }; + } - it("matches an exact expression", () => { - expect(matchTechSteps("Faire mijoter à feu doux", [simmer])).to.deep.equal([1]); + it("returns the whole description as one anchor-less clause when there are no candidates", () => { + const text = "Servir immédiatement"; + const result = splitIntoClauses(text, []); + expect(result).to.deep.equal([{ start: 0, end: text.length, anchor: null }]); }); - it("is case- and accent-insensitive, on both the description and the expression itself", () => { - // `simmer`'s own expression source contains a literal "é" — exercises - // normalizeText being applied to the expression, not just the description. - expect(matchTechSteps("FAIRE MIJOTER", [simmer])).to.deep.equal([1]); - expect(matchTechSteps("faire mijote", [simmer])).to.deep.equal([1]); + it("returns the whole description as one clause anchored on the single candidate", () => { + const melt = candidate("melt", 6, 13); + const text = "Faire fondre le beurre"; + const result = splitIntoClauses(text, [melt]); + expect(result).to.deep.equal([{ start: 0, end: text.length, anchor: melt }]); }); - it("returns an empty sequence when nothing matches", () => { - expect(matchTechSteps("Servir immédiatement", [simmer, cook, bake])).to.deep.equal([]); - }); - - it("returns an empty sequence for an empty mappings list", () => { - expect(matchTechSteps("Faire mijoter à feu doux", [])).to.deep.equal([]); - }); - - it("returns an empty sequence for an empty description", () => { - expect(matchTechSteps("", [simmer, cook, bake])).to.deep.equal([]); - }); - - it("detects several distinct, non-overlapping techniques as an ordered sequence", () => { - // The motivating case: "Dans une poêle chaude, faire chauffer une noix - // de beurre" involves both preheating and melting — a step can name - // more than one technique, in the order they're mentioned. - expect( - matchTechSteps("Préchauffer la poêle, puis faire fondre le beurre", [preheat, melt]), - ).to.deep.equal([4, 5]); - // Order in the output follows order of mention in the text, not - // argument order. - expect( - matchTechSteps("Préchauffer la poêle, puis faire fondre le beurre", [melt, preheat]), - ).to.deep.equal([4, 5]); - }); - - it("reverses the sequence when the techniques are mentioned in the opposite order", () => { - expect( - matchTechSteps("Faire fondre le beurre puis préchauffer le four", [preheat, melt]), - ).to.deep.equal([5, 4]); - }); - - it("keeps only the highest-weight technique when two different techniques' expressions overlap the same words", () => { - // "Cuire au four" matches both `cook` (weight 10) and `bake` (weight - // 25) at essentially the same span — only the more specific `bake` - // should survive, not both. - expect(matchTechSteps("Cuire au four pendant 30 minutes", [cook, bake])).to.deep.equal([3]); - // Order-independent. - expect(matchTechSteps("Cuire au four pendant 30 minutes", [bake, cook])).to.deep.equal([3]); - }); - - it("still keeps a non-overlapping technique alongside an overlap-resolved one", () => { - // `bake` wins over `cook` for "cuire au four" (overlap), but `melt` - // matches an entirely different, non-overlapping span and survives. - const result = matchTechSteps("Faire fondre le beurre, puis cuire au four", [ - cook, - bake, - melt, - ]); - expect(result).to.deep.equal([5, 3]); - }); - - it("breaks a same-span weight tie by lowest techStepId", () => { - const a: TechStepMappingRule = { techStepId: 5, expression: "\\bmelanger\\b", weight: 10 }; - const b: TechStepMappingRule = { techStepId: 2, expression: "\\bmelanger\\b", weight: 10 }; - expect(matchTechSteps("Mélanger les ingrédients", [a, b])).to.deep.equal([2]); - }); - - it("still resolves to one techStep when two of its own mappings both match", () => { - const wholeWord: TechStepMappingRule = { - techStepId: 7, - expression: "\\bmijoter\\b", - weight: 15, - }; - const withAdverb: TechStepMappingRule = { - techStepId: 7, - expression: "\\bmijoter à feu doux\\b", - weight: 15, - }; - expect(matchTechSteps("Faire mijoter à feu doux", [wholeWord, withAdverb])).to.deep.equal([ - 7, - ]); - }); - - it("respects word boundaries — a technique's verb embedded in a longer word doesn't false-positive", () => { - // "recuire"/"précuit" contain "cuire"/"cuit" as a substring, but not as - // a standalone word — the \b-anchored expression must not match them. - expect(matchTechSteps("Faire recuire la sauce", [cook])).to.deep.equal([]); - expect(matchTechSteps("Un plat précuit", [cook])).to.deep.equal([]); - // The standalone forms still match. - expect(matchTechSteps("Faire cuire la sauce", [cook])).to.deep.equal([2]); - expect(matchTechSteps("Le riz est cuit", [cook])).to.deep.equal([2]); - }); - }); - - describe("matchTechStepSpans", () => { - // Same fixtures as `matchTechSteps` above (kept local to this describe - // block rather than shared — each block's fixtures should be readable - // on their own). - const simmer: TechStepMappingRule = { - techStepId: 1, - expression: "\\bmijot(er|ez|e|ant|é)\\b", - weight: 15, - }; - const cook: TechStepMappingRule = { - techStepId: 2, - expression: "\\bcui(re|sez|sant|sson)\\b|\\bcuit(e|es|s)?\\b", - weight: 10, - }; - const bake: TechStepMappingRule = { - techStepId: 3, - expression: - "\\bcuire au four\\b|\\bcuisson au four\\b|\\benfourn(er|ez|é|ée|ées)\\b|\\bau four\\b", - weight: 25, - }; - const preheat: TechStepMappingRule = { - techStepId: 4, - expression: "\\bpr[ée]chauff(er|ez|é|ée)\\b", - weight: 20, - }; - const melt: TechStepMappingRule = { - techStepId: 5, - expression: "\\bfondre\\b|\\bfaire fondre\\b|\\bfaites fondre\\b", - weight: 15, - }; - - it("returns the matched span alongside the techStepId for a simple match", () => { - // "Faire mijoter à feu doux" — "mijoter" starts right after "Faire ". - expect(matchTechStepSpans("Faire mijoter à feu doux", [simmer])).to.deep.equal([ - { techStepId: 1, start: 6, end: 13 }, - ]); - }); - - it("returns an empty list when nothing matches", () => { - expect(matchTechStepSpans("Servir immédiatement", [simmer, cook, bake])).to.deep.equal([]); - }); - - it("returns each distinct technique's own span, in reading order", () => { + it("splits into two clauses at the midpoint of the gap between two candidates", () => { + // "Préchauffer la poêle, puis faire fondre le beurre" + // 0 1 2 3 4 + // 0123456789012345678901234567890123456789012345678901 + const preheat = candidate("preheat", 0, 11); // "Préchauffer" + const melt = candidate("melt", 27, 39); // "faire fondre" const text = "Préchauffer la poêle, puis faire fondre le beurre"; - const result = matchTechStepSpans(text, [preheat, melt]); + + const result = splitIntoClauses(text, [preheat, melt]); + expect(result).to.have.length(2); - expect(result[0].techStepId).to.equal(4); - expect(result[1].techStepId).to.equal(5); - // Each span, sliced back out of the original text, is exactly the - // word(s) that triggered that match — what the frontend needs to - // highlight the right characters. - expect(text.slice(result[0].start, result[0].end).toLowerCase()).to.equal("préchauffer"); - expect(text.slice(result[1].start, result[1].end).toLowerCase()).to.equal("faire fondre"); + expect(result[0]).to.deep.equal({ start: 0, end: 19, anchor: preheat }); + expect(result[1]).to.deep.equal({ start: 19, end: text.length, anchor: melt }); + // The two clauses are contiguous and cover the whole text. + expect( + text.slice(result[0].start, result[0].end) + text.slice(result[1].start, result[1].end), + ).to.equal(text); }); - it("keeps only the winning span when two techniques' expressions overlap", () => { - // `bake` (weight 25) wins over `cook` (weight 10) for "cuire au four" - // — only bake's span survives, not two overlapping entries. - const text = "Cuire au four pendant 30 minutes"; - const result = matchTechStepSpans(text, [cook, bake]); - expect(result).to.deep.equal([{ techStepId: 3, start: 0, end: 13 }]); - expect(text.slice(0, 13)).to.equal("Cuire au four"); + it("sorts out-of-order candidates before splitting, and anchors each clause on the matching one", () => { + const preheat = candidate("preheat", 0, 11); + const melt = candidate("melt", 27, 39); + // Passed in reverse — the function must still produce clauses in + // reading order, each anchored on the right candidate. + const result = splitIntoClauses("Préchauffer la poêle, puis faire fondre le beurre", [ + melt, + preheat, + ]); + expect(result.map((clause) => clause.anchor?.uid)).to.deep.equal(["preheat", "melt"]); + }); + + it("produces N contiguous clauses for N candidates, each anchored on its own", () => { + const a = candidate("a", 0, 3); + const b = candidate("b", 10, 13); + const c = candidate("c", 20, 23); + const text = "x".repeat(30); + + const result = splitIntoClauses(text, [a, b, c]); + + expect(result).to.have.length(3); + expect(result.map((clause) => clause.anchor?.uid)).to.deep.equal(["a", "b", "c"]); + // Contiguous: each clause's end is the next one's start. + expect(result[0].start).to.equal(0); + expect(result[0].end).to.equal(result[1].start); + expect(result[1].end).to.equal(result[2].start); + expect(result[2].end).to.equal(text.length); + }); + + it("clamps the split point to the earlier candidate's own end when two candidates are adjacent/overlapping", () => { + // Gap midpoint would fall *before* `a`'s own end here — must not + // produce a clause that cuts into `a`'s own anchor span. + const a = candidate("a", 0, 10); + const b = candidate("b", 8, 15); + + const result = splitIntoClauses("x".repeat(20), [a, b]); + + expect(result[0].end).to.be.at.least(a.end); + expect(result[1].start).to.equal(result[0].end); }); }); - describe("loadTechStepMappingRules", () => { + describe("techStepClassifier", () => { + // `techStepClassifier` is the one shared singleton (see + // tech-step-matcher.ts's own doc comment on why) — these tests + // exercise it against the real training corpus + // (`tech-step-training-data.ts`) and the real seeded `TechStep` + // catalog, rather than synthetic injectable fixtures the old + // regex-based `matchTechStepSpans(description, mappings)` allowed. + // Training + node-nlp's own one-time per-language setup can take a + // few seconds on the very first call in the whole suite (subsequent + // calls reuse the same trained model and are fast) — comfortably + // inside this suite's default 10s timeout (.mocharc.json). + let simmerId: number; + let cookId: number; + let bakeId: number; + let preheatId: number; + let meltId: number; + let boilId: number; + let chopId: number; + beforeEach(async () => { await resetDatabase(); + const [simmer, cook, bake, preheat, melt, boil, chop] = await Promise.all([ + prisma.techStep.findFirstOrThrow({ where: { key: "simmer" } }), + prisma.techStep.findFirstOrThrow({ where: { key: "cook" } }), + prisma.techStep.findFirstOrThrow({ where: { key: "bake" } }), + prisma.techStep.findFirstOrThrow({ where: { key: "preheat" } }), + prisma.techStep.findFirstOrThrow({ where: { key: "melt" } }), + prisma.techStep.findFirstOrThrow({ where: { key: "boil" } }), + prisma.techStep.findFirstOrThrow({ where: { key: "chop" } }), + ]); + simmerId = simmer.id; + cookId = cook.id; + bakeId = bake.id; + preheatId = preheat.id; + meltId = melt.id; + boilId = boil.id; + chopId = chop.id; }); after(async () => { await prisma.$disconnect(); }); - it("only returns mappings for the requested locale", async () => { - const simmer = await prisma.techStep.findFirstOrThrow({ where: { key: "simmer" } }); - // "de" has no seeded mappings at all (unlike "fr"/"en", which the - // real catalog now both populate) — a clean locale to attach one - // synthetic row to without conflating it with real seed data. - await prisma.techStepMapping.create({ - data: { techStepId: simmer.id, locale: "de", expression: "\\bsimmer\\b", weight: 15 }, + describe("matchTechSteps", () => { + it("matches an exact expression", async () => { + expect( + await techStepClassifier.matchTechSteps("Faire mijoter à feu doux", "fr"), + ).to.deep.equal([simmerId]); }); - // The seeded catalog (26 "fr" mappings) must be untouched by the extra - // "de" row — same count, and none of them carry its expression. - const frRules = await loadTechStepMappingRules("fr"); - expect(frRules).to.have.length(26); - expect(frRules.map((rule) => rule.expression)).to.not.include("\\bsimmer\\b"); + it("is case- and accent-insensitive", async () => { + expect(await techStepClassifier.matchTechSteps("FAIRE MIJOTER", "fr")).to.deep.equal([ + simmerId, + ]); + }); - const deRules = await loadTechStepMappingRules("de"); - expect(deRules).to.deep.equal([ - { techStepId: simmer.id, expression: "\\bsimmer\\b", weight: 15 }, - ]); + it("returns an empty sequence when nothing matches", async () => { + expect( + await techStepClassifier.matchTechSteps("Ranger les couverts dans le tiroir", "fr"), + ).to.deep.equal([]); + }); + + it("returns an empty sequence for an empty description", async () => { + expect(await techStepClassifier.matchTechSteps("", "fr")).to.deep.equal([]); + }); + + it("returns an empty sequence for a locale nothing was trained on", async () => { + expect( + await techStepClassifier.matchTechSteps("Faire mijoter à feu doux", "de"), + ).to.deep.equal([]); + }); + + it("detects several distinct techniques in one step, in reading order", async () => { + expect( + await techStepClassifier.matchTechSteps( + "Préchauffer la poêle, puis faire fondre le beurre", + "fr", + ), + ).to.deep.equal([preheatId, meltId]); + }); + + it("reverses the sequence when the techniques are mentioned in the opposite order", async () => { + expect( + await techStepClassifier.matchTechSteps( + "Faire fondre le beurre puis préchauffer le four", + "fr", + ), + ).to.deep.equal([meltId, preheatId]); + }); + + it("still matches the generic technique on its own when the more specific one isn't implied", async () => { + expect( + await techStepClassifier.matchTechSteps("Faire cuire à feu moyen", "fr"), + ).to.deep.equal([cookId]); + }); + + it("resolves the more specific technique when a generic one's own vocabulary is embedded in it", async () => { + // "Cuire au four" literally contains "cuire" (the generic `cook` + // verb) but means the more specific `bake` — the classifier (not + // a weight table) is what has to get this right now. + expect( + await techStepClassifier.matchTechSteps("Cuire au four pendant 30 minutes", "fr"), + ).to.deep.equal([bakeId]); + }); + + it("understands a technique described without ever naming it — the whole point of moving off pure keyword matching", async () => { + // No literal "fondre"/"fondu" anywhere in this sentence, yet it + // unambiguously means `melt` — this is the exact motivating case + // (see this module's own doc comment) a regex could never catch. + expect( + await techStepClassifier.matchTechSteps( + "jusqu'à ce que le beurre ait disparu dans la poêle", + "fr", + ), + ).to.deep.equal([meltId]); + }); + + it("understands preheating described without the verb 'préchauffer'", async () => { + expect( + await techStepClassifier.matchTechSteps("mettre la poêle sur feu vif", "fr"), + ).to.deep.equal([preheatId]); + }); + + it("matches English text against the English-trained vocabulary", async () => { + expect( + await techStepClassifier.matchTechSteps( + "Bring a large saucepan of salted water to the boil", + "en", + ), + ).to.deep.equal([boilId]); + }); }); - it("returns an empty list for a locale with no mappings at all", async () => { - expect(await loadTechStepMappingRules("de")).to.deep.equal([]); + describe("matchTechStepSpans", () => { + it("returns a tight span around the anchor word for a simple match", async () => { + const text = "Faire mijoter à feu doux"; + const result = await techStepClassifier.matchTechStepSpans(text, "fr"); + expect(result).to.deep.equal([{ techStepId: simmerId, start: 6, end: 13 }]); + expect(text.slice(6, 13).toLowerCase()).to.equal("mijoter"); + }); + + it("returns an empty list when nothing matches", async () => { + expect( + await techStepClassifier.matchTechStepSpans("Ranger les couverts dans le tiroir", "fr"), + ).to.deep.equal([]); + }); + + it("returns each distinct technique's own tight span, in reading order", async () => { + const text = "Préchauffer la poêle, puis faire fondre le beurre"; + const result = await techStepClassifier.matchTechStepSpans(text, "fr"); + + expect(result).to.have.length(2); + expect(result[0].techStepId).to.equal(preheatId); + expect(result[1].techStepId).to.equal(meltId); + // Each span, sliced back out of the original text, is exactly the + // word(s) that anchored that match — what the frontend needs to + // highlight the right characters. + expect(text.slice(result[0].start, result[0].end).toLowerCase()).to.equal("préchauffer"); + expect(text.slice(result[1].start, result[1].end).toLowerCase()).to.equal("faire fondre"); + }); + + it("falls back to highlighting the whole clause when a technique was found with no literal anchor word", async () => { + const text = "jusqu'à ce que le beurre ait disparu dans la poêle"; + const result = await techStepClassifier.matchTechStepSpans(text, "fr"); + expect(result).to.deep.equal([{ techStepId: meltId, start: 0, end: text.length }]); + }); + + it("chop matches English text against the English-trained vocabulary, tight span", async () => { + const text = "Chop the onions finely"; + const result = await techStepClassifier.matchTechStepSpans(text, "en"); + expect(result).to.deep.equal([{ techStepId: chopId, start: 0, end: 4 }]); + expect(text.slice(0, 4)).to.equal("Chop"); + }); }); }); }); diff --git a/apps/api/test/reference.test.ts b/apps/api/test/reference.test.ts index 05de918..a2a1427 100644 --- a/apps/api/test/reference.test.ts +++ b/apps/api/test/reference.test.ts @@ -149,15 +149,13 @@ describe("Reference data", () => { expect(keys).to.deep.equal([...keys].sort()); }); - it("reseeding is idempotent — no duplicate techniques or mappings", async () => { + it("reseeding is idempotent — no duplicate techniques", async () => { // resetDatabase already seeded once in beforeEach; seed a second time // on top of that without truncating, the way a redeploy would. await seedReferenceData(prisma); const res = await request(app).get("/reference/tech-steps"); expect(res.body).to.have.length(26); - // 26 techniques × one "fr" + one "en" mapping each. - expect(await prisma.techStepMapping.count()).to.equal(52); }); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c06548d..d1f9569 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -44,6 +44,9 @@ importers: jsonwebtoken: specifier: ^9.0.3 version: 9.0.3 + node-nlp: + specifier: 4.27.0 + version: 4.27.0 prisma: specifier: ^5.22.0 version: 5.22.0 @@ -850,12 +853,224 @@ packages: resolution: {integrity: sha512-Yhlar6v9WQgUp/He7BdgzOz8lqMQ8sU+jkCq7Wx8Myc5YFJLbEe7lgui/V7G1qB1DJykHSGwreceSaD60Y0PUQ==, tarball: https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.11.tgz} hasBin: true + '@microsoft/recognizers-text-choice@1.3.1': + resolution: {integrity: sha512-HubunMJVq/OetmdvcAmBh5skMlg+yiScm3V2wNyNZIVvLgli4+8nzbg/W/fI9dpaf6wv9ZQ7d2IYvn8swJBo3A==, tarball: https://registry.npmjs.org/@microsoft/recognizers-text-choice/-/recognizers-text-choice-1.3.1.tgz} + engines: {node: '>=10.3.0'} + + '@microsoft/recognizers-text-data-types-timex-expression@1.3.1': + resolution: {integrity: sha512-jarJIFIJZBqeofy3hh0vdQo1yOmTM+jCjj6/zmo9JunsQ6LO750eZHCg9eLptQhsvq321XCt5xdRNLCwU8YeNA==, tarball: https://registry.npmjs.org/@microsoft/recognizers-text-data-types-timex-expression/-/recognizers-text-data-types-timex-expression-1.3.1.tgz} + engines: {node: '>=10.3.0'} + + '@microsoft/recognizers-text-date-time@1.3.2': + resolution: {integrity: sha512-fUEGOTccS55ZY0erzjS1bunJYA9lGXjcZoru5oPOlnxbJS4Lk0ylgdH2Ub2EjAyqr8DIJhdLNOEesCdAXMvlNg==, tarball: https://registry.npmjs.org/@microsoft/recognizers-text-date-time/-/recognizers-text-date-time-1.3.2.tgz} + engines: {node: '>=10.3.0'} + + '@microsoft/recognizers-text-number-with-unit@1.3.1': + resolution: {integrity: sha512-gzCpPP4zQ5Vb+RHaWjzP2t1c+mj6GYOsFoI2NyJkm8OZ52XI+x9SJCgrrD2ujzjOd5/CQVC46rE22rfGwXLDkA==, tarball: https://registry.npmjs.org/@microsoft/recognizers-text-number-with-unit/-/recognizers-text-number-with-unit-1.3.1.tgz} + engines: {node: '>=10.3.0'} + + '@microsoft/recognizers-text-number@1.3.1': + resolution: {integrity: sha512-JBxhSdihdQLQilCtqISEBw5kM+CNGTXzy5j5hNoZECNUEvBUPkAGNEJAeQPMP5abrYks29aSklnSvSyLObXaNQ==, tarball: https://registry.npmjs.org/@microsoft/recognizers-text-number/-/recognizers-text-number-1.3.1.tgz} + engines: {node: '>=10.3.0'} + + '@microsoft/recognizers-text-sequence@1.3.1': + resolution: {integrity: sha512-J7Kg35hpm0NcFHmu69Bb4q7DPDiSpCd8ApUZqNm59itIjrQJHpSdl9HF6JxuQQz0Ftc/li5ZLqSuupJAmA/sgg==, tarball: https://registry.npmjs.org/@microsoft/recognizers-text-sequence/-/recognizers-text-sequence-1.3.1.tgz} + engines: {node: '>=10.3.0'} + + '@microsoft/recognizers-text-suite@1.3.0': + resolution: {integrity: sha512-uqG4vzy5N2CmBaeINny0bLdnGp0jDbT1moNoLC+Yim3G8kHOU9lpDfwA6VN6HTYaDM5854SNMEzLjJdS1TPFTw==, tarball: https://registry.npmjs.org/@microsoft/recognizers-text-suite/-/recognizers-text-suite-1.3.0.tgz} + engines: {node: '>=10.3.0'} + + '@microsoft/recognizers-text@1.3.1': + resolution: {integrity: sha512-HikLoRUgSzM4OKP3JVBzUUp3Q7L4wgI17p/3rERF01HVmopcujY3i6wgx8PenCwbenyTNxjr1AwSDSVuFlYedQ==, tarball: https://registry.npmjs.org/@microsoft/recognizers-text/-/recognizers-text-1.3.1.tgz} + engines: {node: '>=10.3.0'} + '@napi-rs/lzma-linux-x64-gnu@1.5.1': resolution: {integrity: sha512-oTXEIha4SsuXdTA4Iyskj0kpdx2yVXdhd75c2v3xGrHFfVMsbhTPZU/nMPL4sWKo4pBHm3aucLaqGlF696dTyQ==, tarball: https://registry.npmjs.org/@napi-rs/lzma-linux-x64-gnu/-/lzma-linux-x64-gnu-1.5.1.tgz} engines: {node: ^22.20 || ^24.12 || >=25} cpu: [x64] os: [linux] + '@nlpjs/builtin-duckling@4.26.1': + resolution: {integrity: sha512-3qkH955X2g5MXV1EqT3fTAT/lLEdiqqe5IgBDyr+MQB7FOV9R3YhqGIn3DFOl+TSm/tP5n/BAEptkTNn/TOpmQ==, tarball: https://registry.npmjs.org/@nlpjs/builtin-duckling/-/builtin-duckling-4.26.1.tgz} + + '@nlpjs/builtin-microsoft@4.26.1': + resolution: {integrity: sha512-AODgzTcfYUf5Ozm00aQnHImDum7Idtl0F9dSPoaXpfj7rZqP8hPZ7iWwdGTAvISH/da2YhjPOU65QSYk2YpjFA==, tarball: https://registry.npmjs.org/@nlpjs/builtin-microsoft/-/builtin-microsoft-4.26.1.tgz} + + '@nlpjs/core-loader@4.26.1': + resolution: {integrity: sha512-IiRtn65bdiUSQHy2kusco2fmhk39u2Mc2c5Fsm9+9EVG6BtJCmVEFU/btAzGDAmxEA/E4qKecaAT4LvcW6TPbA==, tarball: https://registry.npmjs.org/@nlpjs/core-loader/-/core-loader-4.26.1.tgz} + + '@nlpjs/core@4.26.1': + resolution: {integrity: sha512-M/PeFddsi3y7Z1piFJxsLGm5/xdMhcrpOsml7s6CTEgYo8iduaT30HDd61tZxDyvvJseU6uFqlXSn7XKkAcC1g==, tarball: https://registry.npmjs.org/@nlpjs/core/-/core-4.26.1.tgz} + + '@nlpjs/emoji@4.26.1': + resolution: {integrity: sha512-Q0PoXwIvaB1bnRXK4U/YD7mrqaz29Yfed3s2au0iXl1bffUgoG+hs4GORCvyy7DFCCLlc9d5yDM3oLIX/ggZ+Q==, tarball: https://registry.npmjs.org/@nlpjs/emoji/-/emoji-4.26.1.tgz} + + '@nlpjs/evaluator@4.26.1': + resolution: {integrity: sha512-WeUrC8qq7+V8Jhkkjc2yiXdzy9V0wbETv8/qasQmL0QmEuwBDJF+fvfl4z2vWpBb0vW07A8aNrFElKELzbpkdg==, tarball: https://registry.npmjs.org/@nlpjs/evaluator/-/evaluator-4.26.1.tgz} + + '@nlpjs/lang-all@4.26.1': + resolution: {integrity: sha512-UzRm1JRRAyQqilEOxQ2ySMOitKbhPk5iKYbjD8FREDcPjreUvDxVuQsYUOvYucmEyFcZU2U/TdJx+fX9/bcaKQ==, tarball: https://registry.npmjs.org/@nlpjs/lang-all/-/lang-all-4.26.1.tgz} + + '@nlpjs/lang-ar@4.26.1': + resolution: {integrity: sha512-MUlVtabt9ltG7WyzCQpFJymLJlnEqp3mxhgN9JHyFH7oZMK3REvMovFfvEUAbfiYrJEv/BN5KKLL7yrvUeaHtg==, tarball: https://registry.npmjs.org/@nlpjs/lang-ar/-/lang-ar-4.26.1.tgz} + + '@nlpjs/lang-bn@4.26.1': + resolution: {integrity: sha512-sim1iZKBDdehi/yBUKrLW51QvS9uB+sXW7lj+THVqBy5UsnEQvt4gzE0NsC873uJMh66vt2AlHkhzgPH0qH/nQ==, tarball: https://registry.npmjs.org/@nlpjs/lang-bn/-/lang-bn-4.26.1.tgz} + + '@nlpjs/lang-ca@4.26.1': + resolution: {integrity: sha512-fD4R5tcAB0uYtNxSEF20b1KmF6nUQSbiJqrIUJI5yis4ObjCYRQnSh4bjVDKUKxyONjbD6L8EaK5GrY1/jkwFQ==, tarball: https://registry.npmjs.org/@nlpjs/lang-ca/-/lang-ca-4.26.1.tgz} + + '@nlpjs/lang-cs@4.26.1': + resolution: {integrity: sha512-CqI6VB8toaJ/MlP1D4K9BctA6GpZJhMKyEy+OX9xavDe4r4ao/SxlSaIYK3izK0k+J38lJWC5lXYGazfCdTGjA==, tarball: https://registry.npmjs.org/@nlpjs/lang-cs/-/lang-cs-4.26.1.tgz} + + '@nlpjs/lang-da@4.26.1': + resolution: {integrity: sha512-krI/ojeDSi329ENM/hLIsbUh1x4XRTKAbtPcbFxAY6XVhcSVoWPO7L77jFTL1NQeE1oGRFzGHaeC9hZJ8phVbA==, tarball: https://registry.npmjs.org/@nlpjs/lang-da/-/lang-da-4.26.1.tgz} + + '@nlpjs/lang-de@4.26.1': + resolution: {integrity: sha512-HfZQwsE5FICq9taVZDiyktmdAePVF5948NM80et0d9mx43RWDFhHKQYgtJPwfQXtdCoQtOM5TOJ2FanGwzPeaA==, tarball: https://registry.npmjs.org/@nlpjs/lang-de/-/lang-de-4.26.1.tgz} + + '@nlpjs/lang-el@4.26.1': + resolution: {integrity: sha512-pcOvuSwPCXxI+2xNZZzM4V5pTRDntYoJi0SP/ic2nV4IPQ0nU2j16dYfg1HlvET/E6iN1VTqghrCaf10SMkDGA==, tarball: https://registry.npmjs.org/@nlpjs/lang-el/-/lang-el-4.26.1.tgz} + + '@nlpjs/lang-en-min@4.26.1': + resolution: {integrity: sha512-1sJZ7dy7ysqzbsB8IklguvB88J8EPIv4XGVkZCcwecKtOw+fp5LAsZ3TJVmEf18iK1gD4cEGr7qZg5fpPxTpWQ==, tarball: https://registry.npmjs.org/@nlpjs/lang-en-min/-/lang-en-min-4.26.1.tgz} + + '@nlpjs/lang-en@4.26.1': + resolution: {integrity: sha512-GVoJpOjyk5TtBAqo/fxsiuuH7jXycyakGT0gw5f01u9lOmUnpJegvXyGff/Nb0j14pXcGHXOhmpWrcTrG2B0LQ==, tarball: https://registry.npmjs.org/@nlpjs/lang-en/-/lang-en-4.26.1.tgz} + + '@nlpjs/lang-es@4.26.1': + resolution: {integrity: sha512-fIPQt+WPcNdyxZOCMkOPlMb4Y1iE585QxjB9IAdFz8ZtVg7mc4dlv5f46ud7ppdMh84iLOuOdo6pzu2Cqm14lw==, tarball: https://registry.npmjs.org/@nlpjs/lang-es/-/lang-es-4.26.1.tgz} + + '@nlpjs/lang-eu@4.26.1': + resolution: {integrity: sha512-Ha8GHTbgQYd7dwHM8aWHDyxmbUNUcyu/5xlBKqqBOPxysDyZ6Ad0tvj0FmJBy6mYhqmFTPBnEAo69cfuFSqWIQ==, tarball: https://registry.npmjs.org/@nlpjs/lang-eu/-/lang-eu-4.26.1.tgz} + + '@nlpjs/lang-fa@4.26.1': + resolution: {integrity: sha512-qJCmNXgJZnfNXUnKnxvEGEzSFBdQT4XU7/rMxuFmSJqmQY7fH/Vsmi5CKF94VRBPOIV4ULlEJuLpUWHXRmOnVQ==, tarball: https://registry.npmjs.org/@nlpjs/lang-fa/-/lang-fa-4.26.1.tgz} + + '@nlpjs/lang-fi@4.26.1': + resolution: {integrity: sha512-W/rUcrzSh3KE07q2vOsssTpU1sbX32gbBzKPZfRJ2ZUF4afO+eHxmAywikXubP4kiU3JxVNLvXXEjuGD3SBUbA==, tarball: https://registry.npmjs.org/@nlpjs/lang-fi/-/lang-fi-4.26.1.tgz} + + '@nlpjs/lang-fr@4.26.1': + resolution: {integrity: sha512-LTA852atCJnHtKDmtjx/ui5AnvEIkrPx+MJQ2mB3gn8ko6i2UITnJgPmJE9Kej5bLasVZOAJvU/SrfXEmnPGOw==, tarball: https://registry.npmjs.org/@nlpjs/lang-fr/-/lang-fr-4.26.1.tgz} + + '@nlpjs/lang-ga@4.26.1': + resolution: {integrity: sha512-JsP1CZ8r3Jd6o/Az7cN3exz0HDP3FNYLzh4Vi6ksEkdKF0yCjJ9G5dXZYqS9qFIN5ffemWn29G4WRELY6QH/cQ==, tarball: https://registry.npmjs.org/@nlpjs/lang-ga/-/lang-ga-4.26.1.tgz} + + '@nlpjs/lang-gl@4.26.1': + resolution: {integrity: sha512-y1NNu6NVy/6o5UNfihgg0WkSlVr4IvKA5W193CpRLZWS4FccQDmnFFhyYWRkshyDbgEsfsZ0Rs3BoE82+T2Ubg==, tarball: https://registry.npmjs.org/@nlpjs/lang-gl/-/lang-gl-4.26.1.tgz} + + '@nlpjs/lang-hi@4.26.1': + resolution: {integrity: sha512-Fw9rXqF5l8q9etJG5uOlEFpnMVjQEWMaCIgQfEcA1yTvieSV8mpoSvQkEZl+DFhww+azareoJ7ZCkx0gJ9UDuQ==, tarball: https://registry.npmjs.org/@nlpjs/lang-hi/-/lang-hi-4.26.1.tgz} + + '@nlpjs/lang-hu@4.26.1': + resolution: {integrity: sha512-7dPUn5/ZpLZmsdRwO+dtORuMIiIpnsWbgSLIKdOLh8irhgUR+M2bYTfkdnKcrEcHzHPP8Svn7pU0xk7OKSUA1w==, tarball: https://registry.npmjs.org/@nlpjs/lang-hu/-/lang-hu-4.26.1.tgz} + + '@nlpjs/lang-hy@4.26.1': + resolution: {integrity: sha512-T2brpLGDJryAwWmjtnmY8Ot6ZUkCz+/nRR9/QM1PybvZIqOVLjJqA49bqjJfT5DMN89HbwC7I/15NTT0y09i1Q==, tarball: https://registry.npmjs.org/@nlpjs/lang-hy/-/lang-hy-4.26.1.tgz} + + '@nlpjs/lang-id@4.26.1': + resolution: {integrity: sha512-rVuIkYFKdltFhMT/a2ZxD9ovoZSVZF7OPuqYjTXW9xKd3Ff32yUrzcf/pHXlqmZOSltqOH3E5jZRRDkHvgUOjQ==, tarball: https://registry.npmjs.org/@nlpjs/lang-id/-/lang-id-4.26.1.tgz} + + '@nlpjs/lang-it@4.26.1': + resolution: {integrity: sha512-BZA3QnfQGW91gYaybRmHnCAPBvQggtmHZJrAmuBZUKUS12HoQm8uybjw2fZO+vahEeUQceKNDISRcT1eLLijog==, tarball: https://registry.npmjs.org/@nlpjs/lang-it/-/lang-it-4.26.1.tgz} + + '@nlpjs/lang-ja@4.26.1': + resolution: {integrity: sha512-QgkuJOkHguRFyfnckH2It5/Kg8zecnOMJsHxYeuDC4tBF7jL/5xqWis+679lYLsXtAkrG8+fjVcBbjyopP0KHg==, tarball: https://registry.npmjs.org/@nlpjs/lang-ja/-/lang-ja-4.26.1.tgz} + + '@nlpjs/lang-ko@4.26.1': + resolution: {integrity: sha512-Q0N8bLJJ829ILWCKH1UQWPSNyuLaEURAXCawkDju4pt33DBLcpqz9IzO9dnqiFc+fjSgVzZ7WMaLT18hXZQ9vg==, tarball: https://registry.npmjs.org/@nlpjs/lang-ko/-/lang-ko-4.26.1.tgz} + + '@nlpjs/lang-lt@4.26.1': + resolution: {integrity: sha512-SeYZxRhdCy+ClQNnF/u0MAtcDui/ocdk4NtgNOCuwNTNuzhN3t3rfGeArfBGmZeg1SIeBLUDE9dsTxYCv5AOEg==, tarball: https://registry.npmjs.org/@nlpjs/lang-lt/-/lang-lt-4.26.1.tgz} + + '@nlpjs/lang-ms@4.26.1': + resolution: {integrity: sha512-KxWBS+tFY2U8z9UrjQIqMM40npGDOskP5DcWhaEE3zuhzf3RTDYjy8sdz34jVd0fBdbPihX133h3bFibg2Cm7w==, tarball: https://registry.npmjs.org/@nlpjs/lang-ms/-/lang-ms-4.26.1.tgz} + + '@nlpjs/lang-ne@4.26.1': + resolution: {integrity: sha512-K3E2l+0LTESv+dO+ZTIdvNa+zwMJvvnMiFYYkKvJst6lhc8JgvGOsPxGsjJn6PDhI3wyfQu+dg3b+bnVPu4FDA==, tarball: https://registry.npmjs.org/@nlpjs/lang-ne/-/lang-ne-4.26.1.tgz} + + '@nlpjs/lang-nl@4.26.1': + resolution: {integrity: sha512-I/mP1RRbUN4BQ+8NXAl2FKaLHbb7f6S8JVjxHQ0sKHT4BgQ3+r0yO+DVcEsHg+vWRiY1Fyzh0gq0PhLVnF6HnA==, tarball: https://registry.npmjs.org/@nlpjs/lang-nl/-/lang-nl-4.26.1.tgz} + + '@nlpjs/lang-no@4.26.1': + resolution: {integrity: sha512-a0CLL2c/OCzbg7J7ugyrsAksI96XhkQ3IeBbbx60o5o/9wsFNik6cPWrkpoE5xNtw7gLlAJWabwDiZXkl8Zrcw==, tarball: https://registry.npmjs.org/@nlpjs/lang-no/-/lang-no-4.26.1.tgz} + + '@nlpjs/lang-pl@4.26.1': + resolution: {integrity: sha512-nrDXlq+TzQLE5IpXPIlFMzd8OpquvApWsouh6fmLsD9HZLZI4O3w1M4sXXLzE+9Ggu9Cy1m1QJ0/i7XCcv115g==, tarball: https://registry.npmjs.org/@nlpjs/lang-pl/-/lang-pl-4.26.1.tgz} + + '@nlpjs/lang-pt@4.26.1': + resolution: {integrity: sha512-p6yZHaJ0e+n0avMHpdDw5PMk4HkKXjPbOMbrlg0dF+VRqChjxfH478Q423rDyzu/4MzDsIYB+p6KzL9AARKXpg==, tarball: https://registry.npmjs.org/@nlpjs/lang-pt/-/lang-pt-4.26.1.tgz} + + '@nlpjs/lang-ro@4.26.1': + resolution: {integrity: sha512-baUdTA0DWpDR0Tn6fxo+RDN/6gbuINLCARtHwap2UR/HKQWP2XoH/DIvcjZpwUTalr5MQjso31epcdeRRapczA==, tarball: https://registry.npmjs.org/@nlpjs/lang-ro/-/lang-ro-4.26.1.tgz} + + '@nlpjs/lang-ru@4.26.1': + resolution: {integrity: sha512-NaZ2DAOGxWG2Us9IyIDs3m6vhGpUaUJRVgzzHHyX3LO3xEYjZmtnA0jEpBaTOe2PuNHThv0WCZUNn9BSurV3PA==, tarball: https://registry.npmjs.org/@nlpjs/lang-ru/-/lang-ru-4.26.1.tgz} + + '@nlpjs/lang-sl@4.26.1': + resolution: {integrity: sha512-QBJwcJt+oKUpAnHKNJkLkx9Xm1n4dUPC5GPYfAXTnJZf0hNWJSY21GicdWi7Vu/qFJ3ghIqtSP8D7KIPLnibNw==, tarball: https://registry.npmjs.org/@nlpjs/lang-sl/-/lang-sl-4.26.1.tgz} + + '@nlpjs/lang-sr@4.26.1': + resolution: {integrity: sha512-drH3+UqTW637uLWsnLrcp8jEKUGxV61ZgCBjNkVQNEv1/jbpSg6IqgynSY2JyhtnlV0f870KS0HvSbyo5AD4Ng==, tarball: https://registry.npmjs.org/@nlpjs/lang-sr/-/lang-sr-4.26.1.tgz} + + '@nlpjs/lang-sv@4.26.1': + resolution: {integrity: sha512-2axkrYFC02tAlxCWeiEKISbe4dSteciP1CIggO/dZglnnLWgdF+g7kOeYMn7abCfFVSnh5vLqfDkrwnyIqt7Ag==, tarball: https://registry.npmjs.org/@nlpjs/lang-sv/-/lang-sv-4.26.1.tgz} + + '@nlpjs/lang-ta@4.26.1': + resolution: {integrity: sha512-keeh+croa1TAirV9Fd3OQMo5IkAlTGNWTNweHbi/htYMX0MKOPYxyqg+VH2bml+57VY2aUj/WYgV/p3ATx9EfQ==, tarball: https://registry.npmjs.org/@nlpjs/lang-ta/-/lang-ta-4.26.1.tgz} + + '@nlpjs/lang-th@4.26.1': + resolution: {integrity: sha512-2SWZhrln3rMw8/DsRc9yS5bi3qEdGfw2pq9Uejx/UYED5zvvL6kh9AiCJZT4k0wMBGEwWUV6HxJ0Pq/jOTHogg==, tarball: https://registry.npmjs.org/@nlpjs/lang-th/-/lang-th-4.26.1.tgz} + + '@nlpjs/lang-tl@4.26.1': + resolution: {integrity: sha512-AzmLtg28tm0VXCm0Q0EY3OtA3m4oYxaqh4VX6uhB4J+PoEsIkm0py12SJxMNIsh/r98pobCumH8KH9bvHQoCAg==, tarball: https://registry.npmjs.org/@nlpjs/lang-tl/-/lang-tl-4.26.1.tgz} + + '@nlpjs/lang-tr@4.26.1': + resolution: {integrity: sha512-p30uuXvE9pZeU/5XkrQfvxRgiAOBmP3EyBFGV/+P05PEogaqbsmmtVCgCnR63yeRvVnGbToPBPjRK3OO1y4AEQ==, tarball: https://registry.npmjs.org/@nlpjs/lang-tr/-/lang-tr-4.26.1.tgz} + + '@nlpjs/lang-uk@4.26.1': + resolution: {integrity: sha512-PVEvmlhvl6BL3e/Q4qjMPsnwON3cWEYvDh9dg+Si+sjD2Edu9tajolJKcQ6ZA4I8dXrld5xuXx+DEBH/uB4uWQ==, tarball: https://registry.npmjs.org/@nlpjs/lang-uk/-/lang-uk-4.26.1.tgz} + + '@nlpjs/lang-zh@4.26.1': + resolution: {integrity: sha512-kwqeqeEgMAMvucVX9HNE1p6s/2APP23ZsS8Um/lNvtswb4gL5jjYF9kyCvRfqlPBQSWWdRv7wwcnNXOvXYkxcQ==, tarball: https://registry.npmjs.org/@nlpjs/lang-zh/-/lang-zh-4.26.1.tgz} + + '@nlpjs/language-min@4.25.0': + resolution: {integrity: sha512-g8jtbDbqtRm+dlD/1Vnb4VWfKbKteApEGVTqIMxYkk6N/HMhvLZ5J2svrxzrB98a/HZ0fb//YBfFgymnz9Oukg==, tarball: https://registry.npmjs.org/@nlpjs/language-min/-/language-min-4.25.0.tgz} + + '@nlpjs/language@4.25.0': + resolution: {integrity: sha512-tUF6QENoUQ/E26RYc32IgsttStSF9cNO4ySN+BQECn8VpjukWdwbMw073MlOLXzjfeobxa+3hCVrmPPcW+V3UA==, tarball: https://registry.npmjs.org/@nlpjs/language/-/language-4.25.0.tgz} + + '@nlpjs/ner@4.27.0': + resolution: {integrity: sha512-ptwkxriJdmgHSH9TfP10JQ1jviaSl2SupSFGUvTuWkuJhobQd3hbnlSq40V6XYvJNmqh9M9zEab/AKeghxYOTA==, tarball: https://registry.npmjs.org/@nlpjs/ner/-/ner-4.27.0.tgz} + + '@nlpjs/neural@4.25.0': + resolution: {integrity: sha512-Oz20denGiBe0DlQsS7lN4TNrATN1nXlHKc/HB6jJPegjVmgJVCugDaHwIGoV7qOWyA6F2fRRwOgD+quNT2gVpg==, tarball: https://registry.npmjs.org/@nlpjs/neural/-/neural-4.25.0.tgz} + + '@nlpjs/nlg@4.26.1': + resolution: {integrity: sha512-PCJWiZ7464ChXXUGvjBZIFtoqkC24Oy6X63HgQrSv+63svz22Y5Cmu1MYLk77Nb+4keWv+hKhFJKDkvJoOpBVg==, tarball: https://registry.npmjs.org/@nlpjs/nlg/-/nlg-4.26.1.tgz} + + '@nlpjs/nlp@4.27.0': + resolution: {integrity: sha512-q6X7sY6TYVnQRZJKF/6mfLFlNA5oRYLhgQ5k3i1IBqH9lbWTAZJr31w/dCf97HXaYaj+vJp3h0ucfNumme9EIw==, tarball: https://registry.npmjs.org/@nlpjs/nlp/-/nlp-4.27.0.tgz} + + '@nlpjs/nlu@4.27.0': + resolution: {integrity: sha512-j4DUdoXS/y/Xag6ysYXx7Ve8NBmUVViUSCJhj3r49+zGyYtyVAHuVcqSej5q0tJjn0JSMT+6+ip8klON1q8ixw==, tarball: https://registry.npmjs.org/@nlpjs/nlu/-/nlu-4.27.0.tgz} + + '@nlpjs/request@4.25.0': + resolution: {integrity: sha512-MPVYWfFZY03WyFL7GWkUkv8tw968OXsdxFSJEvjXHzhiCe/vAlPCWbvoR+VnoQTgzLHxs/KIF6sIF2s9AzsLmQ==, tarball: https://registry.npmjs.org/@nlpjs/request/-/request-4.25.0.tgz} + + '@nlpjs/sentiment@4.26.1': + resolution: {integrity: sha512-U2WmcW3w6yDDO45+Y7v5e6DPQj8e0x+RUUePPyRu2uIZmUtIKG+qCPMWnNLMmYQZoSQEFxmMMlLcGDC7tN7o3w==, tarball: https://registry.npmjs.org/@nlpjs/sentiment/-/sentiment-4.26.1.tgz} + + '@nlpjs/similarity@4.26.1': + resolution: {integrity: sha512-QutSBFGo/huNuz60PgqCjub0oBd9S8MLrjme33U5GzxuSvToQzXtn9/ynIia8qDm009D09VXV+LPeNE4h7yuSg==, tarball: https://registry.npmjs.org/@nlpjs/similarity/-/similarity-4.26.1.tgz} + + '@nlpjs/slot@4.26.1': + resolution: {integrity: sha512-mK8EEy5O+mRGne822PIKMxHSFh8j+iC7hGJ6T31XdFsNhFEYXLI/0dmeBstZgTSKBTe27HNFgCCwuGb77u0o9w==, tarball: https://registry.npmjs.org/@nlpjs/slot/-/slot-4.26.1.tgz} + + '@nlpjs/xtables@4.25.0': + resolution: {integrity: sha512-+baCtMZIp+aDqODLQs8Wyyke5qUqQkL8AGWsZzwYuJV8S7xdW2+XklRnHnkFc3p3foC248TkzG5L8j9r6INOtg==, tarball: https://registry.npmjs.org/@nlpjs/xtables/-/xtables-4.25.0.tgz} + '@noble/hashes@1.8.0': resolution: {integrity: sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==, tarball: https://registry.npmjs.org/@noble/hashes/-/hashes-1.8.0.tgz} engines: {node: ^14.21.3 || >=16} @@ -1118,6 +1333,10 @@ packages: resolution: {integrity: sha512-ID7fosbc50TbT0MK0EG12O+gAP3W3Aa/Pz4DaTtQtEvlc9Odaqi0de+xuZ7Li2GtK4HzEX7IuRWS/JmZLksR3Q==, tarball: https://registry.npmjs.org/@teppeis/multimaps/-/multimaps-3.0.0.tgz} engines: {node: '>=14'} + '@tootallnate/once@2.0.1': + resolution: {integrity: sha512-HqmEUIGRJ5fSXchkVgR5F7qn48bDBzv0kWj/Kfu5e6uci4UlEeng4331LnBkWffb++Ei3FOVLxo8JJWMFBDMeQ==, tarball: https://registry.npmjs.org/@tootallnate/once/-/once-2.0.1.tgz} + engines: {node: '>= 10'} + '@types/babel__core@7.20.5': resolution: {integrity: sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==, tarball: https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz} @@ -1287,6 +1506,10 @@ packages: engines: {node: '>=0.4.0'} hasBin: true + adler-32@1.3.1: + resolution: {integrity: sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A==, tarball: https://registry.npmjs.org/adler-32/-/adler-32-1.3.1.tgz} + engines: {node: '>=0.8'} + agent-base@6.0.2: resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==, tarball: https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz} engines: {node: '>= 6.0.0'} @@ -1390,6 +1613,9 @@ packages: resolution: {integrity: sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==, tarball: https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz} engines: {node: '>=8'} + async@2.6.4: + resolution: {integrity: sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==, tarball: https://registry.npmjs.org/async/-/async-2.6.4.tgz} + async@3.2.6: resolution: {integrity: sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==, tarball: https://registry.npmjs.org/async/-/async-3.2.6.tgz} @@ -1431,6 +1657,9 @@ packages: bcrypt-pbkdf@1.0.2: resolution: {integrity: sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==, tarball: https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz} + bignumber.js@7.2.1: + resolution: {integrity: sha512-S4XzBk5sMB+Rcb/LNcpzXr57VRTxgAvaAEDAl1AwRx27j00hT84O6OkteE7u8UB3NuaaygCRrEpqox4uDOrbdQ==, tarball: https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz} + binary-extensions@2.3.0: resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==, tarball: https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz} engines: {node: '>=8'} @@ -1519,6 +1748,10 @@ packages: caseless@0.12.0: resolution: {integrity: sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==, tarball: https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz} + cfb@1.2.2: + resolution: {integrity: sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA==, tarball: https://registry.npmjs.org/cfb/-/cfb-1.2.2.tgz} + engines: {node: '>=0.8'} + chai@5.3.3: resolution: {integrity: sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==, tarball: https://registry.npmjs.org/chai/-/chai-5.3.3.tgz} engines: {node: '>=18'} @@ -1589,6 +1822,10 @@ packages: resolution: {integrity: sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==, tarball: https://registry.npmjs.org/clone/-/clone-1.0.4.tgz} engines: {node: '>=0.8'} + codepage@1.15.0: + resolution: {integrity: sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA==, tarball: https://registry.npmjs.org/codepage/-/codepage-1.15.0.tgz} + engines: {node: '>=0.8'} + color-convert@2.0.1: resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==, tarball: https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz} engines: {node: '>=7.0.0'} @@ -1703,6 +1940,11 @@ packages: typescript: optional: true + crc-32@1.2.2: + resolution: {integrity: sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==, tarball: https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz} + engines: {node: '>=0.8'} + hasBin: true + cross-env@10.1.0: resolution: {integrity: sha512-GsYosgnACZTADcmEyJctkJIoqAhHjttw7RsFrVoJNXbsWWqaq6Ym+7kZjq6mS45O0jij6vtiReppKQEtqWy6Dw==, tarball: https://registry.npmjs.org/cross-env/-/cross-env-10.1.0.tgz} engines: {node: '>=20'} @@ -1890,6 +2132,9 @@ packages: resolution: {integrity: sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==, tarball: https://registry.npmjs.org/dotenv/-/dotenv-16.6.1.tgz} engines: {node: '>=12'} + doublearray@0.0.2: + resolution: {integrity: sha512-aw55FtZzT6AmiamEj2kvmR6BuFqvYgKZUkfQ7teqVRNqD5UE0rw8IeW/3gieHNKQ5sPuDKlljWEn4bzv5+1bHw==, tarball: https://registry.npmjs.org/doublearray/-/doublearray-0.0.2.tgz} + dunder-proto@1.0.1: resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==, tarball: https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz} engines: {node: '>= 0.4'} @@ -2156,6 +2401,10 @@ packages: resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==, tarball: https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz} engines: {node: '>= 0.6'} + frac@1.1.2: + resolution: {integrity: sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA==, tarball: https://registry.npmjs.org/frac/-/frac-1.1.2.tgz} + engines: {node: '>=0.8'} + fresh@0.5.2: resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==, tarball: https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz} engines: {node: '>= 0.6'} @@ -2261,6 +2510,9 @@ packages: graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==, tarball: https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz} + grapheme-splitter@1.0.4: + resolution: {integrity: sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ==, tarball: https://registry.npmjs.org/grapheme-splitter/-/grapheme-splitter-1.0.4.tgz} + has-ansi@4.0.1: resolution: {integrity: sha512-Qr4RtTm30xvEdqUXbSBVWDu+PrTokJOwe/FU+VdfJPk+MXAPoeOzKpRyrDTnZIJwAkQ4oBLTU53nu0HrkF/Z2A==, tarball: https://registry.npmjs.org/has-ansi/-/has-ansi-4.0.1.tgz} engines: {node: '>=8'} @@ -2306,6 +2558,10 @@ packages: resolution: {integrity: sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==, tarball: https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz} engines: {node: '>= 0.8'} + http-proxy-agent@5.0.0: + resolution: {integrity: sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==, tarball: https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz} + engines: {node: '>= 6'} + http-signature@1.4.0: resolution: {integrity: sha512-G5akfn7eKbpDN+8nPS/cb57YeA1jLTVxjpCj7tmm3QKPdyDy7T+qSC40e9ptydSWvkwjSXw1VbkpyEm39ukeAg==, tarball: https://registry.npmjs.org/http-signature/-/http-signature-1.4.0.tgz} engines: {node: '>=0.10'} @@ -2558,6 +2814,9 @@ packages: knuth-shuffle-seeded@1.0.6: resolution: {integrity: sha512-9pFH0SplrfyKyojCLxZfMcvkhf5hH0d+UwR9nTVJ/DDQJGuzcXjTwB7TP7sDfehSudlGGaOLblmEWqv04ERVWg==, tarball: https://registry.npmjs.org/knuth-shuffle-seeded/-/knuth-shuffle-seeded-1.0.6.tgz} + kuromoji@0.1.2: + resolution: {integrity: sha512-V0dUf+C2LpcPEXhoHLMAop/bOht16Dyr+mDiIE39yX3vqau7p80De/koFqpiTcL1zzdZlc3xuHZ8u5gjYRfFaQ==, tarball: https://registry.npmjs.org/kuromoji/-/kuromoji-0.1.2.tgz} + lazy-ass@1.6.0: resolution: {integrity: sha512-cc8oEVoctTvsFZ/Oje/kGnHbpWHYBe8IAJe4C0QNc3t8uM/0Y8+erSz/7Y1ALuXTEZTMvxXwO6YbX1ey3ujiZw==, tarball: https://registry.npmjs.org/lazy-ass/-/lazy-ass-1.6.0.tgz} engines: {node: '> 0.8'} @@ -2822,6 +3081,9 @@ packages: node-html-parser@5.3.3: resolution: {integrity: sha512-ncg1033CaX9UexbyA7e1N0aAoAYRDiV8jkTvzEnfd1GDvzFdrsXLzR4p4ik8mwLgnaKP/jyUFWDy9q3jvRT2Jw==, tarball: https://registry.npmjs.org/node-html-parser/-/node-html-parser-5.3.3.tgz} + node-nlp@4.27.0: + resolution: {integrity: sha512-LnkhOUPXX0CMFbSzJ1gHI+7Yb3ULLip5gRsqedXb6pryjcRCbNzPgHXcH/6G9B1vSbDfO+y3X2B4QZpfP12OyQ==, tarball: https://registry.npmjs.org/node-nlp/-/node-nlp-4.27.0.tgz} + node-releases@2.0.53: resolution: {integrity: sha512-D9UOmYG3UH1V+ENW56t5QXBwJw1YEY18ruVeus89Rw+SyIgjPkCO84bRzO3uNIYosJbNwiabWVn48o3uJLjxFQ==, tarball: https://registry.npmjs.org/node-releases/-/node-releases-2.0.53.tgz} engines: {node: '>=18'} @@ -3371,6 +3633,10 @@ packages: split@1.0.1: resolution: {integrity: sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==, tarball: https://registry.npmjs.org/split/-/split-1.0.1.tgz} + ssf@0.11.2: + resolution: {integrity: sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g==, tarball: https://registry.npmjs.org/ssf/-/ssf-0.11.2.tgz} + engines: {node: '>=0.8'} + sshpk@1.18.0: resolution: {integrity: sha512-2p2KJZTSqQ/I3+HX42EpYOa2l3f8Erv8MWKsy2I9uf4wA7yFIkXRffYdsx86y6z4vHtV8u7g+pPlr8/4ouAxsQ==, tarball: https://registry.npmjs.org/sshpk/-/sshpk-1.18.0.tgz} engines: {node: '>=0.10.0'} @@ -3711,6 +3977,14 @@ packages: wide-align@1.1.5: resolution: {integrity: sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==, tarball: https://registry.npmjs.org/wide-align/-/wide-align-1.1.5.tgz} + wmf@1.0.2: + resolution: {integrity: sha512-/p9K7bEh0Dj6WbXg4JG0xvLQmIadrner1bi45VMJTfnbVHsc7yIajZyoSoK60/dtVBs12Fm6WkUI5/3WAVsNMw==, tarball: https://registry.npmjs.org/wmf/-/wmf-1.0.2.tgz} + engines: {node: '>=0.8'} + + word@0.3.0: + resolution: {integrity: sha512-OELeY0Q61OXpdUfTp+oweA/vtLVg5VDOXh+3he3PNzLGG/y0oylSOC1xRVj0+l4vQ3tj/bB1HVHv1ocXkQceFA==, tarball: https://registry.npmjs.org/word/-/word-0.3.0.tgz} + engines: {node: '>=0.8'} + workerpool@6.5.1: resolution: {integrity: sha512-Fs4dNYcsdpYSAfVxhnl1L5zTksjvOJxtC5hzMNl+1t9B8hTJTdKDyZ5ju7ztgPy+ft9tBFXoOlDNiOT9WUXZlA==, tarball: https://registry.npmjs.org/workerpool/-/workerpool-6.5.1.tgz} @@ -3732,6 +4006,11 @@ packages: wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==, tarball: https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz} + xlsx@0.18.5: + resolution: {integrity: sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ==, tarball: https://registry.npmjs.org/xlsx/-/xlsx-0.18.5.tgz} + engines: {node: '>=0.8'} + hasBin: true + xmlbuilder@15.1.1: resolution: {integrity: sha512-yMqGBqtXyeN1e3TGYvgNgDVZ3j84W4cwkOXQswghol6APgZWaff9lnbvN7MHYJOiXsvGPXtjTYJEiC9J2wv9Eg==, tarball: https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-15.1.1.tgz} engines: {node: '>=8.0'} @@ -3785,6 +4064,9 @@ packages: yup@1.6.1: resolution: {integrity: sha512-JED8pB50qbA4FOkDol0bYF/p60qSEDQqBD0/qeIrUCG1KbPBIQ776fCUNb9ldbPcSTxA69g/47XTo4TqWiuXOA==, tarball: https://registry.npmjs.org/yup/-/yup-1.6.1.tgz} + zlibjs@0.3.1: + resolution: {integrity: sha512-+J9RrgTKOmlxFSDHo0pI1xM6BLVUv+o0ZT9ANtCxGkjIVCCUdx9alUF8Gm+dGLKbkkkidWIHFDZHDMpfITt4+w==, tarball: https://registry.npmjs.org/zlibjs/-/zlibjs-0.3.1.tgz} + zod@3.25.76: resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==, tarball: https://registry.npmjs.org/zod/-/zod-3.25.76.tgz} @@ -4393,9 +4675,344 @@ snapshots: - encoding - supports-color + '@microsoft/recognizers-text-choice@1.3.1': + dependencies: + '@microsoft/recognizers-text': 1.3.1 + grapheme-splitter: 1.0.4 + + '@microsoft/recognizers-text-data-types-timex-expression@1.3.1': {} + + '@microsoft/recognizers-text-date-time@1.3.2': + dependencies: + '@microsoft/recognizers-text': 1.3.1 + '@microsoft/recognizers-text-number': 1.3.1 + '@microsoft/recognizers-text-number-with-unit': 1.3.1 + lodash: 4.18.1 + + '@microsoft/recognizers-text-number-with-unit@1.3.1': + dependencies: + '@microsoft/recognizers-text': 1.3.1 + '@microsoft/recognizers-text-number': 1.3.1 + lodash: 4.18.1 + + '@microsoft/recognizers-text-number@1.3.1': + dependencies: + '@microsoft/recognizers-text': 1.3.1 + bignumber.js: 7.2.1 + lodash: 4.18.1 + + '@microsoft/recognizers-text-sequence@1.3.1': + dependencies: + '@microsoft/recognizers-text': 1.3.1 + grapheme-splitter: 1.0.4 + + '@microsoft/recognizers-text-suite@1.3.0': + dependencies: + '@microsoft/recognizers-text': 1.3.1 + '@microsoft/recognizers-text-choice': 1.3.1 + '@microsoft/recognizers-text-data-types-timex-expression': 1.3.1 + '@microsoft/recognizers-text-date-time': 1.3.2 + '@microsoft/recognizers-text-number': 1.3.1 + '@microsoft/recognizers-text-number-with-unit': 1.3.1 + '@microsoft/recognizers-text-sequence': 1.3.1 + + '@microsoft/recognizers-text@1.3.1': {} + '@napi-rs/lzma-linux-x64-gnu@1.5.1': optional: true + '@nlpjs/builtin-duckling@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/builtin-microsoft@4.26.1': + dependencies: + '@microsoft/recognizers-text-suite': 1.3.0 + '@nlpjs/core': 4.26.1 + + '@nlpjs/core-loader@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + '@nlpjs/request': 4.25.0 + transitivePeerDependencies: + - supports-color + + '@nlpjs/core@4.26.1': {} + + '@nlpjs/emoji@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/evaluator@4.26.1': + dependencies: + escodegen: 2.1.0 + esprima: 4.0.1 + + '@nlpjs/lang-all@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + '@nlpjs/lang-ar': 4.26.1 + '@nlpjs/lang-bn': 4.26.1 + '@nlpjs/lang-ca': 4.26.1 + '@nlpjs/lang-cs': 4.26.1 + '@nlpjs/lang-da': 4.26.1 + '@nlpjs/lang-de': 4.26.1 + '@nlpjs/lang-el': 4.26.1 + '@nlpjs/lang-en': 4.26.1 + '@nlpjs/lang-es': 4.26.1 + '@nlpjs/lang-eu': 4.26.1 + '@nlpjs/lang-fa': 4.26.1 + '@nlpjs/lang-fi': 4.26.1 + '@nlpjs/lang-fr': 4.26.1 + '@nlpjs/lang-ga': 4.26.1 + '@nlpjs/lang-gl': 4.26.1 + '@nlpjs/lang-hi': 4.26.1 + '@nlpjs/lang-hu': 4.26.1 + '@nlpjs/lang-hy': 4.26.1 + '@nlpjs/lang-id': 4.26.1 + '@nlpjs/lang-it': 4.26.1 + '@nlpjs/lang-ja': 4.26.1 + '@nlpjs/lang-ko': 4.26.1 + '@nlpjs/lang-lt': 4.26.1 + '@nlpjs/lang-ms': 4.26.1 + '@nlpjs/lang-ne': 4.26.1 + '@nlpjs/lang-nl': 4.26.1 + '@nlpjs/lang-no': 4.26.1 + '@nlpjs/lang-pl': 4.26.1 + '@nlpjs/lang-pt': 4.26.1 + '@nlpjs/lang-ro': 4.26.1 + '@nlpjs/lang-ru': 4.26.1 + '@nlpjs/lang-sl': 4.26.1 + '@nlpjs/lang-sr': 4.26.1 + '@nlpjs/lang-sv': 4.26.1 + '@nlpjs/lang-ta': 4.26.1 + '@nlpjs/lang-th': 4.26.1 + '@nlpjs/lang-tl': 4.26.1 + '@nlpjs/lang-tr': 4.26.1 + '@nlpjs/lang-uk': 4.26.1 + '@nlpjs/lang-zh': 4.26.1 + '@nlpjs/language': 4.25.0 + + '@nlpjs/lang-ar@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-bn@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-ca@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-cs@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-da@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-de@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-el@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-en-min@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-en@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + '@nlpjs/lang-en-min': 4.26.1 + + '@nlpjs/lang-es@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-eu@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-fa@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-fi@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-fr@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-ga@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-gl@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-hi@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-hu@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-hy@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-id@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-it@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-ja@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + kuromoji: 0.1.2 + + '@nlpjs/lang-ko@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-lt@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-ms@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + '@nlpjs/lang-id': 4.26.1 + + '@nlpjs/lang-ne@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-nl@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-no@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-pl@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-pt@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-ro@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-ru@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-sl@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-sr@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-sv@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-ta@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-th@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-tl@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-tr@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-uk@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/lang-zh@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/language-min@4.25.0': {} + + '@nlpjs/language@4.25.0': {} + + '@nlpjs/ner@4.27.0': + dependencies: + '@nlpjs/core': 4.26.1 + '@nlpjs/language-min': 4.25.0 + '@nlpjs/similarity': 4.26.1 + + '@nlpjs/neural@4.25.0': {} + + '@nlpjs/nlg@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + + '@nlpjs/nlp@4.27.0': + dependencies: + '@nlpjs/core': 4.26.1 + '@nlpjs/ner': 4.27.0 + '@nlpjs/nlg': 4.26.1 + '@nlpjs/nlu': 4.27.0 + '@nlpjs/sentiment': 4.26.1 + '@nlpjs/slot': 4.26.1 + + '@nlpjs/nlu@4.27.0': + dependencies: + '@nlpjs/core': 4.26.1 + '@nlpjs/language-min': 4.25.0 + '@nlpjs/neural': 4.25.0 + '@nlpjs/similarity': 4.26.1 + + '@nlpjs/request@4.25.0': + dependencies: + http-proxy-agent: 5.0.0 + https-proxy-agent: 5.0.1 + transitivePeerDependencies: + - supports-color + + '@nlpjs/sentiment@4.26.1': + dependencies: + '@nlpjs/core': 4.26.1 + '@nlpjs/language-min': 4.25.0 + '@nlpjs/neural': 4.25.0 + + '@nlpjs/similarity@4.26.1': {} + + '@nlpjs/slot@4.26.1': {} + + '@nlpjs/xtables@4.25.0': + dependencies: + xlsx: 0.18.5 + '@noble/hashes@1.8.0': {} '@nodelib/fs.scandir@2.1.5': @@ -4582,6 +5199,8 @@ snapshots: '@teppeis/multimaps@3.0.0': {} + '@tootallnate/once@2.0.1': {} + '@types/babel__core@7.20.5': dependencies: '@babel/parser': 7.29.8 @@ -4804,6 +5423,8 @@ snapshots: acorn@8.18.0: {} + adler-32@1.3.1: {} + agent-base@6.0.2: dependencies: debug: 4.4.3(supports-color@8.1.1) @@ -4893,6 +5514,10 @@ snapshots: astral-regex@2.0.0: {} + async@2.6.4: + dependencies: + lodash: 4.18.1 + async@3.2.6: {} asynckit@0.4.0: {} @@ -4929,6 +5554,8 @@ snapshots: dependencies: tweetnacl: 0.14.5 + bignumber.js@7.2.1: {} + binary-extensions@2.3.0: {} blob-util@2.0.2: {} @@ -5027,6 +5654,11 @@ snapshots: caseless@0.12.0: {} + cfb@1.2.2: + dependencies: + adler-32: 1.3.1 + crc-32: 1.2.2 + chai@5.3.3: dependencies: assertion-error: 2.0.1 @@ -5106,6 +5738,8 @@ snapshots: clone@1.0.4: optional: true + codepage@1.15.0: {} + color-convert@2.0.1: dependencies: color-name: 1.1.4 @@ -5187,6 +5821,8 @@ snapshots: optionalDependencies: typescript: 5.9.3 + crc-32@1.2.2: {} + cross-env@10.1.0: dependencies: '@epic-web/invariant': 1.0.0 @@ -5431,6 +6067,8 @@ snapshots: dotenv@16.6.1: {} + doublearray@0.0.2: {} + dunder-proto@1.0.1: dependencies: call-bind-apply-helpers: 1.0.2 @@ -5823,6 +6461,8 @@ snapshots: forwarded@0.2.0: {} + frac@1.1.2: {} + fresh@0.5.2: {} from@0.1.7: {} @@ -5944,6 +6584,8 @@ snapshots: graceful-fs@4.2.11: {} + grapheme-splitter@1.0.4: {} + has-ansi@4.0.1: dependencies: ansi-regex: 4.1.1 @@ -5984,6 +6626,14 @@ snapshots: statuses: 2.0.2 toidentifier: 1.0.1 + http-proxy-agent@5.0.0: + dependencies: + '@tootallnate/once': 2.0.1 + agent-base: 6.0.2 + debug: 4.4.3(supports-color@8.1.1) + transitivePeerDependencies: + - supports-color + http-signature@1.4.0: dependencies: assert-plus: 1.0.0 @@ -6226,6 +6876,12 @@ snapshots: dependencies: seed-random: 2.2.0 + kuromoji@0.1.2: + dependencies: + async: 2.6.4 + doublearray: 0.0.2 + zlibjs: 0.3.1 + lazy-ass@1.6.0: {} lazy-ass@2.0.3: {} @@ -6477,6 +7133,26 @@ snapshots: css-select: 4.3.0 he: 1.2.0 + node-nlp@4.27.0: + dependencies: + '@nlpjs/builtin-duckling': 4.26.1 + '@nlpjs/builtin-microsoft': 4.26.1 + '@nlpjs/core-loader': 4.26.1 + '@nlpjs/emoji': 4.26.1 + '@nlpjs/evaluator': 4.26.1 + '@nlpjs/lang-all': 4.26.1 + '@nlpjs/language': 4.25.0 + '@nlpjs/neural': 4.25.0 + '@nlpjs/nlg': 4.26.1 + '@nlpjs/nlp': 4.27.0 + '@nlpjs/nlu': 4.27.0 + '@nlpjs/request': 4.25.0 + '@nlpjs/sentiment': 4.26.1 + '@nlpjs/similarity': 4.26.1 + '@nlpjs/xtables': 4.25.0 + transitivePeerDependencies: + - supports-color + node-releases@2.0.53: {} node-source-walk@7.0.2: @@ -7075,6 +7751,10 @@ snapshots: dependencies: through: 2.3.8 + ssf@0.11.2: + dependencies: + frac: 1.1.2 + sshpk@1.18.0: dependencies: asn1: 0.2.6 @@ -7399,6 +8079,10 @@ snapshots: dependencies: string-width: 4.2.3 + wmf@1.0.2: {} + + word@0.3.0: {} + workerpool@6.5.1: {} workerpool@9.3.4: {} @@ -7423,6 +8107,16 @@ snapshots: wrappy@1.0.2: {} + xlsx@0.18.5: + dependencies: + adler-32: 1.3.1 + cfb: 1.2.2 + codepage: 1.15.0 + crc-32: 1.2.2 + ssf: 0.11.2 + wmf: 1.0.2 + word: 0.3.0 + xmlbuilder@15.1.1: {} y18n@5.0.8: {} @@ -7480,4 +8174,6 @@ snapshots: toposort: 2.0.2 type-fest: 2.19.0 + zlibjs@0.3.1: {} + zod@3.25.76: {} diff --git a/specs/backend-architecture.md b/specs/backend-architecture.md index 95e60b6..9fa7f65 100644 --- a/specs/backend-architecture.md +++ b/specs/backend-architecture.md @@ -424,31 +424,75 @@ invisible tant que le profil n'a pas rejoint/créé de foyer. ### Détection des techniques — `tech-step-matcher.ts` -`normalizeText` : décomposition NFD + suppression des diacritiques combinants -+ minuscule (ex. "Déglacer" → "deglacer"), appliquée à la fois au texte de -l'étape et aux expressions des mappings — permet d'écrire les expressions -françaises accentuées naturellement dans `reference-seed-data.ts` tout en -matchant indépendamment des accents/de la casse. +Historiquement une table `TechStepMapping` de regex par technique/locale +(`weight` pour départager les chevauchements) — remplacée par un pipeline +`node-nlp` (`TechStepClassifierService`) une fois constaté que les regex ne +généralisaient jamais au-delà de leur propre vocabulaire : une étape décrivant +la fonte du beurre comme "jusqu'à ce que le beurre ait disparu dans la poêle" +ne contient aucun verbe sur lequel une regex pourrait s'ancrer, alors que le +sens est sans ambiguïté. `TechStepMapping` a été supprimée (migration +`20260821130000_drop_tech_step_mapping`) — plus aucune table n'est +interrogée/éditée à l'exécution, les données de matching vivent en code +(`tech-step-training-data.ts`). -`matchTechStepSpans(description, mappings)` — algorithme en 4 étapes : -1. teste chaque `expression` (source de regex) contre la description - normalisée ; -2. pour une même technique, ne garde que le meilleur candidat (`weight` le - plus élevé, égalité départagée par la position la plus précoce) ; -3. entre techniques **différentes** dont les spans se chevauchent encore - (ex. `cook` générique matchant dans "cuire au four", plus spécifique - `bake`), résolution gloutonne par poids décroissant — un candidat n'est - accepté que s'il ne chevauche aucun déjà accepté (ce qui permet à des - techniques non-chevauchantes de coexister dans une même phrase, tout en - éliminant un match redondant) ; -4. tri final par position de départ. +`normalizeText` (décomposition NFD + suppression des diacritiques + minuscule) +reste utilisée par `ingredient-matcher.ts`, mais n'intervient plus dans la +détection des techniques elle-même — node-nlp gère sa propre normalisation +par langue. -`start`/`end` renvoyés sont des offsets dans le texte **normalisé**, réutilisés -tels quels contre le texte **original** pour le surlignage — repose sur -l'hypothèse documentée (et acceptée) que la décomposition NFD n'augmente -jamais le nombre de caractères d'un texte français en pratique. -`loadTechStepMappingRules(locale)` est la seule pièce qui touche la base — à -appeler une fois par requête, pas par étape. +**Pipeline en 3 étapes** (`TechStepClassifierService.matchTechStepSpans`) : +1. **NER** (entités enum node-nlp, `synonyms` de `TECH_STEP_TRAINING_DATA`) + trouve chaque mention *candidate* d'une technique dans la description + entière, avec sa position exacte — équivalent mécanique des anciennes + regex, en listes de synonymes plutôt qu'en patterns écrits à la main. + `ner.threshold: 1` (exact après normalisation, pas de tolérance floue + Levenshtein) — le défaut à 0.8 faisait matcher "faire" (verbe auxiliaire + omniprésent en français) contre le synonyme "frire" de `fry` par pure + proximité de chaîne, un faux positif détecté en calibrant contre le + corpus réel. +2. La description est découpée en clauses autour de ces candidats + (`splitIntoClauses`, pure/testable sans modèle) — une étape nommant deux + techniques a besoin que chacune soit jugée sur son propre contexte, pas + la phrase entière classée d'un bloc. +3. **Classification d'intention NLP** (le même `NlpManager`, entraîné sur les + `utterances` de `TECH_STEP_TRAINING_DATA`) classe chaque clause + individuellement — c'est ce qui apporte la compréhension du **sens** : + le corpus d'entraînement mélange volontairement des tournures ancrées sur + le mot-clé et des paraphrases qui ne l'emploient jamais (ex. "jusqu'à ce + que le beurre ait disparu" pour `melt`), donc le verdict final d'une + clause vient de ce que le modèle reconnaît comme *signifiant* la + technique, pas du mot littéral qui a déclenché son découpage. En dessous + de `CONFIDENCE_THRESHOLD` (0.65 — ajusté empiriquement contre le corpus + réel, voir `test/tech-step-matcher.test.ts`), retombe sur la technique + impliquée par l'ancre NER de la clause plutôt que d'abandonner un match + clairement ancré sur un mot-clé juste parce qu'un petit modèle n'est pas + assez confiant. + +Entraînement (`_train`) et résolution `TechStep.key -> id` sont mémoïsés une +seule fois sur le singleton partagé `techStepClassifier` (jamais par requête). +Le tout premier appel réel à `NlpManager.process()` déclenche aussi le +chargement paresseux des ressources par langue de node-nlp (plusieurs +secondes, mesuré) — `server.ts` appelle `techStepClassifier.warmUp()` avant +d'accepter du trafic pour que ce ne soit jamais la première vraie requête qui +attend. + +**Deux pièges rencontrés en construisant ce pipeline**, tous deux corrigés +dans le code (pas juste contournés) : +- `db/prisma.ts` construisait `new PrismaClient()` sans jamais importer + `config/env.ts` — dans le run de test complet, un *autre* fichier + chargeait toujours `config/env.ts` (donc `.env.test`) en premier par pur + hasard d'ordre de résolution des modules ; lancer un seul fichier de test + isolément pouvait faire gagner la course au chargement `.env` interne de + Prisma (le chemin `.env` de dev, baké dans le client généré) — silencieux + tant que `resetDatabase()` ne throw pas (heureusement son garde-fou le + fait). Fixé en import `config/env.js` pour effet de bord tout en haut de + `prisma.ts`, avant `new PrismaClient()`. +- `NlpManager` a `autoSave`/`autoLoad: true` par défaut — persiste le + modèle entraîné dans un fichier `model.nlp` (cwd du process) et le + recharge *au lieu de* ré-entraîner au prochain démarrage s'il existe déjà. + Un modèle obsolète sur disque masquerait silencieusement toute mise à + jour de `TECH_STEP_TRAINING_DATA`/`CONFIDENCE_THRESHOLD`. Les deux sont + explicitement à `false` dans le constructeur de `TechStepClassifierService`. ### Résolution ingrédients/unités — `ingredient-matcher.ts` diff --git a/specs/batch-cooking-modele.md b/specs/batch-cooking-modele.md index dde2798..ea2b71d 100644 --- a/specs/batch-cooking-modele.md +++ b/specs/batch-cooking-modele.md @@ -16,7 +16,7 @@ d'ingrédients/unités normalisé, techniques détectées, visibilité) : - **Utilisateurs & foyer** — `UserProfile`, `House`, `Diet`, `Category`, `Allergy`, `UserPreference` (thème) - **Planification** — `Planning`, `PlanningItem` -- **Recettes** — `Recipe`, `RecipeIngredient`, `Step`, `TechStep`, `TechStepMapping`, +- **Recettes** — `Recipe`, `RecipeIngredient`, `Step`, `TechStep`, `StepTechStep`, `RecipeDiet`, `RecipeFavorite` - **Sources externes** — `Source`, `HouseSource` - **Catalogue ingrédients/unités** — `Ingredient`, `Unit`, `IngredientDiet`, @@ -351,11 +351,14 @@ fiable. | `order` | Position dans la recette | `tech_step` (`TechStep`, `key` unique, ex. `"simmer"`) est le catalogue des -techniques (mijoter, préchauffer…). `tech_step_mapping` (`TechStepMapping`) -porte les règles de détection : `expression` (regex testée contre la -description), `weight` (départage en cas de règles concurrentes), `locale` -(une même technique peut avoir un jeu de règles par langue — voir -[backend-architecture.md](./backend-architecture.md#détection-des-techniques--tech-step-matcherts)). +techniques (mijoter, préchauffer…) — juste un id/clé stable référencé par +`step_tech_step`. Les données de détection elles-mêmes (synonymes + phrases +d'exemple par langue, entraînant un classifieur `node-nlp`) vivent en code +(`tech-step-training-data.ts`), pas dans une table — l'ancienne +`tech_step_mapping` (`TechStepMapping`, une regex par technique/locale) a +été supprimée une fois constaté que les regex ne généralisaient jamais +au-delà de leur propre vocabulaire — voir +[backend-architecture.md](./backend-architecture.md#détection-des-techniques--tech-step-matcherts). `step_tech_step` (`StepTechStep`) est la **séquence ordonnée** des techniques détectées pour une étape — une instruction peut en impliquer plusieurs (ex. @@ -386,7 +389,6 @@ dans `Step.description`, utilisé pour le surlignage côté web | `recipe` | `authorId` | `user_profiles` | | `recipe` | `authorHouseId` | `house` | | `step` | `recipeId` | `recipe` | -| `tech_step_mapping` | `techStepId` | `tech_step` | ### Many-to-many (tables de jointure explicites, avec ou sans champ additionnel)