feat(admin): monitoring des microservices + heartbeat du worker LLM
PR 4 du chantier admin. Board de sante temps reel des dependances.
- POST /internal/tech-steps/heartbeat (requireInternalWorker) ->
recordWorkerHeartbeat : upsert WorkerHeartbeat (cle fixe
"tech-step-llm-worker"), lastRunAt/lastResult pour un ping "job".
Schema workerHeartbeatSchema dans packages/shared.
- services/tech-step-llm-worker : api-client.postHeartbeat (best-effort,
ne throw jamais) appele au boot (index.ts), a chaque tick et apres
chaque job (scheduler.ts, avec job/ok/counts).
- admin-monitoring.service.ts + GET /admin/monitoring (requireAdmin) :
sonde active bornee (~2 s) de Postgres (SELECT 1), l'API (uptime/RSS),
tech-step-intent-service (/health), et le worker via son heartbeat.
Statut up/degraded/down/unknown ; une sonde down ne casse ni les autres
ni l'endpoint. Seuils worker : > 8 j degraded, > 21 j down.
- MonitoringView / ServiceHealthView dans packages/shared.
- Front : MonitoringPage (grille de cartes coloree par statut, re-poll
15 s), logique pure monitoring.ts, i18n admin.monitoring.*,
AdminApiClient.getMonitoring.
- Tests : Mocha admin-monitoring.test.ts (heartbeat 401/400/upsert
job+boot ; GET /admin/monitoring 401, board 4 cibles, worker unknown
sans heartbeat puis up apres) ; Cypress monitoring.cy.ts (2 verts).
Worker mocha : 6/6 toujours verts.
- specs/backend-architecture.md : section monitoring. .gitignore :
apps/admin-web/cypress/{screenshots,videos,downloads}.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
parent
3a416ea955
commit
29432e5bf2
19 changed files with 858 additions and 7 deletions
3
.gitignore
vendored
3
.gitignore
vendored
|
|
@ -163,3 +163,6 @@ tmp-mockups/
|
|||
apps/web/cypress/screenshots/
|
||||
apps/web/cypress/videos/
|
||||
apps/web/cypress/downloads/
|
||||
apps/admin-web/cypress/screenshots/
|
||||
apps/admin-web/cypress/videos/
|
||||
apps/admin-web/cypress/downloads/
|
||||
|
|
|
|||
83
apps/admin-web/cypress/e2e/monitoring.cy.ts
Normal file
83
apps/admin-web/cypress/e2e/monitoring.cy.ts
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
// Mocks the admin API via cy.intercept — no live backend.
|
||||
|
||||
const adminBody = {
|
||||
id: 1,
|
||||
email: "ops@example.com",
|
||||
name: "Ops",
|
||||
createdAt: "2026-08-01T00:00:00.000Z",
|
||||
lastLoginAt: "2026-08-28T09:00:00.000Z",
|
||||
};
|
||||
|
||||
function monitoringFixture() {
|
||||
return {
|
||||
generatedAt: "2026-08-28T09:15:00.000Z",
|
||||
services: [
|
||||
{
|
||||
key: "postgres",
|
||||
status: "up",
|
||||
latencyMs: 3.2,
|
||||
detail: null,
|
||||
checkedAt: "2026-08-28T09:15:00.000Z",
|
||||
},
|
||||
{
|
||||
key: "api",
|
||||
status: "up",
|
||||
latencyMs: 0,
|
||||
detail: "uptime 3 h 12 min · RSS 120 Mo",
|
||||
checkedAt: "2026-08-28T09:15:00.000Z",
|
||||
},
|
||||
{
|
||||
key: "intent-service",
|
||||
status: "down",
|
||||
latencyMs: null,
|
||||
detail: "fetch failed",
|
||||
checkedAt: "2026-08-28T09:15:00.000Z",
|
||||
},
|
||||
{
|
||||
key: "tech-step-llm-worker",
|
||||
status: "degraded",
|
||||
latencyMs: null,
|
||||
detail: "dernier battement il y a 9 j",
|
||||
checkedAt: "2026-08-28T09:15:00.000Z",
|
||||
lastRunAt: "2026-08-19T03:00:00.000Z",
|
||||
lastResult: { job: "audit-low-confidence", ok: true, counts: { suggestions: 2 } },
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
describe("Admin monitoring", () => {
|
||||
beforeEach(() => {
|
||||
cy.viewport(1400, 900);
|
||||
cy.intercept("GET", "**/admin/auth/me", { statusCode: 200, body: adminBody });
|
||||
});
|
||||
|
||||
it("renders one card per service with its status and details", () => {
|
||||
cy.intercept("GET", "**/admin/monitoring", { statusCode: 200, body: monitoringFixture() }).as(
|
||||
"getMonitoring",
|
||||
);
|
||||
cy.visit("/monitoring");
|
||||
cy.wait("@getMonitoring");
|
||||
|
||||
cy.get(".monitoring-card").should("have.length", 4);
|
||||
|
||||
cy.contains(".monitoring-card", "Base de données")
|
||||
.should("have.class", "monitoring-card--up")
|
||||
.and("contain.text", "3.2 ms");
|
||||
cy.contains(".monitoring-card", "Service NLP (spaCy)")
|
||||
.should("have.class", "monitoring-card--down")
|
||||
.and("contain.text", "Hors service");
|
||||
cy.contains(".monitoring-card", "Worker LLM")
|
||||
.should("have.class", "monitoring-card--degraded")
|
||||
.and("contain.text", "audit-low-confidence");
|
||||
});
|
||||
|
||||
it("shows an error state when the request fails", () => {
|
||||
cy.intercept("GET", "**/admin/monitoring", {
|
||||
statusCode: 500,
|
||||
body: { code: 5000, message: "x" },
|
||||
});
|
||||
cy.visit("/monitoring");
|
||||
cy.contains("Impossible de charger").should("be.visible");
|
||||
});
|
||||
});
|
||||
|
|
@ -4,6 +4,7 @@ import {
|
|||
type ApiErrorResponse,
|
||||
ErrorCode,
|
||||
type MetricsView,
|
||||
type MonitoringView,
|
||||
} from "@batch-cooking/shared";
|
||||
|
||||
/**
|
||||
|
|
@ -96,6 +97,11 @@ export class AdminApiClient {
|
|||
public getMetrics(days: number): Promise<MetricsView> {
|
||||
return this._request(`/admin/metrics?days=${days}`);
|
||||
}
|
||||
|
||||
/** Live health of Postgres, the API, the intent-service and the LLM worker — polled by the monitoring board. */
|
||||
public getMonitoring(): Promise<MonitoringView> {
|
||||
return this._request("/admin/monitoring");
|
||||
}
|
||||
}
|
||||
|
||||
/** Single shared instance — stateless, same reasoning as apps/web's `apiClient`. */
|
||||
|
|
|
|||
|
|
@ -59,7 +59,25 @@
|
|||
},
|
||||
"monitoring": {
|
||||
"title": "Monitoring",
|
||||
"lead": "Santé des microservices et de la base de données."
|
||||
"lead": "Santé des microservices et de la base de données.",
|
||||
"lastChecked": "Dernière vérification à {{time}}",
|
||||
"latency": "Latence",
|
||||
"detail": "Détail",
|
||||
"lastRun": "Dernier job",
|
||||
"never": "jamais",
|
||||
"jobFailed": "échec",
|
||||
"status": {
|
||||
"up": "OK",
|
||||
"degraded": "Dégradé",
|
||||
"down": "Hors service",
|
||||
"unknown": "Inconnu"
|
||||
},
|
||||
"service": {
|
||||
"postgres": "Base de données",
|
||||
"api": "API",
|
||||
"intent-service": "Service NLP (spaCy)",
|
||||
"tech-step-llm-worker": "Worker LLM"
|
||||
}
|
||||
},
|
||||
"corrections": {
|
||||
"title": "Corrections",
|
||||
|
|
|
|||
|
|
@ -1,18 +1,110 @@
|
|||
import type { MonitoringView, ServiceHealthView } from "@batch-cooking/shared";
|
||||
import { useEffect, useState } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { adminApiClient } from "../../api/client";
|
||||
import "../admin-page.scss";
|
||||
import "./monitoring-page.scss";
|
||||
import { clockTime, POLL_INTERVAL_MS, statusModifier } from "./monitoring";
|
||||
|
||||
type MonitoringState =
|
||||
| { status: "loading" }
|
||||
| { status: "loaded"; data: MonitoringView }
|
||||
| { status: "error" };
|
||||
|
||||
/**
|
||||
* Microservice health board — active probes of Postgres, the API, the
|
||||
* intent-service and the LLM worker's heartbeat, fed by
|
||||
* `GET /admin/monitoring`. Placeholder until PR 4 (monitoring).
|
||||
* Microservice health board. Fetches `GET /admin/monitoring` on mount and
|
||||
* re-polls every {@link POLL_INTERVAL_MS} ms. One card per probed target
|
||||
* (Postgres, API, intent-service, LLM worker), coloured by status.
|
||||
*/
|
||||
export function MonitoringPage() {
|
||||
const { t } = useTranslation();
|
||||
const [state, setState] = useState<MonitoringState>({ status: "loading" });
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
|
||||
function load() {
|
||||
adminApiClient
|
||||
.getMonitoring()
|
||||
.then((data) => {
|
||||
if (!cancelled) setState({ status: "loaded", data });
|
||||
})
|
||||
.catch(() => {
|
||||
if (!cancelled)
|
||||
setState((prev) => (prev.status === "loaded" ? prev : { status: "error" }));
|
||||
});
|
||||
}
|
||||
|
||||
load();
|
||||
const timer = setInterval(load, POLL_INTERVAL_MS);
|
||||
return () => {
|
||||
cancelled = true;
|
||||
clearInterval(timer);
|
||||
};
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<div className="admin-page">
|
||||
<h1 className="admin-page__title">{t("admin.monitoring.title")}</h1>
|
||||
<p className="admin-page__lead">{t("admin.monitoring.lead")}</p>
|
||||
<p className="admin-page__placeholder">{t("admin.common.comingSoon")}</p>
|
||||
|
||||
{state.status === "loading" && (
|
||||
<p className="admin-page__placeholder">{t("admin.common.loading")}</p>
|
||||
)}
|
||||
{state.status === "error" && (
|
||||
<p className="admin-page__placeholder">{t("admin.common.loadError")}</p>
|
||||
)}
|
||||
{state.status === "loaded" && (
|
||||
<>
|
||||
<p className="monitoring-checked">
|
||||
{t("admin.monitoring.lastChecked", { time: clockTime(state.data.generatedAt) })}
|
||||
</p>
|
||||
<div className="monitoring-grid">
|
||||
{state.data.services.map((service) => (
|
||||
<ServiceCard key={service.key} service={service} />
|
||||
))}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function ServiceCard({ service }: { service: ServiceHealthView }) {
|
||||
const { t } = useTranslation();
|
||||
return (
|
||||
<article className={`monitoring-card monitoring-card--${statusModifier(service.status)}`}>
|
||||
<header className="monitoring-card__head">
|
||||
<span className="monitoring-card__dot" aria-hidden="true" />
|
||||
<h2>{t(`admin.monitoring.service.${service.key}`, { defaultValue: service.key })}</h2>
|
||||
<span className="monitoring-card__status">
|
||||
{t(`admin.monitoring.status.${service.status}`)}
|
||||
</span>
|
||||
</header>
|
||||
<dl className="monitoring-card__meta">
|
||||
{service.latencyMs !== null && (
|
||||
<div>
|
||||
<dt>{t("admin.monitoring.latency")}</dt>
|
||||
<dd>{service.latencyMs} ms</dd>
|
||||
</div>
|
||||
)}
|
||||
{service.detail && (
|
||||
<div>
|
||||
<dt>{t("admin.monitoring.detail")}</dt>
|
||||
<dd>{service.detail}</dd>
|
||||
</div>
|
||||
)}
|
||||
{service.lastRunAt !== undefined && (
|
||||
<div>
|
||||
<dt>{t("admin.monitoring.lastRun")}</dt>
|
||||
<dd>
|
||||
{service.lastRunAt ? clockTime(service.lastRunAt) : t("admin.monitoring.never")}
|
||||
{service.lastResult?.job ? ` · ${service.lastResult.job}` : ""}
|
||||
{service.lastResult?.ok === false ? ` · ${t("admin.monitoring.jobFailed")}` : ""}
|
||||
</dd>
|
||||
</div>
|
||||
)}
|
||||
</dl>
|
||||
</article>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
98
apps/admin-web/src/pages/monitoring/monitoring-page.scss
Normal file
98
apps/admin-web/src/pages/monitoring/monitoring-page.scss
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
// =============================================================================
|
||||
// MonitoringPage — a grid of service health cards, one per probed target.
|
||||
// Status drives a coloured left border + dot.
|
||||
// =============================================================================
|
||||
|
||||
.monitoring-checked {
|
||||
margin: 0 0 var(--space-md);
|
||||
font-size: var(--font-size-sm);
|
||||
color: var(--color-text-muted);
|
||||
}
|
||||
|
||||
.monitoring-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fill, minmax(18rem, 1fr));
|
||||
gap: var(--space-md);
|
||||
}
|
||||
|
||||
.monitoring-card {
|
||||
padding: var(--space-md);
|
||||
background: var(--color-surface);
|
||||
border: 1px solid var(--color-border);
|
||||
border-left: 4px solid var(--color-border);
|
||||
border-radius: var(--radius-md);
|
||||
|
||||
--status-color: var(--color-text-muted);
|
||||
&--up {
|
||||
--status-color: var(--color-success);
|
||||
}
|
||||
&--degraded {
|
||||
--status-color: var(--color-warning);
|
||||
}
|
||||
&--down {
|
||||
--status-color: var(--color-error);
|
||||
}
|
||||
&--unknown {
|
||||
--status-color: var(--color-text-muted);
|
||||
}
|
||||
|
||||
border-left-color: var(--status-color);
|
||||
|
||||
&__head {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: var(--space-sm);
|
||||
margin-bottom: var(--space-sm);
|
||||
|
||||
h2 {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
font-size: var(--font-size-md);
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
}
|
||||
|
||||
&__dot {
|
||||
flex-shrink: 0;
|
||||
width: 0.6rem;
|
||||
height: 0.6rem;
|
||||
border-radius: 50%;
|
||||
background: var(--status-color);
|
||||
}
|
||||
|
||||
&__status {
|
||||
flex-shrink: 0;
|
||||
font-size: var(--font-size-xs);
|
||||
font-weight: 700;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.04em;
|
||||
color: var(--status-color);
|
||||
}
|
||||
|
||||
&__meta {
|
||||
margin: 0;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: var(--space-xs);
|
||||
|
||||
div {
|
||||
display: flex;
|
||||
gap: var(--space-sm);
|
||||
font-size: var(--font-size-sm);
|
||||
}
|
||||
|
||||
dt {
|
||||
flex-shrink: 0;
|
||||
color: var(--color-text-muted);
|
||||
min-width: 5rem;
|
||||
}
|
||||
|
||||
dd {
|
||||
margin: 0;
|
||||
color: var(--color-text);
|
||||
word-break: break-word;
|
||||
}
|
||||
}
|
||||
}
|
||||
19
apps/admin-web/src/pages/monitoring/monitoring.ts
Normal file
19
apps/admin-web/src/pages/monitoring/monitoring.ts
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
import type { ServiceStatus } from "@batch-cooking/shared";
|
||||
|
||||
/**
|
||||
* Pure helpers for `MonitoringPage` — kept out of the `.tsx` per repo
|
||||
* convention.
|
||||
*/
|
||||
|
||||
/** CSS modifier suffix for a status pill (`monitoring-card--up`, etc.). */
|
||||
export function statusModifier(status: ServiceStatus): string {
|
||||
return status;
|
||||
}
|
||||
|
||||
/** How often the board re-polls `GET /admin/monitoring`, in ms. */
|
||||
export const POLL_INTERVAL_MS = 15_000;
|
||||
|
||||
/** `"2026-08-28T09:00:00.000Z"` → `"09:00:00"` (local time) for the "last checked" line. */
|
||||
export function clockTime(iso: string): string {
|
||||
return new Date(iso).toLocaleTimeString("fr-FR");
|
||||
}
|
||||
20
apps/api/src/modules/admin/admin-monitoring.routes.ts
Normal file
20
apps/api/src/modules/admin/admin-monitoring.routes.ts
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
import { wrapAsyncHandler } from "@batch-cooking/express-tools";
|
||||
import { Router } from "express";
|
||||
import { requireAdmin } from "../../middlewares/require-admin.js";
|
||||
import { getMonitoring } from "./admin-monitoring.service.js";
|
||||
|
||||
/** Router mounted at `/admin/monitoring` (via `admin.routes.ts`) — behind {@link requireAdmin}. */
|
||||
export const adminMonitoringRouter = Router();
|
||||
|
||||
/**
|
||||
* Actively probes Postgres, the API, `tech-step-intent-service` and the LLM
|
||||
* worker's heartbeat, returning a {@link MonitoringView} status board (see
|
||||
* {@link getMonitoring}). No params; the admin UI polls it on an interval.
|
||||
*/
|
||||
adminMonitoringRouter.get(
|
||||
"/",
|
||||
requireAdmin,
|
||||
wrapAsyncHandler(async (_req, res) => {
|
||||
res.status(200).json(await getMonitoring());
|
||||
}),
|
||||
);
|
||||
174
apps/api/src/modules/admin/admin-monitoring.service.ts
Normal file
174
apps/api/src/modules/admin/admin-monitoring.service.ts
Normal file
|
|
@ -0,0 +1,174 @@
|
|||
import type { MonitoringView, ServiceHealthView, ServiceStatus } from "@batch-cooking/shared";
|
||||
import { env } from "../../config/env.js";
|
||||
import { prisma } from "../../db/prisma.js";
|
||||
|
||||
/** How long each outbound probe (Postgres query, intent-service HTTP) is allowed to take before it counts as `down`. */
|
||||
const PROBE_TIMEOUT_MS = 2000;
|
||||
|
||||
/**
|
||||
* Heartbeat-age thresholds for the LLM worker. Its default cron is weekly
|
||||
* (`TECH_STEP_WORKER_CRON`, `0 3 * * 0`), and it also pings on boot/tick —
|
||||
* so no ping for **8 days** means it likely missed its last scheduled fire
|
||||
* (`degraded`), and none for **3 weeks** means it's almost certainly not
|
||||
* running at all (`down`).
|
||||
*/
|
||||
const WORKER_STALE_AFTER_MS = 8 * 24 * 60 * 60 * 1000;
|
||||
const WORKER_DOWN_AFTER_MS = 21 * 24 * 60 * 60 * 1000;
|
||||
|
||||
const WORKER_KEY = "tech-step-llm-worker";
|
||||
|
||||
function nowIso(): string {
|
||||
return new Date().toISOString();
|
||||
}
|
||||
|
||||
function roundMs(value: number): number {
|
||||
return Math.round(value * 10) / 10;
|
||||
}
|
||||
|
||||
function errMessage(err: unknown): string {
|
||||
return err instanceof Error ? err.message : String(err);
|
||||
}
|
||||
|
||||
/** `12500` → `"il y a 12 s"`, `90000` → `"il y a 1 min"`, `172800000` → `"il y a 2 j"`. */
|
||||
function formatAgo(ms: number): string {
|
||||
const s = Math.round(ms / 1000);
|
||||
if (s < 60) return `il y a ${s} s`;
|
||||
const m = Math.round(s / 60);
|
||||
if (m < 60) return `il y a ${m} min`;
|
||||
const h = Math.round(m / 60);
|
||||
if (h < 48) return `il y a ${h} h`;
|
||||
return `il y a ${Math.round(h / 24)} j`;
|
||||
}
|
||||
|
||||
/** `process.uptime()` seconds → `"3 h 12 min"` / `"5 min"` / `"42 s"`. */
|
||||
function formatUptime(seconds: number): string {
|
||||
const s = Math.floor(seconds);
|
||||
if (s < 60) return `${s} s`;
|
||||
const m = Math.floor(s / 60);
|
||||
if (m < 60) return `${m} min`;
|
||||
const h = Math.floor(m / 60);
|
||||
return `${h} h ${m % 60} min`;
|
||||
}
|
||||
|
||||
/** `prisma.$queryRaw\`SELECT 1\`` with a bounded timeout — the DB connectivity probe. */
|
||||
async function probePostgres(): Promise<ServiceHealthView> {
|
||||
const start = performance.now();
|
||||
try {
|
||||
// `$queryRaw` doesn't take an AbortSignal — bound it with a race instead.
|
||||
await Promise.race([
|
||||
prisma.$queryRaw`SELECT 1`,
|
||||
new Promise((_resolve, reject) =>
|
||||
setTimeout(() => reject(new Error("timeout")), PROBE_TIMEOUT_MS),
|
||||
),
|
||||
]);
|
||||
return {
|
||||
key: "postgres",
|
||||
status: "up",
|
||||
latencyMs: roundMs(performance.now() - start),
|
||||
detail: null,
|
||||
checkedAt: nowIso(),
|
||||
};
|
||||
} catch (err) {
|
||||
return {
|
||||
key: "postgres",
|
||||
status: "down",
|
||||
latencyMs: null,
|
||||
detail: errMessage(err),
|
||||
checkedAt: nowIso(),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/** The API itself — trivially "up" (it's answering), reported with its process uptime/memory. */
|
||||
function probeApi(): ServiceHealthView {
|
||||
const mem = process.memoryUsage();
|
||||
return {
|
||||
key: "api",
|
||||
status: "up",
|
||||
latencyMs: 0,
|
||||
detail: `uptime ${formatUptime(process.uptime())} · RSS ${Math.round(mem.rss / 1_000_000)} Mo`,
|
||||
checkedAt: nowIso(),
|
||||
};
|
||||
}
|
||||
|
||||
/** `GET {INTENT_SERVICE_BASE_URL}/health` — no secret needed on that route (see the service's `routes/health.py`). */
|
||||
async function probeIntentService(): Promise<ServiceHealthView> {
|
||||
const start = performance.now();
|
||||
try {
|
||||
const res = await fetch(`${env.INTENT_SERVICE_BASE_URL}/health`, {
|
||||
signal: AbortSignal.timeout(PROBE_TIMEOUT_MS),
|
||||
});
|
||||
const latencyMs = roundMs(performance.now() - start);
|
||||
return {
|
||||
key: "intent-service",
|
||||
status: res.ok ? "up" : "degraded",
|
||||
latencyMs,
|
||||
detail: `HTTP ${res.status}`,
|
||||
checkedAt: nowIso(),
|
||||
};
|
||||
} catch (err) {
|
||||
return {
|
||||
key: "intent-service",
|
||||
status: "down",
|
||||
latencyMs: null,
|
||||
detail: errMessage(err),
|
||||
checkedAt: nowIso(),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/** Reads the LLM worker's stored `WorkerHeartbeat` (it has no HTTP surface to probe directly) and grades it by age + last job outcome. */
|
||||
async function probeWorker(): Promise<ServiceHealthView> {
|
||||
const heartbeat = await prisma.workerHeartbeat.findUnique({ where: { workerKey: WORKER_KEY } });
|
||||
if (!heartbeat) {
|
||||
return {
|
||||
key: WORKER_KEY,
|
||||
status: "unknown",
|
||||
latencyMs: null,
|
||||
detail: "aucun battement reçu",
|
||||
checkedAt: nowIso(),
|
||||
lastRunAt: null,
|
||||
lastResult: null,
|
||||
};
|
||||
}
|
||||
|
||||
const ageMs = Date.now() - heartbeat.lastSeenAt.getTime();
|
||||
const lastResult = (heartbeat.lastResult ?? null) as ServiceHealthView["lastResult"];
|
||||
|
||||
let status: ServiceStatus = "up";
|
||||
if (ageMs > WORKER_DOWN_AFTER_MS) status = "down";
|
||||
else if (ageMs > WORKER_STALE_AFTER_MS || lastResult?.ok === false) status = "degraded";
|
||||
|
||||
return {
|
||||
key: WORKER_KEY,
|
||||
status,
|
||||
latencyMs: null,
|
||||
detail: `dernier battement ${formatAgo(ageMs)}`,
|
||||
checkedAt: nowIso(),
|
||||
lastRunAt: heartbeat.lastRunAt?.toISOString() ?? null,
|
||||
lastResult,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Actively probes every dependency the admin monitoring board watches —
|
||||
* Postgres, the API itself, `tech-step-intent-service` (`/health`), and the
|
||||
* LLM worker (via its stored heartbeat). Each probe is independent and
|
||||
* bounded ({@link PROBE_TIMEOUT_MS}); one being `down` never fails the
|
||||
* others or the endpoint.
|
||||
*/
|
||||
export async function getMonitoring(): Promise<MonitoringView> {
|
||||
try {
|
||||
const [postgres, intentService, worker] = await Promise.all([
|
||||
probePostgres(),
|
||||
probeIntentService(),
|
||||
probeWorker(),
|
||||
]);
|
||||
return {
|
||||
generatedAt: nowIso(),
|
||||
services: [postgres, probeApi(), intentService, worker],
|
||||
};
|
||||
} catch (err) {
|
||||
throw err; // see recipe.service.ts's equivalent catch comment
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
import { Router } from "express";
|
||||
import { adminAuthRouter } from "./admin-auth.routes.js";
|
||||
import { adminMetricsRouter } from "./admin-metrics.routes.js";
|
||||
import { adminMonitoringRouter } from "./admin-monitoring.routes.js";
|
||||
|
||||
/**
|
||||
* Aggregator for the admin application's API surface, mounted at `/admin`
|
||||
|
|
@ -13,3 +14,4 @@ export const adminRouter = Router();
|
|||
|
||||
adminRouter.use("/auth", adminAuthRouter);
|
||||
adminRouter.use("/metrics", adminMetricsRouter);
|
||||
adminRouter.use("/monitoring", adminMonitoringRouter);
|
||||
|
|
|
|||
|
|
@ -3,12 +3,14 @@ import {
|
|||
auditBatchQuerySchema,
|
||||
submitTrainingSuggestionsSchema,
|
||||
workerBatchQuerySchema,
|
||||
workerHeartbeatSchema,
|
||||
} from "@batch-cooking/shared";
|
||||
import { Router } from "express";
|
||||
import { requireInternalWorker } from "../../middlewares/require-internal-worker.js";
|
||||
import {
|
||||
getAuditBatch,
|
||||
getPendingCorrections,
|
||||
recordWorkerHeartbeat,
|
||||
submitTrainingSuggestions,
|
||||
} from "./tech-step-worker.service.js";
|
||||
|
||||
|
|
@ -47,3 +49,19 @@ techStepWorkerRouter.post(
|
|||
res.status(201).json(await submitTrainingSuggestions(input));
|
||||
}),
|
||||
);
|
||||
|
||||
/**
|
||||
* Liveness ping from the worker (which has no inbound HTTP surface of its
|
||||
* own) — upserts its `WorkerHeartbeat` row so the admin monitoring board
|
||||
* can show it as up / stale / down and surface its last job result. Sent
|
||||
* on boot, on every scheduler tick, and after each job.
|
||||
*/
|
||||
techStepWorkerRouter.post(
|
||||
"/heartbeat",
|
||||
requireInternalWorker,
|
||||
wrapAsyncHandler(async (req, res) => {
|
||||
const input = workerHeartbeatSchema.parse(req.body);
|
||||
await recordWorkerHeartbeat(input);
|
||||
res.status(200).json({ ok: true });
|
||||
}),
|
||||
);
|
||||
|
|
|
|||
|
|
@ -4,7 +4,9 @@ import {
|
|||
type PendingTechStepCorrectionView,
|
||||
type SubmitTrainingSuggestionsInput,
|
||||
type TechStepAuditClauseView,
|
||||
type WorkerHeartbeatInput,
|
||||
} from "@batch-cooking/shared";
|
||||
import type { Prisma } from "@prisma/client";
|
||||
import { prisma } from "../../db/prisma.js";
|
||||
import {
|
||||
CONFIDENCE_THRESHOLD,
|
||||
|
|
@ -199,3 +201,44 @@ export async function submitTrainingSuggestions(
|
|||
throw err; // see recipe.service.ts's equivalent catch comment
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The one worker with a `WorkerHeartbeat` row today — a fixed key, not
|
||||
* something the caller supplies (only one worker exists, and letting it
|
||||
* name itself would just be a spoofing surface behind the same shared
|
||||
* secret).
|
||||
*/
|
||||
const WORKER_KEY = "tech-step-llm-worker";
|
||||
|
||||
/**
|
||||
* Upserts `services/tech-step-llm-worker`'s heartbeat row (see
|
||||
* `POST /internal/tech-steps/heartbeat`). Every ping bumps `lastSeenAt`; a
|
||||
* `"job"` ping also records `lastRunAt` + a small `lastResult` summary so
|
||||
* the admin monitoring board can show what the worker last did and whether
|
||||
* it worked.
|
||||
*/
|
||||
export async function recordWorkerHeartbeat(input: WorkerHeartbeatInput): Promise<void> {
|
||||
try {
|
||||
const now = new Date();
|
||||
const jobResult: Prisma.InputJsonValue | undefined =
|
||||
input.event === "job"
|
||||
? { job: input.job ?? null, ok: input.ok ?? null, counts: input.counts ?? {} }
|
||||
: undefined;
|
||||
|
||||
await prisma.workerHeartbeat.upsert({
|
||||
where: { workerKey: WORKER_KEY },
|
||||
create: {
|
||||
workerKey: WORKER_KEY,
|
||||
lastSeenAt: now,
|
||||
lastRunAt: input.event === "job" ? now : null,
|
||||
lastResult: jobResult,
|
||||
},
|
||||
update: {
|
||||
lastSeenAt: now,
|
||||
...(input.event === "job" ? { lastRunAt: now, lastResult: jobResult } : {}),
|
||||
},
|
||||
});
|
||||
} catch (err) {
|
||||
throw err; // see recipe.service.ts's equivalent catch comment
|
||||
}
|
||||
}
|
||||
|
|
|
|||
160
apps/api/test/admin-monitoring.test.ts
Normal file
160
apps/api/test/admin-monitoring.test.ts
Normal file
|
|
@ -0,0 +1,160 @@
|
|||
import { faker } from "@faker-js/faker";
|
||||
import { expect } from "chai";
|
||||
import request from "supertest";
|
||||
import { createApp } from "../src/app.js";
|
||||
import { env } from "../src/config/env.js";
|
||||
import { prisma } from "../src/db/prisma.js";
|
||||
import { hashAdminPassword } from "../src/modules/admin/admin-auth.service.js";
|
||||
import { resetDatabase } from "../test-support/reset-db.js";
|
||||
|
||||
const SECRET_HEADER = "X-Internal-Worker-Secret";
|
||||
const VALID_STATUSES = ["up", "degraded", "down", "unknown"];
|
||||
|
||||
async function seedAdmin(): Promise<{ email: string; password: string }> {
|
||||
const email = faker.internet.email().toLowerCase();
|
||||
const password = faker.internet.password({ length: 16 });
|
||||
await prisma.adminUser.create({
|
||||
data: { email, name: faker.person.fullName(), passwordHash: await hashAdminPassword(password) },
|
||||
});
|
||||
return { email, password };
|
||||
}
|
||||
|
||||
const adminSecretConfigured = env.ADMIN_JWT_SECRET !== undefined;
|
||||
const workerSecretConfigured = env.INTERNAL_WORKER_SECRET !== undefined;
|
||||
|
||||
describe("Admin monitoring", () => {
|
||||
const app = createApp();
|
||||
|
||||
beforeEach(async () => {
|
||||
await resetDatabase();
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
await prisma.$disconnect();
|
||||
});
|
||||
|
||||
describe("POST /internal/tech-steps/heartbeat", () => {
|
||||
it("rejects a request with no worker secret with 401", async () => {
|
||||
const res = await request(app).post("/internal/tech-steps/heartbeat").send({ event: "boot" });
|
||||
expect(res.status).to.equal(401);
|
||||
});
|
||||
|
||||
it("rejects a malformed body with 400", async function () {
|
||||
if (!workerSecretConfigured) {
|
||||
// biome-ignore lint/suspicious/noExplicitAny: mocha's `this.skip()` isn't typed here.
|
||||
(this as any).skip();
|
||||
return;
|
||||
}
|
||||
const res = await request(app)
|
||||
.post("/internal/tech-steps/heartbeat")
|
||||
.set(SECRET_HEADER, env.INTERNAL_WORKER_SECRET as string)
|
||||
.send({ event: "not-a-real-event" });
|
||||
expect(res.status).to.equal(400);
|
||||
});
|
||||
|
||||
it("upserts the worker heartbeat, recording lastRunAt/lastResult for a job ping", async function () {
|
||||
if (!workerSecretConfigured) {
|
||||
// biome-ignore lint/suspicious/noExplicitAny: see above.
|
||||
(this as any).skip();
|
||||
return;
|
||||
}
|
||||
const res = await request(app)
|
||||
.post("/internal/tech-steps/heartbeat")
|
||||
.set(SECRET_HEADER, env.INTERNAL_WORKER_SECRET as string)
|
||||
.send({
|
||||
event: "job",
|
||||
job: "audit-low-confidence",
|
||||
ok: true,
|
||||
counts: { suggestions: 3 },
|
||||
});
|
||||
expect(res.status).to.equal(200);
|
||||
expect(res.body).to.deep.equal({ ok: true });
|
||||
|
||||
const stored = await prisma.workerHeartbeat.findUniqueOrThrow({
|
||||
where: { workerKey: "tech-step-llm-worker" },
|
||||
});
|
||||
expect(stored.lastRunAt).to.not.equal(null);
|
||||
expect(stored.lastResult).to.deep.equal({
|
||||
job: "audit-low-confidence",
|
||||
ok: true,
|
||||
counts: { suggestions: 3 },
|
||||
});
|
||||
});
|
||||
|
||||
it("leaves lastRunAt null for a boot/tick ping", async function () {
|
||||
if (!workerSecretConfigured) {
|
||||
// biome-ignore lint/suspicious/noExplicitAny: see above.
|
||||
(this as any).skip();
|
||||
return;
|
||||
}
|
||||
await request(app)
|
||||
.post("/internal/tech-steps/heartbeat")
|
||||
.set(SECRET_HEADER, env.INTERNAL_WORKER_SECRET as string)
|
||||
.send({ event: "boot" });
|
||||
|
||||
const stored = await prisma.workerHeartbeat.findUniqueOrThrow({
|
||||
where: { workerKey: "tech-step-llm-worker" },
|
||||
});
|
||||
expect(stored.lastSeenAt).to.be.instanceOf(Date);
|
||||
expect(stored.lastRunAt).to.equal(null);
|
||||
});
|
||||
});
|
||||
|
||||
describe("GET /admin/monitoring", () => {
|
||||
it("rejects a request with no admin session with 401", async () => {
|
||||
const res = await request(app).get("/admin/monitoring");
|
||||
expect(res.status).to.equal(401);
|
||||
});
|
||||
|
||||
it("returns a status board covering all four targets, never crashing on an unreachable probe", async function () {
|
||||
if (!adminSecretConfigured) {
|
||||
// biome-ignore lint/suspicious/noExplicitAny: see above.
|
||||
(this as any).skip();
|
||||
return;
|
||||
}
|
||||
const { email, password } = await seedAdmin();
|
||||
const agent = request.agent(app);
|
||||
await agent.post("/admin/auth/login").send({ email, password });
|
||||
|
||||
const res = await agent.get("/admin/monitoring");
|
||||
expect(res.status).to.equal(200);
|
||||
|
||||
const keys = res.body.services.map((s: { key: string }) => s.key);
|
||||
expect(keys).to.have.members(["postgres", "api", "intent-service", "tech-step-llm-worker"]);
|
||||
for (const service of res.body.services) {
|
||||
expect(VALID_STATUSES).to.include(service.status);
|
||||
}
|
||||
|
||||
const byKey = Object.fromEntries(res.body.services.map((s: { key: string }) => [s.key, s]));
|
||||
// The DB is up during the test run, and the API is answering us.
|
||||
expect(byKey.postgres.status).to.equal("up");
|
||||
expect(byKey.api.status).to.equal("up");
|
||||
// No heartbeat has ever been recorded (resetDatabase truncated it).
|
||||
expect(byKey["tech-step-llm-worker"].status).to.equal("unknown");
|
||||
});
|
||||
|
||||
it("reports the worker as up once it has sent a recent heartbeat", async function () {
|
||||
if (!adminSecretConfigured || !workerSecretConfigured) {
|
||||
// biome-ignore lint/suspicious/noExplicitAny: see above.
|
||||
(this as any).skip();
|
||||
return;
|
||||
}
|
||||
await request(app)
|
||||
.post("/internal/tech-steps/heartbeat")
|
||||
.set(SECRET_HEADER, env.INTERNAL_WORKER_SECRET as string)
|
||||
.send({ event: "job", job: "transform-corrections", ok: true, counts: { suggestions: 0 } });
|
||||
|
||||
const { email, password } = await seedAdmin();
|
||||
const agent = request.agent(app);
|
||||
await agent.post("/admin/auth/login").send({ email, password });
|
||||
|
||||
const res = await agent.get("/admin/monitoring");
|
||||
const worker = res.body.services.find(
|
||||
(s: { key: string }) => s.key === "tech-step-llm-worker",
|
||||
);
|
||||
expect(worker.status).to.equal("up");
|
||||
expect(worker.lastResult.job).to.equal("transform-corrections");
|
||||
expect(worker.lastRunAt).to.be.a("string");
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -28,3 +28,22 @@ export const getMetricsSchema = z.object({
|
|||
});
|
||||
/** Inferred TS type for {@link getMetricsSchema}'s validated output. */
|
||||
export type GetMetricsInput = z.infer<typeof getMetricsSchema>;
|
||||
|
||||
/**
|
||||
* Payload of `POST /internal/tech-steps/heartbeat` — `services/tech-step-llm-worker`
|
||||
* (which has no inbound HTTP of its own) reporting that it's alive. Sent on
|
||||
* `boot`, on every scheduler `tick`, and after each `job` (with that job's
|
||||
* name, outcome and counts). The worker key is fixed server-side (only one
|
||||
* worker exists), so it isn't in the payload.
|
||||
*/
|
||||
export const workerHeartbeatSchema = z.object({
|
||||
event: z.enum(["boot", "tick", "job"]),
|
||||
/** The job that just ran — present only when `event === "job"`. */
|
||||
job: z.string().max(100).optional(),
|
||||
/** Whether that job succeeded — present only when `event === "job"`. */
|
||||
ok: z.boolean().optional(),
|
||||
/** Small `{ label: number }` summary of that job (e.g. `{ suggestions: 3 }`). */
|
||||
counts: z.record(z.string(), z.number()).optional(),
|
||||
});
|
||||
/** Inferred TS type for {@link workerHeartbeatSchema}'s validated output. */
|
||||
export type WorkerHeartbeatInput = z.infer<typeof workerHeartbeatSchema>;
|
||||
|
|
|
|||
|
|
@ -85,3 +85,34 @@ export interface MetricsView {
|
|||
};
|
||||
events: MetricsEventSeries[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Health of one thing the admin monitoring board watches:
|
||||
* - `"up"` — reachable and healthy.
|
||||
* - `"degraded"` — reachable but not fully OK (e.g. the worker's last heartbeat
|
||||
* is old, or a job it ran failed).
|
||||
* - `"down"` — unreachable / erroring.
|
||||
* - `"unknown"` — never observed (e.g. the worker has never sent a heartbeat).
|
||||
*/
|
||||
export type ServiceStatus = "up" | "degraded" | "down" | "unknown";
|
||||
|
||||
/** One row of `GET /admin/monitoring` — a single probed target. */
|
||||
export interface ServiceHealthView {
|
||||
/** Stable id — `"postgres"`, `"api"`, `"intent-service"`, `"tech-step-llm-worker"`. Resolved to a label client-side via `admin.monitoring.service.<key>`. */
|
||||
key: string;
|
||||
status: ServiceStatus;
|
||||
/** Round-trip of the probe in ms, or `null` when there was nothing to time (the worker, read from a stored heartbeat). */
|
||||
latencyMs: number | null;
|
||||
/** Short human-readable extra (`"HTTP 200"`, `"uptime 3h 12m"`, `"dernier battement il y a 9 j"`). */
|
||||
detail: string | null;
|
||||
checkedAt: string;
|
||||
/** Worker only — when it last actually ran a job, and that job's summary. */
|
||||
lastRunAt?: string | null;
|
||||
lastResult?: { job?: string; ok?: boolean; counts?: Record<string, number> } | null;
|
||||
}
|
||||
|
||||
/** Response of `GET /admin/monitoring` — an actively-probed status board. */
|
||||
export interface MonitoringView {
|
||||
generatedAt: string;
|
||||
services: ServiceHealthView[];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -94,6 +94,35 @@ export function getPendingCorrections(limit: number): Promise<PendingCorrection[
|
|||
return request(`/internal/tech-steps/pending-corrections?${params}`);
|
||||
}
|
||||
|
||||
/** Payload of `POST /internal/tech-steps/heartbeat` — mirrors `workerHeartbeatSchema` (`packages/shared`), duplicated here for the same "outside the workspace" reason as {@link AuditClause}. */
|
||||
export interface HeartbeatPayload {
|
||||
event: "boot" | "tick" | "job";
|
||||
job?: string;
|
||||
ok?: boolean;
|
||||
counts?: Record<string, number>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Best-effort liveness ping to `apps/api` so the admin monitoring board can
|
||||
* see this worker (which has no inbound HTTP surface of its own). Sent on
|
||||
* boot, on every scheduler tick, and after each job. **Never throws** — a
|
||||
* failed heartbeat must never break or abort a run; it's logged and
|
||||
* swallowed here.
|
||||
*/
|
||||
export async function postHeartbeat(payload: HeartbeatPayload): Promise<void> {
|
||||
try {
|
||||
await request("/internal/tech-steps/heartbeat", {
|
||||
method: "POST",
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
} catch (err) {
|
||||
console.warn(
|
||||
"[tech-step-llm-worker] heartbeat failed:",
|
||||
err instanceof Error ? err.message : err,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** Submits a batch of suggestions — a no-op (resolves immediately) if `suggestions` is empty, so a job with nothing to report doesn't need its own guard at every call site. */
|
||||
export function postTrainingSuggestions(
|
||||
suggestions: TrainingSuggestionInput[],
|
||||
|
|
|
|||
|
|
@ -1,6 +1,12 @@
|
|||
import { postHeartbeat } from "./api-client.js";
|
||||
import { env } from "./config.js";
|
||||
import { runOnce, startScheduler } from "./scheduler.js";
|
||||
|
||||
// Tell `apps/api` we're alive as early as possible — before either the
|
||||
// one-shot run or the cron loop — so the admin monitoring board reflects a
|
||||
// fresh deploy immediately, not only after the first scheduled fire.
|
||||
await postHeartbeat({ event: "boot" });
|
||||
|
||||
/**
|
||||
* Entrypoint — `RUN_ONCE=true` runs both jobs a single time and exits
|
||||
* (manual/CI-triggered invocation, `pnpm start`), otherwise starts the
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import cron from "node-cron";
|
||||
import { postHeartbeat } from "./api-client.js";
|
||||
import { env } from "./config.js";
|
||||
import { runAuditLowConfidenceJob } from "./jobs/audit-low-confidence.js";
|
||||
import { runTransformCorrectionsJob } from "./jobs/transform-corrections.js";
|
||||
|
|
@ -27,11 +28,23 @@ export async function runOnce(): Promise<void> {
|
|||
limit: env.TECH_STEP_WORKER_BATCH_LIMIT,
|
||||
});
|
||||
console.info(`[tech-step-llm-worker] audit-low-confidence: ${auditCount} suggestion(s)`);
|
||||
await postHeartbeat({
|
||||
event: "job",
|
||||
job: "audit-low-confidence",
|
||||
ok: true,
|
||||
counts: { suggestions: auditCount },
|
||||
});
|
||||
|
||||
const correctionCount = await runTransformCorrectionsJob(llm, {
|
||||
limit: env.TECH_STEP_WORKER_BATCH_LIMIT,
|
||||
});
|
||||
console.info(`[tech-step-llm-worker] transform-corrections: ${correctionCount} suggestion(s)`);
|
||||
await postHeartbeat({
|
||||
event: "job",
|
||||
job: "transform-corrections",
|
||||
ok: true,
|
||||
counts: { suggestions: correctionCount },
|
||||
});
|
||||
} finally {
|
||||
await llm.dispose();
|
||||
}
|
||||
|
|
@ -49,8 +62,11 @@ export async function runOnce(): Promise<void> {
|
|||
export function startScheduler(): void {
|
||||
console.info(`[tech-step-llm-worker] scheduling runs on "${env.TECH_STEP_WORKER_CRON}"`);
|
||||
cron.schedule(env.TECH_STEP_WORKER_CRON, () => {
|
||||
// Prove liveness even for a fire that then fails inside `runOnce`.
|
||||
void postHeartbeat({ event: "tick" });
|
||||
runOnce().catch((err: unknown) => {
|
||||
console.error("[tech-step-llm-worker] scheduled run failed:", err);
|
||||
void postHeartbeat({ event: "job", ok: false });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -387,8 +387,22 @@ est un `String` libre (`"user.signup"`, `"recipe.imported"`, `"recipe.created"`,
|
|||
ligne, **sans migration**. `AnalyticsEvent.actorId` n'a **pas** de FK (un
|
||||
évènement est un fait historique qui survit au compte qu'il décrit).
|
||||
|
||||
*(La table `WorkerHeartbeat` est créée par la même migration mais n'est
|
||||
câblée que par le monitoring — section à venir.)*
|
||||
**Monitoring** (`admin-monitoring.service.ts`, `GET /admin/monitoring`,
|
||||
`requireAdmin`) — sonde active, chaque cible bornée à ~2 s, latence mesurée :
|
||||
Postgres (`SELECT 1`), l'API elle-même (uptime/RSS), `tech-step-intent-service`
|
||||
(`GET /health`, sans secret), et le **worker LLM** via sa ligne
|
||||
`WorkerHeartbeat`. `MonitoringView { services: ServiceHealthView[] }`, statut
|
||||
`up | degraded | down | unknown` ; une sonde `down` ne fait jamais échouer les
|
||||
autres ni l'endpoint. Le front (`MonitoringPage`) re-poll toutes les 15 s.
|
||||
|
||||
Le worker (`services/tech-step-llm-worker`) n'a **aucune** surface HTTP
|
||||
entrante — il devient observable via `POST /internal/tech-steps/heartbeat`
|
||||
(`requireInternalWorker`, `tech-step-worker.service.ts`'s
|
||||
`recordWorkerHeartbeat` → upsert `WorkerHeartbeat`, clé fixe
|
||||
`"tech-step-llm-worker"`). Le worker l'appelle au boot, à chaque tick du
|
||||
scheduler, et après chaque job (avec `job`/`ok`/`counts`) — best-effort, un
|
||||
heartbeat en échec ne casse jamais un run. Seuils d'âge : > 8 j ⇒ `degraded`,
|
||||
> 21 j ⇒ `down` (cron par défaut hebdomadaire).
|
||||
|
||||
---
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue