mediaDrop/src/routes/upload-chunked.ts
Nicolas 64a38b1e41
All checks were successful
Publier l'image Docker / test (push) Successful in 17s
Publier l'image Docker / publish (push) Successful in 11s
Upload fragmenté : pool de connexions partagé et configurable
Remplace les 3 workers par fichier par un pool global de morceaux en vol,
réparti équitablement entre les fichiers fragmentés actifs : 1 fichier
utilise tout le pool, N fichiers en reçoivent une part chacun
(ceil(taille / N)), la capacité libérée par un fichier terminé est
redistribuée. Un fichier en cours d'init compte dans la part pour ne pas
être affamé ; les morceaux déjà en vol ne sont jamais interrompus.

La taille du pool est réglable via `chunkPoolSize` (config.json, défaut
30, borné à 256) et transmise au navigateur par un nouveau `GET /config`.

Corps des morceaux repassé en streaming (writeChunkAt sur un flux + garde
`maxBytes`) : mémoire ~constante même à pool élevé, plus de bufferisation
de 16 Mo par requête.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-10 19:12:04 +02:00

189 lines
5.5 KiB
TypeScript

import { stat } from 'node:fs/promises';
import type { Readable } from 'node:stream';
import type { FastifyPluginCallback, FastifyReply } from 'fastify';
import { PayloadTooLargeError, ValidationError } from '../errors.js';
import { isAllowedMedia } from '../lib/media-types.js';
import {
createEmptyFile,
finalizeFile,
removeFile,
resolveDestination,
writeChunkAt,
} from '../lib/storage.js';
import { resolvePlacement, resolveTarget } from '../lib/targets.js';
import { requireSession } from '../session.js';
const MIN_CHUNK = 1024 * 1024;
const MAX_CHUNK = 32 * 1024 * 1024;
const DEFAULT_CHUNK = 16 * 1024 * 1024;
const PART_SUFFIX = '.mediadrop-part';
interface InitBody {
target?: unknown;
path?: unknown;
name?: unknown;
size?: unknown;
type?: unknown;
chunkSize?: unknown;
}
interface InitResponse {
id: string;
chunkSize: number;
chunkCount: number;
}
interface ChunkAck {
received: number;
total: number;
}
interface CompleteResponse {
target: string;
storedName: string;
size: number;
mimeType: string;
}
function positiveInt(value: unknown, name: string): number {
if (typeof value !== 'number' || !Number.isInteger(value) || value <= 0) {
throw new ValidationError(`${name} doit être un entier positif`);
}
return value;
}
function nonEmptyString(value: unknown, fallback: string): string {
return typeof value === 'string' && value.length > 0 ? value : fallback;
}
function clampChunkSize(value: unknown): number {
if (value === undefined) {
return DEFAULT_CHUNK;
}
return Math.min(MAX_CHUNK, Math.max(MIN_CHUNK, positiveInt(value, 'chunkSize')));
}
/**
* Upload fragmenté et reprenable :
* - `POST /upload/init` crée une session, renvoie `{ id, chunkSize, chunkCount }`
* - `PUT /upload/:id/:index` envoie un morceau (corps brut octet-stream)
* - `POST /upload/:id/complete` assemble et finalise, renvoie 201 + métadonnées
* - `DELETE /upload/:id` abandonne (supprime le fichier partiel)
*/
export const chunkedUploadRoutes: FastifyPluginCallback = (app, _options, done) => {
const cfg = app.appConfig;
app.post(
'/upload/init',
{ preHandler: requireSession },
async (request, reply): Promise<InitResponse> => {
const body = (request.body ?? {}) as InitBody;
const target = resolveTarget(cfg.targets, body.target);
const placement = resolvePlacement(target, body.path);
const mimeType = nonEmptyString(body.type, 'application/octet-stream');
const totalSize = positiveInt(body.size, 'size');
if (totalSize > cfg.maxFileSizeBytes) {
throw new PayloadTooLargeError();
}
if (!isAllowedMedia(mimeType, cfg.allowedMimeTypes)) {
throw new ValidationError(`Type de média non autorisé : ${mimeType}`);
}
const chunkSize = clampChunkSize(body.chunkSize);
const chunkCount = Math.ceil(totalSize / chunkSize);
const { storedName, destination } = await resolveDestination({
targetDir: target.dir,
originalName: nonEmptyString(body.name, 'fichier'),
mimeType,
...(placement !== undefined ? { placement } : {}),
});
const partPath = destination + PART_SUFFIX;
await createEmptyFile(partPath);
const session = app.uploadSessions.create({
target,
storedName,
mimeType,
partPath,
finalPath: destination,
totalSize,
chunkSize,
chunkCount,
});
reply.status(201);
return { id: session.id, chunkSize, chunkCount };
},
);
app.put(
'/upload/:id/:index',
{ preHandler: requireSession },
async (request): Promise<ChunkAck> => {
const { id, index } = request.params as { id: string; index: string };
const session = app.uploadSessions.use(id);
const chunkIndex = Number(index);
if (!Number.isInteger(chunkIndex) || chunkIndex < 0 || chunkIndex >= session.chunkCount) {
throw new ValidationError('Index de morceau hors limites');
}
await writeChunkAt({
source: request.body as Readable,
filePath: session.partPath,
offset: chunkIndex * session.chunkSize,
maxBytes: session.chunkSize,
});
session.received.add(chunkIndex);
return { received: session.received.size, total: session.chunkCount };
},
);
app.post(
'/upload/:id/complete',
{ preHandler: requireSession },
async (request, reply): Promise<CompleteResponse> => {
const { id } = request.params as { id: string };
const session = app.uploadSessions.use(id);
if (session.received.size !== session.chunkCount) {
throw new ValidationError('Upload incomplet : des morceaux manquent');
}
const { size } = await stat(session.partPath);
if (size !== session.totalSize) {
throw new ValidationError('Taille finale incohérente');
}
await finalizeFile(session.partPath, session.finalPath);
app.uploadSessions.remove(id);
reply.status(201);
return {
target: session.target.name,
storedName: session.storedName,
size: session.totalSize,
mimeType: session.mimeType,
};
},
);
app.delete(
'/upload/:id',
{ preHandler: requireSession },
async (request, reply): Promise<FastifyReply> => {
const { id } = request.params as { id: string };
const session = app.uploadSessions.remove(id);
if (session !== undefined) {
await removeFile(session.partPath);
}
return reply.status(204).send();
},
);
done();
};