From 41dc623a28dd1908a26ff9c80f6972c72a1957ab Mon Sep 17 00:00:00 2001 From: stringsplit <77242831+nbitzz@users.noreply.github.com> Date: Mon, 4 Mar 2024 17:12:20 -0800 Subject: [PATCH] back to formidable --- package-lock.json | 110 +++++++++++++++++++++++++ package.json | 2 + src/server/lib/files.ts | 2 +- src/server/routes/api/v0/primaryApi.ts | 91 ++++++++++++++------ 4 files changed, 177 insertions(+), 28 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5d00b38..2a9c518 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,6 +21,7 @@ "cookie-parser": "^1.4.6", "dotenv": "^16.0.2", "express": "^4.18.1", + "formidable": "^3.5.1", "hono": "^3.8.3", "multer": "^1.4.5-lts.1", "node-fetch": "^3.3.2", @@ -31,6 +32,7 @@ "@sveltejs/vite-plugin-svelte": "^2.4.6", "@types/bytes": "^3.1.1", "@types/cookie-parser": "^1.4.3", + "@types/formidable": "^3.4.5", "@types/range-parser": "^1.2.6", "discord-api-types": "^0.37.61", "sass": "^1.57.1", @@ -545,6 +547,15 @@ "@types/range-parser": "*" } }, + "node_modules/@types/formidable": { + "version": "3.4.5", + "resolved": "https://registry.npmjs.org/@types/formidable/-/formidable-3.4.5.tgz", + "integrity": "sha512-s7YPsNVfnsng5L8sKnG/Gbb2tiwwJTY1conOkJzTMRvJAlLFW1nEua+ADsJQu8N1c0oTHx9+d5nqg10WuT9gHQ==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/mime": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-3.0.1.tgz", @@ -625,6 +636,11 @@ "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" }, + "node_modules/asap": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", + "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==" + }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -886,6 +902,15 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/dezalgo": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz", + "integrity": "sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==", + "dependencies": { + "asap": "^2.0.0", + "wrappy": "1" + } + }, "node_modules/discord-api-types": { "version": "0.37.61", "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.37.61.tgz", @@ -1098,6 +1123,19 @@ "node": ">=12.20.0" } }, + "node_modules/formidable": { + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/formidable/-/formidable-3.5.1.tgz", + "integrity": "sha512-WJWKelbRHN41m5dumb0/k8TeAx7Id/y3a+Z7QfhxP/htI9Js5zYaEDtG8uMgG0vM0lOlqnmjE99/kfpOYi/0Og==", + "dependencies": { + "dezalgo": "^1.0.4", + "hexoid": "^1.0.0", + "once": "^1.4.0" + }, + "funding": { + "url": "https://ko-fi.com/tunnckoCore/commissions" + } + }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -1180,6 +1218,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/hexoid": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/hexoid/-/hexoid-1.0.0.tgz", + "integrity": "sha512-QFLV0taWQOZtvIRIAdBChesmogZrtuXvVWsFHZTk2SU+anspqZ2vMnoLg7IE1+Uk16N19APic1BuF8bC8c2m5g==", + "engines": { + "node": ">=8" + } + }, "node_modules/hono": { "version": "3.8.3", "resolved": "https://registry.npmjs.org/hono/-/hono-3.8.3.tgz", @@ -1495,6 +1541,14 @@ "node": ">= 0.8" } }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dependencies": { + "wrappy": "1" + } + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -1969,6 +2023,11 @@ "node": ">= 8" } }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + }, "node_modules/xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", @@ -2255,6 +2314,15 @@ "@types/range-parser": "*" } }, + "@types/formidable": { + "version": "3.4.5", + "resolved": "https://registry.npmjs.org/@types/formidable/-/formidable-3.4.5.tgz", + "integrity": "sha512-s7YPsNVfnsng5L8sKnG/Gbb2tiwwJTY1conOkJzTMRvJAlLFW1nEua+ADsJQu8N1c0oTHx9+d5nqg10WuT9gHQ==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/mime": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-3.0.1.tgz", @@ -2329,6 +2397,11 @@ "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" }, + "asap": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", + "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==" + }, "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -2522,6 +2595,15 @@ "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==" }, + "dezalgo": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz", + "integrity": "sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==", + "requires": { + "asap": "^2.0.0", + "wrappy": "1" + } + }, "discord-api-types": { "version": "0.37.61", "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.37.61.tgz", @@ -2676,6 +2758,16 @@ "fetch-blob": "^3.1.2" } }, + "formidable": { + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/formidable/-/formidable-3.5.1.tgz", + "integrity": "sha512-WJWKelbRHN41m5dumb0/k8TeAx7Id/y3a+Z7QfhxP/htI9Js5zYaEDtG8uMgG0vM0lOlqnmjE99/kfpOYi/0Og==", + "requires": { + "dezalgo": "^1.0.4", + "hexoid": "^1.0.0", + "once": "^1.4.0" + } + }, "forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -2730,6 +2822,11 @@ "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==" }, + "hexoid": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/hexoid/-/hexoid-1.0.0.tgz", + "integrity": "sha512-QFLV0taWQOZtvIRIAdBChesmogZrtuXvVWsFHZTk2SU+anspqZ2vMnoLg7IE1+Uk16N19APic1BuF8bC8c2m5g==" + }, "hono": { "version": "3.8.3", "resolved": "https://registry.npmjs.org/hono/-/hono-3.8.3.tgz", @@ -2941,6 +3038,14 @@ "ee-first": "1.1.1" } }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "requires": { + "wrappy": "1" + } + }, "parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -3246,6 +3351,11 @@ "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==" }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + }, "xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", diff --git a/package.json b/package.json index 6d91801..86dd14f 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "cookie-parser": "^1.4.6", "dotenv": "^16.0.2", "express": "^4.18.1", + "formidable": "^3.5.1", "hono": "^3.8.3", "multer": "^1.4.5-lts.1", "node-fetch": "^3.3.2", @@ -40,6 +41,7 @@ "@sveltejs/vite-plugin-svelte": "^2.4.6", "@types/bytes": "^3.1.1", "@types/cookie-parser": "^1.4.3", + "@types/formidable": "^3.4.5", "@types/range-parser": "^1.2.6", "discord-api-types": "^0.37.61", "sass": "^1.57.1", diff --git a/src/server/lib/files.ts b/src/server/lib/files.ts index d5abcc8..07ab3bd 100644 --- a/src/server/lib/files.ts +++ b/src/server/lib/files.ts @@ -198,7 +198,7 @@ export class UploadStream extends Writable { if (this.errored) throw this.error if (!this.writableFinished) { let err = Error("attempted to commit file when the stream was still unfinished") - this.destroy(err); throw err + if (!this.destroyed) {this.destroy(err)}; throw err } // Perform checks diff --git a/src/server/routes/api/v0/primaryApi.ts b/src/server/routes/api/v0/primaryApi.ts index f0c8c25..6c7c3b1 100644 --- a/src/server/routes/api/v0/primaryApi.ts +++ b/src/server/routes/api/v0/primaryApi.ts @@ -10,13 +10,16 @@ import { getAccount, requiresPermissions } from "../../../lib/middleware.js" import FormDataParser, { Field } from "../../../lib/formdata.js" import {Readable} from "node:stream" import {ReadableStream as StreamWebReadable} from "node:stream/web" +import formidable from "formidable" +import { HttpBindings } from "@hono/node-server" export let primaryApi = new Hono<{ Variables: { account: Accounts.Account - } + }, + Bindings: HttpBindings }>() -primaryApi.use(getAccount) +primaryApi.all("*", getAccount) export default function (files: Files) { primaryApi.get( @@ -98,43 +101,77 @@ export default function (files: Files) { "/upload", requiresPermissions("upload"), (ctx) => { return new Promise((resolve,reject) => { - let acc = ctx.get("account") as Accounts.Account + ctx.env.incoming.removeAllListeners("data") // remove hono's buffering + console.log('awawa') + let acc = ctx.get("account") as Accounts.Account | undefined if (!ctx.req.header("Content-Type")?.startsWith("multipart/form-data")) { ctx.status(400) resolve(ctx.body("[err] must be multipart/form-data")) + return } if (!ctx.req.raw.body) { ctx.status(400) resolve(ctx.body("[err] body must be supplied")) + return + } + console.log('awawawawa') + + let file = files.createWriteStream(acc?.id) + let parser = formidable({ + maxFieldsSize: 65536, + maxFileSize: files.config.maxDiscordFileSize*files.config.maxDiscordFiles, + maxFiles: 1 + }) + + parser.onPart = function(part) { + console.log(part) + if (part.originalFilename == "" || !part.mimetype) { + parser._handlePart(part); return + } + // lol + if (part.name == "file") { + file.on("drain", () => { + ctx.env.incoming.resume() + }) + part.addListener("data", (data: Buffer) => { + if (!file.write(data)) + ctx.env.incoming.pause() + }) + } } - let file = files.createWriteStream(acc.id) - let formDataParser = new FormDataParser('') - - Readable.fromWeb(ctx.req.raw.body as StreamWebReadable) - .pipe(formDataParser) - .on("data", async (field: Field) => { - if (field.headers["content-disposition"]?.filename) { - field.pipe(file) - } else { - switch(field.headers["content-disposition"]?.name) { - case "uploadId": - file.setUploadId((await field.collect(65536).catch(e => {formDataParser.destroy(new WebError(413, e.message))}))?.toString() || "") - } - } - }) - .on("end", async () => { - if (!file.writableEnded) await new Promise((res, rej) => {file.once("finish", res); file.once("error", res)}) - if (file.errored || !(await file.commit().catch(e => file.error = e))) { - ctx.status(file.error instanceof WebError ? file.error.statusCode : 500) - resolve(`[err] ${file.error instanceof WebError ? file.error.message : file.error?.toString()}`) - return - } - - resolve(ctx.body(file.uploadId!)) + parser.on("field", (k,v) => { + console.log(k,v) + if (k == "uploadId") + file.setUploadId(v) + }) + + parser.parse(ctx.env.incoming).catch(e => console.log(e)) + console.log("Parsing") + + parser.on('error', (err) => { + if ("httpCode" in err) + ctx.status(err.httpCode) + else ctx.status(400) + resolve(ctx.body(err.message)) + }) + + file.on("error", (err) => { + if (err instanceof WebError) + ctx.status(err.statusCode) + resolve(ctx.body(err?.message)) + }) + + file.on("finish", () => { + file.commit().then(id => resolve(ctx.body(id!))).catch((err) => { + if (err instanceof WebError) + ctx.status(err.statusCode) + resolve(ctx.body(err?.message)) }) + }) + })} ) /*