back to formidable

2024-11-21 21:36:26 -08:00 · 2024-03-04 17:12:20 -08:00 · 2024-03-04 17:12:20 -08:00 · 41dc623a28
parent 53c9c5fe58
commit 41dc623a28
4 changed files with 177 additions and 28 deletions
--- a/package-lock.json
+++ b/package-lock.json
@ -21,6 +21,7 @@
        "cookie-parser": "^1.4.6",
        "dotenv": "^16.0.2",
        "express": "^4.18.1",
+        "formidable": "^3.5.1",
        "hono": "^3.8.3",
        "multer": "^1.4.5-lts.1",
        "node-fetch": "^3.3.2",
@ -31,6 +32,7 @@
        "@sveltejs/vite-plugin-svelte": "^2.4.6",
        "@types/bytes": "^3.1.1",
        "@types/cookie-parser": "^1.4.3",
+        "@types/formidable": "^3.4.5",
        "@types/range-parser": "^1.2.6",
        "discord-api-types": "^0.37.61",
        "sass": "^1.57.1",
@ -545,6 +547,15 @@
        "@types/range-parser": "*"
      }
    },
+    "node_modules/@types/formidable": {
+      "version": "3.4.5",
+      "resolved": "https://registry.npmjs.org/@types/formidable/-/formidable-3.4.5.tgz",
+      "integrity": "sha512-s7YPsNVfnsng5L8sKnG/Gbb2tiwwJTY1conOkJzTMRvJAlLFW1nEua+ADsJQu8N1c0oTHx9+d5nqg10WuT9gHQ==",
+      "dev": true,
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
    "node_modules/@types/mime": {
      "version": "3.0.1",
      "resolved": "https://registry.npmjs.org/@types/mime/-/mime-3.0.1.tgz",
@ -625,6 +636,11 @@
      "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
      "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg=="
    },
+    "node_modules/asap": {
+      "version": "2.0.6",
+      "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz",
+      "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA=="
+    },
    "node_modules/asynckit": {
      "version": "0.4.0",
      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
@ -886,6 +902,15 @@
        "npm": "1.2.8000 || >= 1.4.16"
      }
    },
+    "node_modules/dezalgo": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz",
+      "integrity": "sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==",
+      "dependencies": {
+        "asap": "^2.0.0",
+        "wrappy": "1"
+      }
+    },
    "node_modules/discord-api-types": {
      "version": "0.37.61",
      "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.37.61.tgz",
@ -1098,6 +1123,19 @@
        "node": ">=12.20.0"
      }
    },
+    "node_modules/formidable": {
+      "version": "3.5.1",
+      "resolved": "https://registry.npmjs.org/formidable/-/formidable-3.5.1.tgz",
+      "integrity": "sha512-WJWKelbRHN41m5dumb0/k8TeAx7Id/y3a+Z7QfhxP/htI9Js5zYaEDtG8uMgG0vM0lOlqnmjE99/kfpOYi/0Og==",
+      "dependencies": {
+        "dezalgo": "^1.0.4",
+        "hexoid": "^1.0.0",
+        "once": "^1.4.0"
+      },
+      "funding": {
+        "url": "https://ko-fi.com/tunnckoCore/commissions"
+      }
+    },
    "node_modules/forwarded": {
      "version": "0.2.0",
      "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
@ -1180,6 +1218,14 @@
        "url": "https://github.com/sponsors/ljharb"
      }
    },
+    "node_modules/hexoid": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/hexoid/-/hexoid-1.0.0.tgz",
+      "integrity": "sha512-QFLV0taWQOZtvIRIAdBChesmogZrtuXvVWsFHZTk2SU+anspqZ2vMnoLg7IE1+Uk16N19APic1BuF8bC8c2m5g==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
    "node_modules/hono": {
      "version": "3.8.3",
      "resolved": "https://registry.npmjs.org/hono/-/hono-3.8.3.tgz",
@ -1495,6 +1541,14 @@
        "node": ">= 0.8"
      }
    },
+    "node_modules/once": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
+      "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
+      "dependencies": {
+        "wrappy": "1"
+      }
+    },
    "node_modules/parseurl": {
      "version": "1.3.3",
      "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
@ -1969,6 +2023,11 @@
        "node": ">= 8"
      }
    },
+    "node_modules/wrappy": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
+      "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="
+    },
    "node_modules/xtend": {
      "version": "4.0.2",
      "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
@ -2255,6 +2314,15 @@
        "@types/range-parser": "*"
      }
    },
+    "@types/formidable": {
+      "version": "3.4.5",
+      "resolved": "https://registry.npmjs.org/@types/formidable/-/formidable-3.4.5.tgz",
+      "integrity": "sha512-s7YPsNVfnsng5L8sKnG/Gbb2tiwwJTY1conOkJzTMRvJAlLFW1nEua+ADsJQu8N1c0oTHx9+d5nqg10WuT9gHQ==",
+      "dev": true,
+      "requires": {
+        "@types/node": "*"
+      }
+    },
    "@types/mime": {
      "version": "3.0.1",
      "resolved": "https://registry.npmjs.org/@types/mime/-/mime-3.0.1.tgz",
@ -2329,6 +2397,11 @@
      "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
      "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg=="
    },
+    "asap": {
+      "version": "2.0.6",
+      "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz",
+      "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA=="
+    },
    "asynckit": {
      "version": "0.4.0",
      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
@ -2522,6 +2595,15 @@
      "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz",
      "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg=="
    },
+    "dezalgo": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz",
+      "integrity": "sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==",
+      "requires": {
+        "asap": "^2.0.0",
+        "wrappy": "1"
+      }
+    },
    "discord-api-types": {
      "version": "0.37.61",
      "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.37.61.tgz",
@ -2676,6 +2758,16 @@
        "fetch-blob": "^3.1.2"
      }
    },
+    "formidable": {
+      "version": "3.5.1",
+      "resolved": "https://registry.npmjs.org/formidable/-/formidable-3.5.1.tgz",
+      "integrity": "sha512-WJWKelbRHN41m5dumb0/k8TeAx7Id/y3a+Z7QfhxP/htI9Js5zYaEDtG8uMgG0vM0lOlqnmjE99/kfpOYi/0Og==",
+      "requires": {
+        "dezalgo": "^1.0.4",
+        "hexoid": "^1.0.0",
+        "once": "^1.4.0"
+      }
+    },
    "forwarded": {
      "version": "0.2.0",
      "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
@ -2730,6 +2822,11 @@
      "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz",
      "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A=="
    },
+    "hexoid": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/hexoid/-/hexoid-1.0.0.tgz",
+      "integrity": "sha512-QFLV0taWQOZtvIRIAdBChesmogZrtuXvVWsFHZTk2SU+anspqZ2vMnoLg7IE1+Uk16N19APic1BuF8bC8c2m5g=="
+    },
    "hono": {
      "version": "3.8.3",
      "resolved": "https://registry.npmjs.org/hono/-/hono-3.8.3.tgz",
@ -2941,6 +3038,14 @@
        "ee-first": "1.1.1"
      }
    },
+    "once": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
+      "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
+      "requires": {
+        "wrappy": "1"
+      }
+    },
    "parseurl": {
      "version": "1.3.3",
      "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
@ -3246,6 +3351,11 @@
      "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz",
      "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q=="
    },
+    "wrappy": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
+      "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="
+    },
    "xtend": {
      "version": "4.0.2",
      "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
--- a/package.json
+++ b/package.json
@ -30,6 +30,7 @@
    "cookie-parser": "^1.4.6",
    "dotenv": "^16.0.2",
    "express": "^4.18.1",
+    "formidable": "^3.5.1",
    "hono": "^3.8.3",
    "multer": "^1.4.5-lts.1",
    "node-fetch": "^3.3.2",
@ -40,6 +41,7 @@
    "@sveltejs/vite-plugin-svelte": "^2.4.6",
    "@types/bytes": "^3.1.1",
    "@types/cookie-parser": "^1.4.3",
+    "@types/formidable": "^3.4.5",
    "@types/range-parser": "^1.2.6",
    "discord-api-types": "^0.37.61",
    "sass": "^1.57.1",
--- a/src/server/lib/files.ts
+++ b/src/server/lib/files.ts
@ -198,7 +198,7 @@ export class UploadStream extends Writable {
        if (this.errored) throw this.error
        if (!this.writableFinished) {
            let err = Error("attempted to commit file when the stream was still unfinished")
-            this.destroy(err); throw err
+            if (!this.destroyed) {this.destroy(err)}; throw err
        }

        // Perform checks
--- a/src/server/routes/api/v0/primaryApi.ts
+++ b/src/server/routes/api/v0/primaryApi.ts
@ -10,13 +10,16 @@ import { getAccount, requiresPermissions } from "../../../lib/middleware.js"
 import FormDataParser, { Field } from "../../../lib/formdata.js"
 import {Readable} from "node:stream"
 import {ReadableStream as StreamWebReadable} from "node:stream/web"
+import formidable from "formidable"
+import { HttpBindings } from "@hono/node-server"
 export let primaryApi = new Hono<{
    Variables: {
        account: Accounts.Account
-    }
+    },
+    Bindings: HttpBindings
 }>()

-primaryApi.use(getAccount)
+primaryApi.all("*", getAccount)

 export default function (files: Files) {
    primaryApi.get(
@ -98,43 +101,77 @@ export default function (files: Files) {
        "/upload",
        requiresPermissions("upload"),
        (ctx) => { return new Promise((resolve,reject) => {
-            let acc = ctx.get("account") as Accounts.Account
+            ctx.env.incoming.removeAllListeners("data") // remove hono's buffering
+            console.log('awawa')
+            let acc = ctx.get("account") as Accounts.Account | undefined

            if (!ctx.req.header("Content-Type")?.startsWith("multipart/form-data")) {
                ctx.status(400)
                resolve(ctx.body("[err] must be multipart/form-data"))
+                return
            }

            if (!ctx.req.raw.body) {
                ctx.status(400)
                resolve(ctx.body("[err] body must be supplied"))
-            }
-
-            let file = files.createWriteStream(acc.id)
-            let formDataParser = new FormDataParser('')
-            
-            Readable.fromWeb(ctx.req.raw.body as StreamWebReadable)
-                .pipe(formDataParser)
-                .on("data", async (field: Field) => {
-                    if (field.headers["content-disposition"]?.filename) {
-                        field.pipe(file)
-                    } else {
-                        switch(field.headers["content-disposition"]?.name) {
-                            case "uploadId":
-                                file.setUploadId((await field.collect(65536).catch(e => {formDataParser.destroy(new WebError(413, e.message))}))?.toString() || "")
-                        }
-                    }
-                })
-                .on("end", async () => {
-                    if (!file.writableEnded) await new Promise((res, rej) => {file.once("finish", res); file.once("error", res)})
-                    if (file.errored || !(await file.commit().catch(e => file.error = e))) {
-                        ctx.status(file.error instanceof WebError ? file.error.statusCode : 500)
-                        resolve(`[err] ${file.error instanceof WebError ? file.error.message : file.error?.toString()}`)
                return
            }
+            console.log('awawawawa')

-                    resolve(ctx.body(file.uploadId!))
+            let file = files.createWriteStream(acc?.id)
+            let parser = formidable({
+                maxFieldsSize: 65536,
+                maxFileSize: files.config.maxDiscordFileSize*files.config.maxDiscordFiles,
+                maxFiles: 1
            })
+
+            parser.onPart = function(part) {
+                console.log(part)
+                if (part.originalFilename == "" || !part.mimetype) {
+                    parser._handlePart(part); return
+                }
+                // lol
+                if (part.name == "file") {
+                    file.on("drain", () => {
+                        ctx.env.incoming.resume()
+                    })
+                    part.addListener("data", (data: Buffer) => {
+                        if (!file.write(data))
+                            ctx.env.incoming.pause()
+                    })
+                }
+            }
+
+            parser.on("field", (k,v) => {
+                console.log(k,v)
+                if (k == "uploadId")
+                    file.setUploadId(v)
+            })
+
+            parser.parse(ctx.env.incoming).catch(e => console.log(e))
+            console.log("Parsing")
+
+            parser.on('error', (err) => {
+                if ("httpCode" in err)
+                    ctx.status(err.httpCode)
+                else ctx.status(400)
+                resolve(ctx.body(err.message))
+            })
+
+            file.on("error", (err) => {
+                if (err instanceof WebError)
+                    ctx.status(err.statusCode)
+                resolve(ctx.body(err?.message))
+            })
+
+            file.on("finish", () => {
+                file.commit().then(id => resolve(ctx.body(id!))).catch((err) => {
+                    if (err instanceof WebError)
+                        ctx.status(err.statusCode)
+                    resolve(ctx.body(err?.message))
+                })
+            })
+
        })}
    )
 /*