back to formidable

package-lock.json

@@ -21,6 +21,7 @@
         "cookie-parser": "^1.4.6",
         "dotenv": "^16.0.2",
         "express": "^4.18.1",
+        "formidable": "^3.5.1",
         "hono": "^3.8.3",
         "multer": "^1.4.5-lts.1",
         "node-fetch": "^3.3.2",
@@ -31,6 +32,7 @@
         "@sveltejs/vite-plugin-svelte": "^2.4.6",
         "@types/bytes": "^3.1.1",
         "@types/cookie-parser": "^1.4.3",
+        "@types/formidable": "^3.4.5",
         "@types/range-parser": "^1.2.6",
         "discord-api-types": "^0.37.61",
         "sass": "^1.57.1",
@@ -545,6 +547,15 @@
         "@types/range-parser": "*"
       }
     },
+    "node_modules/@types/formidable": {
+      "version": "3.4.5",
+      "resolved": "https://registry.npmjs.org/@types/formidable/-/formidable-3.4.5.tgz",
+      "integrity": "sha512-s7YPsNVfnsng5L8sKnG/Gbb2tiwwJTY1conOkJzTMRvJAlLFW1nEua+ADsJQu8N1c0oTHx9+d5nqg10WuT9gHQ==",
+      "dev": true,
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
     "node_modules/@types/mime": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/@types/mime/-/mime-3.0.1.tgz",
@@ -625,6 +636,11 @@
       "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
       "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg=="
     },
+    "node_modules/asap": {
+      "version": "2.0.6",
+      "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz",
+      "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA=="
+    },
     "node_modules/asynckit": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
@@ -886,6 +902,15 @@
         "npm": "1.2.8000 || >= 1.4.16"
       }
     },
+    "node_modules/dezalgo": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz",
+      "integrity": "sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==",
+      "dependencies": {
+        "asap": "^2.0.0",
+        "wrappy": "1"
+      }
+    },
     "node_modules/discord-api-types": {
       "version": "0.37.61",
       "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.37.61.tgz",
@@ -1098,6 +1123,19 @@
         "node": ">=12.20.0"
       }
     },
+    "node_modules/formidable": {
+      "version": "3.5.1",
+      "resolved": "https://registry.npmjs.org/formidable/-/formidable-3.5.1.tgz",
+      "integrity": "sha512-WJWKelbRHN41m5dumb0/k8TeAx7Id/y3a+Z7QfhxP/htI9Js5zYaEDtG8uMgG0vM0lOlqnmjE99/kfpOYi/0Og==",
+      "dependencies": {
+        "dezalgo": "^1.0.4",
+        "hexoid": "^1.0.0",
+        "once": "^1.4.0"
+      },
+      "funding": {
+        "url": "https://ko-fi.com/tunnckoCore/commissions"
+      }
+    },
     "node_modules/forwarded": {
       "version": "0.2.0",
       "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
@@ -1180,6 +1218,14 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/hexoid": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/hexoid/-/hexoid-1.0.0.tgz",
+      "integrity": "sha512-QFLV0taWQOZtvIRIAdBChesmogZrtuXvVWsFHZTk2SU+anspqZ2vMnoLg7IE1+Uk16N19APic1BuF8bC8c2m5g==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/hono": {
       "version": "3.8.3",
       "resolved": "https://registry.npmjs.org/hono/-/hono-3.8.3.tgz",
@@ -1495,6 +1541,14 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/once": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
+      "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
+      "dependencies": {
+        "wrappy": "1"
+      }
+    },
     "node_modules/parseurl": {
       "version": "1.3.3",
       "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
@@ -1969,6 +2023,11 @@
         "node": ">= 8"
       }
     },
+    "node_modules/wrappy": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
+      "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="
+    },
     "node_modules/xtend": {
       "version": "4.0.2",
       "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
@@ -2255,6 +2314,15 @@
         "@types/range-parser": "*"
       }
     },
+    "@types/formidable": {
+      "version": "3.4.5",
+      "resolved": "https://registry.npmjs.org/@types/formidable/-/formidable-3.4.5.tgz",
+      "integrity": "sha512-s7YPsNVfnsng5L8sKnG/Gbb2tiwwJTY1conOkJzTMRvJAlLFW1nEua+ADsJQu8N1c0oTHx9+d5nqg10WuT9gHQ==",
+      "dev": true,
+      "requires": {
+        "@types/node": "*"
+      }
+    },
     "@types/mime": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/@types/mime/-/mime-3.0.1.tgz",
@@ -2329,6 +2397,11 @@
       "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
       "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg=="
     },
+    "asap": {
+      "version": "2.0.6",
+      "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz",
+      "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA=="
+    },
     "asynckit": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
@@ -2522,6 +2595,15 @@
       "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz",
       "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg=="
     },
+    "dezalgo": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz",
+      "integrity": "sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==",
+      "requires": {
+        "asap": "^2.0.0",
+        "wrappy": "1"
+      }
+    },
     "discord-api-types": {
       "version": "0.37.61",
       "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.37.61.tgz",
@@ -2676,6 +2758,16 @@
         "fetch-blob": "^3.1.2"
       }
     },
+    "formidable": {
+      "version": "3.5.1",
+      "resolved": "https://registry.npmjs.org/formidable/-/formidable-3.5.1.tgz",
+      "integrity": "sha512-WJWKelbRHN41m5dumb0/k8TeAx7Id/y3a+Z7QfhxP/htI9Js5zYaEDtG8uMgG0vM0lOlqnmjE99/kfpOYi/0Og==",
+      "requires": {
+        "dezalgo": "^1.0.4",
+        "hexoid": "^1.0.0",
+        "once": "^1.4.0"
+      }
+    },
     "forwarded": {
       "version": "0.2.0",
       "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
@@ -2730,6 +2822,11 @@
       "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz",
       "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A=="
     },
+    "hexoid": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/hexoid/-/hexoid-1.0.0.tgz",
+      "integrity": "sha512-QFLV0taWQOZtvIRIAdBChesmogZrtuXvVWsFHZTk2SU+anspqZ2vMnoLg7IE1+Uk16N19APic1BuF8bC8c2m5g=="
+    },
     "hono": {
       "version": "3.8.3",
       "resolved": "https://registry.npmjs.org/hono/-/hono-3.8.3.tgz",
@@ -2941,6 +3038,14 @@
         "ee-first": "1.1.1"
       }
     },
+    "once": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
+      "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
+      "requires": {
+        "wrappy": "1"
+      }
+    },
     "parseurl": {
       "version": "1.3.3",
       "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
@@ -3246,6 +3351,11 @@
       "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz",
       "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q=="
     },
+    "wrappy": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
+      "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="
+    },
     "xtend": {
       "version": "4.0.2",
       "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",

package.json

@@ -30,6 +30,7 @@
     "cookie-parser": "^1.4.6",
     "dotenv": "^16.0.2",
     "express": "^4.18.1",
+    "formidable": "^3.5.1",
     "hono": "^3.8.3",
     "multer": "^1.4.5-lts.1",
     "node-fetch": "^3.3.2",
@@ -40,6 +41,7 @@
     "@sveltejs/vite-plugin-svelte": "^2.4.6",
     "@types/bytes": "^3.1.1",
     "@types/cookie-parser": "^1.4.3",
+    "@types/formidable": "^3.4.5",
     "@types/range-parser": "^1.2.6",
     "discord-api-types": "^0.37.61",
     "sass": "^1.57.1",

src/server/lib/files.ts

@@ -198,7 +198,7 @@ export class UploadStream extends Writable {
         if (this.errored) throw this.error
         if (!this.writableFinished) {
             let err = Error("attempted to commit file when the stream was still unfinished")
-            this.destroy(err); throw err
+            if (!this.destroyed) {this.destroy(err)}; throw err
         }
 
         // Perform checks

src/server/routes/api/v0/primaryApi.ts

@@ -10,13 +10,16 @@ import { getAccount, requiresPermissions } from "../../../lib/middleware.js"
 import FormDataParser, { Field } from "../../../lib/formdata.js"
 import {Readable} from "node:stream"
 import {ReadableStream as StreamWebReadable} from "node:stream/web"
+import formidable from "formidable"
+import { HttpBindings } from "@hono/node-server"
 export let primaryApi = new Hono<{
     Variables: {
         account: Accounts.Account
-    }
+    },
+    Bindings: HttpBindings
 }>()
 
-primaryApi.use(getAccount)
+primaryApi.all("*", getAccount)
 
 export default function (files: Files) {
     primaryApi.get(
@@ -98,43 +101,77 @@ export default function (files: Files) {
         "/upload",
         requiresPermissions("upload"),
         (ctx) => { return new Promise((resolve,reject) => {
-            let acc = ctx.get("account") as Accounts.Account
+            ctx.env.incoming.removeAllListeners("data") // remove hono's buffering
+            console.log('awawa')
+            let acc = ctx.get("account") as Accounts.Account | undefined
 
             if (!ctx.req.header("Content-Type")?.startsWith("multipart/form-data")) {
                 ctx.status(400)
                 resolve(ctx.body("[err] must be multipart/form-data"))
+                return
             }
 
             if (!ctx.req.raw.body) {
                 ctx.status(400)
                 resolve(ctx.body("[err] body must be supplied"))
+                return
+            }
+            console.log('awawawawa')
+
+            let file = files.createWriteStream(acc?.id)
+            let parser = formidable({
+                maxFieldsSize: 65536,
+                maxFileSize: files.config.maxDiscordFileSize*files.config.maxDiscordFiles,
+                maxFiles: 1
+            })
+
+            parser.onPart = function(part) {
+                console.log(part)
+                if (part.originalFilename == "" || !part.mimetype) {
+                    parser._handlePart(part); return
+                }
+                // lol
+                if (part.name == "file") {
+                    file.on("drain", () => {
+                        ctx.env.incoming.resume()
+                    })
+                    part.addListener("data", (data: Buffer) => {
+                        if (!file.write(data))
+                            ctx.env.incoming.pause()
+                    })
+                }
             }
 
-            let file = files.createWriteStream(acc.id)
+            parser.on("field", (k,v) => {
-            let formDataParser = new FormDataParser('')
+                console.log(k,v)
-            
+                if (k == "uploadId")
-            Readable.fromWeb(ctx.req.raw.body as StreamWebReadable)
+                    file.setUploadId(v)
-                .pipe(formDataParser)
+            })
-                .on("data", async (field: Field) => {
+
-                    if (field.headers["content-disposition"]?.filename) {
+            parser.parse(ctx.env.incoming).catch(e => console.log(e))
-                        field.pipe(file)
+            console.log("Parsing")
-                    } else {
+
-                        switch(field.headers["content-disposition"]?.name) {
+            parser.on('error', (err) => {
-                            case "uploadId":
+                if ("httpCode" in err)
-                                file.setUploadId((await field.collect(65536).catch(e => {formDataParser.destroy(new WebError(413, e.message))}))?.toString() || "")
+                    ctx.status(err.httpCode)
-                        }
+                else ctx.status(400)
-                    }
+                resolve(ctx.body(err.message))
-                })
+            })
-                .on("end", async () => {
+
-                    if (!file.writableEnded) await new Promise((res, rej) => {file.once("finish", res); file.once("error", res)})
+            file.on("error", (err) => {
-                    if (file.errored || !(await file.commit().catch(e => file.error = e))) {
+                if (err instanceof WebError)
-                        ctx.status(file.error instanceof WebError ? file.error.statusCode : 500)
+                    ctx.status(err.statusCode)
-                        resolve(`[err] ${file.error instanceof WebError ? file.error.message : file.error?.toString()}`)
+                resolve(ctx.body(err?.message))
-                        return
+            })
-                    }
+
-                    
+            file.on("finish", () => {
-                    resolve(ctx.body(file.uploadId!))
+                file.commit().then(id => resolve(ctx.body(id!))).catch((err) => {
+                    if (err instanceof WebError)
+                        ctx.status(err.statusCode)
+                    resolve(ctx.body(err?.message))
                 })
+            })
+
         })}
     )
 /*