mirror of
https://github.com/mollersuite/monofile.git
synced 2024-11-22 13:56:27 -08:00
refactor: ♻️ Use real async in file.ts, change FileUploadSettings to match FilePointer properties
This commit is contained in:
parent
0405f89542
commit
365aace294
|
@ -1,14 +1,16 @@
|
||||||
import axios from "axios";
|
import axios from "axios"
|
||||||
import Discord, { Client, TextBasedChannel } from "discord.js";
|
import Discord, { Client, Message, TextBasedChannel } from "discord.js"
|
||||||
import { readFile, writeFile } from "fs";
|
import { readFile, writeFile } from "node:fs/promises"
|
||||||
import { Readable } from "node:stream";
|
import { Readable } from "node:stream"
|
||||||
import crypto from "node:crypto";
|
import crypto from "node:crypto"
|
||||||
import { files } from "./accounts";
|
import { files } from "./accounts"
|
||||||
|
|
||||||
import * as Accounts from "./accounts";
|
import * as Accounts from "./accounts"
|
||||||
|
|
||||||
export let id_check_regex = /[A-Za-z0-9_\-\.\!\=\:\&\$\,\+\;\@\~\*\(\)\']+/
|
export let id_check_regex = /[A-Za-z0-9_\-\.\!\=\:\&\$\,\+\;\@\~\*\(\)\']+/
|
||||||
export let alphanum = Array.from("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890")
|
export let alphanum = Array.from(
|
||||||
|
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890"
|
||||||
|
)
|
||||||
|
|
||||||
// bad solution but whatever
|
// bad solution but whatever
|
||||||
|
|
||||||
|
@ -27,60 +29,54 @@ export function generateFileId(length:number=5) {
|
||||||
return fid
|
return fid
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface FileUploadSettings {
|
export type FileUploadSettings = Partial<Pick<FilePointer, "mime" | "owner">> &
|
||||||
name?: string,
|
Pick<FilePointer, "mime" | "filename"> & { uploadId?: string }
|
||||||
mime: string,
|
|
||||||
uploadId?: string,
|
|
||||||
owner?:string
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface Configuration {
|
export interface Configuration {
|
||||||
maxDiscordFiles: number,
|
maxDiscordFiles: number
|
||||||
maxDiscordFileSize: number,
|
maxDiscordFileSize: number
|
||||||
targetGuild: string,
|
targetGuild: string
|
||||||
targetChannel: string,
|
targetChannel: string
|
||||||
requestTimeout: number,
|
requestTimeout: number
|
||||||
maxUploadIdLength: number,
|
maxUploadIdLength: number
|
||||||
|
|
||||||
accounts: {
|
accounts: {
|
||||||
registrationEnabled: boolean,
|
registrationEnabled: boolean
|
||||||
requiredForUpload: boolean
|
requiredForUpload: boolean
|
||||||
},
|
}
|
||||||
|
|
||||||
trustProxy: boolean,
|
trustProxy: boolean
|
||||||
forceSSL: boolean
|
forceSSL: boolean
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface FilePointer {
|
export interface FilePointer {
|
||||||
filename:string,
|
filename: string
|
||||||
mime:string,
|
mime: string
|
||||||
messageids:string[],
|
messageids: string[]
|
||||||
owner?:string,
|
owner?: string
|
||||||
sizeInBytes?:number,
|
sizeInBytes?: number
|
||||||
tag?:string,
|
tag?: string
|
||||||
visibility?:FileVisibility,
|
visibility?: FileVisibility
|
||||||
reserved?: boolean,
|
reserved?: boolean
|
||||||
chunkSize?: number
|
chunkSize?: number
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface StatusCodeError {
|
export interface StatusCodeError {
|
||||||
status: number,
|
status: number
|
||||||
message: string
|
message: string
|
||||||
}
|
}
|
||||||
|
|
||||||
/* */
|
/* */
|
||||||
|
|
||||||
export default class Files {
|
export default class Files {
|
||||||
|
|
||||||
config: Configuration
|
config: Configuration
|
||||||
client: Client
|
client: Client
|
||||||
files: { [key: string]: FilePointer } = {}
|
files: { [key: string]: FilePointer } = {}
|
||||||
uploadChannel?: TextBasedChannel
|
uploadChannel?: TextBasedChannel
|
||||||
|
|
||||||
constructor(client: Client, config: Configuration) {
|
constructor(client: Client, config: Configuration) {
|
||||||
|
this.config = config
|
||||||
this.config = config;
|
this.client = client
|
||||||
this.client = client;
|
|
||||||
|
|
||||||
client.on("ready", () => {
|
client.on("ready", () => {
|
||||||
console.log("Discord OK!")
|
console.log("Discord OK!")
|
||||||
|
@ -94,100 +90,94 @@ export default class Files {
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
readFile(process.cwd()+"/.data/files.json",(err,buf) => {
|
readFile(process.cwd() + "/.data/files.json")
|
||||||
if (err) {console.log(err);return}
|
.then((buf) => {
|
||||||
this.files = JSON.parse(buf.toString() || "{}")
|
this.files = JSON.parse(buf.toString() || "{}")
|
||||||
})
|
})
|
||||||
|
.catch(console.error)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @description Uploads a new file
|
* @description Uploads a new file
|
||||||
* @param settings Settings for your new upload
|
* @param metadata Settings for your new upload
|
||||||
* @param fBuffer Buffer containing file content
|
* @param buffer Buffer containing file content
|
||||||
* @returns Promise which resolves to the ID of the new file
|
* @returns Promise which resolves to the ID of the new file
|
||||||
*/
|
*/
|
||||||
uploadFile(settings:FileUploadSettings,fBuffer:Buffer):Promise<string|StatusCodeError> {
|
async uploadFile(
|
||||||
return new Promise<string>(async (resolve,reject) => {
|
metadata: FileUploadSettings,
|
||||||
if (!this.uploadChannel) {
|
buffer: Buffer
|
||||||
reject({status:503,message:"server is not ready - please try again later"})
|
): Promise<string | StatusCodeError> {
|
||||||
return
|
if (!this.uploadChannel)
|
||||||
|
throw {
|
||||||
|
status: 503,
|
||||||
|
message: "server is not ready - please try again later",
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!settings.name || !settings.mime) {
|
if (!metadata.filename || !metadata.mime)
|
||||||
reject({status:400,message:"missing name/mime"});
|
throw { status: 400, message: "missing filename/mime" }
|
||||||
return
|
|
||||||
|
let uploadId = (metadata.uploadId || generateFileId()).toString()
|
||||||
|
|
||||||
|
if (
|
||||||
|
(uploadId.match(id_check_regex) || [])[0] != uploadId ||
|
||||||
|
uploadId.length > this.config.maxUploadIdLength
|
||||||
|
)
|
||||||
|
throw { status: 400, message: "invalid id" }
|
||||||
|
|
||||||
|
if (
|
||||||
|
this.files[uploadId] &&
|
||||||
|
(metadata.owner
|
||||||
|
? this.files[uploadId].owner != metadata.owner
|
||||||
|
: true)
|
||||||
|
)
|
||||||
|
throw {
|
||||||
|
status: 400,
|
||||||
|
message: "you are not the owner of this file id",
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!settings.owner && this.config.accounts.requiredForUpload) {
|
if (this.files[uploadId] && this.files[uploadId].reserved)
|
||||||
reject({status:401,message:"an account is required for upload"});
|
throw {
|
||||||
return
|
status: 400,
|
||||||
|
message:
|
||||||
|
"already uploading this file. if your file is stuck in this state, contact an administrator",
|
||||||
}
|
}
|
||||||
|
|
||||||
let uploadId = (settings.uploadId || generateFileId()).toString();
|
if (metadata.filename.length > 128)
|
||||||
|
throw { status: 400, message: "name too long" }
|
||||||
|
|
||||||
if ((uploadId.match(id_check_regex) || [])[0] != uploadId || uploadId.length > this.config.maxUploadIdLength) {
|
if (metadata.mime.length > 128)
|
||||||
reject({status:400,message:"invalid id"});return
|
throw { status: 400, message: "mime too long" }
|
||||||
}
|
|
||||||
|
|
||||||
if (this.files[uploadId] && (settings.owner ? this.files[uploadId].owner != settings.owner : true)) {
|
|
||||||
reject({status:400,message:"you are not the owner of this file id"});
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if (this.files[uploadId] && this.files[uploadId].reserved) {
|
|
||||||
reject({status:400,message:"already uploading this file. if your file is stuck in this state, contact an administrator"});
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if (settings.name.length > 128) {
|
|
||||||
reject({status:400,message:"name too long"});
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if (settings.mime.length > 128) {
|
|
||||||
reject({status:400,message:"mime too long"});
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// reserve file, hopefully should prevent
|
// reserve file, hopefully should prevent
|
||||||
// large files breaking
|
// large files breaking
|
||||||
|
|
||||||
let ogf = this.files[uploadId]
|
let existingFile = this.files[uploadId]
|
||||||
|
|
||||||
this.files[uploadId] = {
|
|
||||||
filename:settings.name,
|
|
||||||
messageids:[],
|
|
||||||
mime:settings.mime,
|
|
||||||
sizeInBytes:0,
|
|
||||||
|
|
||||||
owner:settings.owner,
|
|
||||||
visibility: settings.owner ? "private" : "public",
|
|
||||||
reserved: true,
|
|
||||||
|
|
||||||
chunkSize: this.config.maxDiscordFileSize
|
|
||||||
}
|
|
||||||
|
|
||||||
// save
|
// save
|
||||||
|
|
||||||
if (settings.owner) {
|
if (metadata.owner) {
|
||||||
await files.index(settings.owner,uploadId)
|
await files.index(metadata.owner, uploadId)
|
||||||
}
|
}
|
||||||
|
|
||||||
// get buffer
|
// get buffer
|
||||||
if (fBuffer.byteLength >= (this.config.maxDiscordFileSize*this.config.maxDiscordFiles)) {
|
if (
|
||||||
reject({status:400,message:"file too large"});
|
buffer.byteLength >=
|
||||||
return
|
this.config.maxDiscordFileSize * this.config.maxDiscordFiles
|
||||||
}
|
)
|
||||||
|
throw { status: 400, message: "file too large" }
|
||||||
|
|
||||||
// generate buffers to upload
|
// generate buffers to upload
|
||||||
let toUpload = []
|
let toUpload = []
|
||||||
for (let i = 0; i < Math.ceil(fBuffer.byteLength/this.config.maxDiscordFileSize); i++) {
|
for (
|
||||||
|
let i = 0;
|
||||||
|
i < Math.ceil(buffer.byteLength / this.config.maxDiscordFileSize);
|
||||||
|
i++
|
||||||
|
) {
|
||||||
toUpload.push(
|
toUpload.push(
|
||||||
fBuffer.subarray(
|
buffer.subarray(
|
||||||
i * this.config.maxDiscordFileSize,
|
i * this.config.maxDiscordFileSize,
|
||||||
Math.min(
|
Math.min(
|
||||||
fBuffer.byteLength,
|
buffer.byteLength,
|
||||||
(i + 1) * this.config.maxDiscordFileSize
|
(i + 1) * this.config.maxDiscordFileSize
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
@ -196,63 +186,64 @@ export default class Files {
|
||||||
|
|
||||||
// begin uploading
|
// begin uploading
|
||||||
let uploadTmplt: Discord.AttachmentBuilder[] = toUpload.map((e) => {
|
let uploadTmplt: Discord.AttachmentBuilder[] = toUpload.map((e) => {
|
||||||
return new Discord.AttachmentBuilder(e)
|
return new Discord.AttachmentBuilder(e).setName(
|
||||||
.setName(Math.random().toString().slice(2))
|
Math.random().toString().slice(2)
|
||||||
|
)
|
||||||
})
|
})
|
||||||
let uploadGroups = []
|
let uploadGroups = []
|
||||||
|
|
||||||
for (let i = 0; i < Math.ceil(uploadTmplt.length / 10); i++) {
|
for (let i = 0; i < Math.ceil(uploadTmplt.length / 10); i++) {
|
||||||
uploadGroups.push(uploadTmplt.slice(i*10,((i+1)*10)))
|
uploadGroups.push(uploadTmplt.slice(i * 10, (i + 1) * 10))
|
||||||
}
|
}
|
||||||
|
|
||||||
let msgIds = []
|
let msgIds = []
|
||||||
|
|
||||||
for (let i = 0; i < uploadGroups.length; i++) {
|
for (const uploadGroup of uploadGroups) {
|
||||||
|
let message = await this.uploadChannel
|
||||||
|
.send({
|
||||||
|
files: uploadGroup,
|
||||||
|
})
|
||||||
|
.catch((e) => {
|
||||||
|
console.error(e)
|
||||||
|
})
|
||||||
|
|
||||||
let ms = await this.uploadChannel.send({
|
if (message && message instanceof Message) {
|
||||||
files:uploadGroups[i]
|
msgIds.push(message.id)
|
||||||
}).catch((e) => {console.error(e)})
|
|
||||||
|
|
||||||
if (ms) {
|
|
||||||
msgIds.push(ms.id)
|
|
||||||
} else {
|
} else {
|
||||||
if (!ogf) delete this.files[uploadId]
|
if (!existingFile) delete this.files[uploadId]
|
||||||
else this.files[uploadId] = ogf
|
else this.files[uploadId] = existingFile
|
||||||
reject({status:500,message:"please try again"}); return
|
throw { status: 500, message: "please try again" }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// this code deletes the files from discord, btw
|
// this code deletes the files from discord, btw
|
||||||
// if need be, replace with job queue system
|
// if need be, replace with job queue system
|
||||||
|
|
||||||
if (ogf&&this.uploadChannel) {
|
if (existingFile && this.uploadChannel) {
|
||||||
for (let x of ogf.messageids) {
|
for (let x of existingFile.messageids) {
|
||||||
this.uploadChannel.messages.delete(x).catch(err => console.error(err))
|
this.uploadChannel.messages
|
||||||
|
.delete(x)
|
||||||
|
.catch((err) => console.error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
resolve(await this.writeFile(
|
const { filename, mime, owner } = metadata
|
||||||
uploadId,
|
return this.writeFile(uploadId, {
|
||||||
{
|
filename,
|
||||||
filename:settings.name,
|
|
||||||
messageids: msgIds,
|
messageids: msgIds,
|
||||||
mime:settings.mime,
|
mime,
|
||||||
sizeInBytes:fBuffer.byteLength,
|
owner,
|
||||||
|
sizeInBytes: buffer.byteLength,
|
||||||
|
|
||||||
owner:settings.owner,
|
visibility: existingFile
|
||||||
visibility: ogf ? ogf.visibility
|
? existingFile.visibility
|
||||||
: (
|
: metadata.owner
|
||||||
settings.owner
|
? Accounts.getFromId(metadata.owner)?.defaultFileVisibility
|
||||||
? Accounts.getFromId(settings.owner)?.defaultFileVisibility
|
: undefined,
|
||||||
: undefined
|
|
||||||
),
|
|
||||||
// so that json.stringify doesnt include tag:undefined
|
// so that json.stringify doesnt include tag:undefined
|
||||||
...((ogf||{}).tag ? {tag:ogf.tag} : {}),
|
...((existingFile || {}).tag ? { tag: existingFile.tag } : {}),
|
||||||
|
|
||||||
chunkSize: this.config.maxDiscordFileSize
|
|
||||||
}
|
|
||||||
))
|
|
||||||
|
|
||||||
|
|
||||||
|
chunkSize: this.config.maxDiscordFileSize,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -264,23 +255,25 @@ export default class Files {
|
||||||
* @param file FilePointer representing the new file
|
* @param file FilePointer representing the new file
|
||||||
* @returns Promise which resolves to the file's ID
|
* @returns Promise which resolves to the file's ID
|
||||||
*/
|
*/
|
||||||
writeFile(uploadId: string, file: FilePointer):Promise<string> {
|
async writeFile(uploadId: string, file: FilePointer): Promise<string> {
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
|
|
||||||
this.files[uploadId] = file
|
this.files[uploadId] = file
|
||||||
|
|
||||||
writeFile(process.cwd()+"/.data/files.json",JSON.stringify(this.files),(err) => {
|
return writeFile(
|
||||||
|
process.cwd() + "/.data/files.json",
|
||||||
if (err) {
|
JSON.stringify(
|
||||||
reject({status:500,message:"server may be misconfigured, contact admin for help"});
|
this.files,
|
||||||
delete this.files[uploadId];
|
null,
|
||||||
return
|
process.env.NODE_ENV === "development" ? 4 : undefined
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.then(() => uploadId)
|
||||||
|
.catch(() => {
|
||||||
|
delete this.files[uploadId]
|
||||||
|
throw {
|
||||||
|
status: 500,
|
||||||
|
message:
|
||||||
|
"server may be misconfigured, contact admin for help",
|
||||||
}
|
}
|
||||||
|
|
||||||
resolve(uploadId)
|
|
||||||
|
|
||||||
})
|
|
||||||
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -290,28 +283,30 @@ export default class Files {
|
||||||
* @param range Byte range to get
|
* @param range Byte range to get
|
||||||
* @returns A `Readable` containing the file's contents
|
* @returns A `Readable` containing the file's contents
|
||||||
*/
|
*/
|
||||||
readFileStream(uploadId: string, range?: {start:number, end:number}):Promise<Readable> {
|
async readFileStream(
|
||||||
return new Promise(async (resolve,reject) => {
|
uploadId: string,
|
||||||
|
range?: { start: number; end: number }
|
||||||
|
): Promise<Readable> {
|
||||||
if (!this.uploadChannel) {
|
if (!this.uploadChannel) {
|
||||||
reject({status:503,message:"server is not ready - please try again later"})
|
throw {
|
||||||
return
|
status: 503,
|
||||||
|
message: "server is not ready - please try again later",
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (this.files[uploadId]) {
|
if (this.files[uploadId]) {
|
||||||
let file = this.files[uploadId]
|
let file = this.files[uploadId]
|
||||||
|
|
||||||
let
|
let scan_msg_begin = 0,
|
||||||
scan_msg_begin = 0,
|
|
||||||
scan_msg_end = file.messageids.length - 1,
|
scan_msg_end = file.messageids.length - 1,
|
||||||
scan_files_begin = 0,
|
scan_files_begin = 0,
|
||||||
scan_files_end = -1
|
scan_files_end = -1
|
||||||
|
|
||||||
let useRanges = range && file.chunkSize && file.sizeInBytes;
|
let useRanges = range && file.chunkSize && file.sizeInBytes
|
||||||
|
|
||||||
// todo: figure out how to get typesccript to accept useRanges
|
// todo: figure out how to get typesccript to accept useRanges
|
||||||
// i'm too tired to look it up or write whatever it wnats me to do
|
// i'm too tired to look it up or write whatever it wnats me to do
|
||||||
if (range && file.chunkSize && file.sizeInBytes) {
|
if (range && file.chunkSize && file.sizeInBytes) {
|
||||||
|
|
||||||
// Calculate where to start file scans...
|
// Calculate where to start file scans...
|
||||||
|
|
||||||
scan_files_begin = Math.floor(range.start / file.chunkSize)
|
scan_files_begin = Math.floor(range.start / file.chunkSize)
|
||||||
|
@ -319,51 +314,72 @@ export default class Files {
|
||||||
|
|
||||||
scan_msg_begin = Math.floor(scan_files_begin / 10)
|
scan_msg_begin = Math.floor(scan_files_begin / 10)
|
||||||
scan_msg_end = Math.ceil(scan_files_end / 10)
|
scan_msg_end = Math.ceil(scan_files_end / 10)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let attachments: Discord.Attachment[] = [];
|
let attachments: Discord.Attachment[] = []
|
||||||
|
|
||||||
/* File updates */
|
/* File updates */
|
||||||
let file_updates: Pick<FilePointer, "chunkSize" | "sizeInBytes"> = {}
|
let file_updates: Pick<FilePointer, "chunkSize" | "sizeInBytes"> =
|
||||||
|
{}
|
||||||
let atSIB: number[] = [] // kepes track of the size of each file...
|
let atSIB: number[] = [] // kepes track of the size of each file...
|
||||||
|
|
||||||
for (let xi = scan_msg_begin; xi < scan_msg_end + 1; xi++) {
|
for (let xi = scan_msg_begin; xi < scan_msg_end + 1; xi++) {
|
||||||
|
let msg = await this.uploadChannel.messages
|
||||||
let msg = await this.uploadChannel.messages.fetch(file.messageids[xi]).catch(() => {return null})
|
.fetch(file.messageids[xi])
|
||||||
|
.catch(() => {
|
||||||
|
return null
|
||||||
|
})
|
||||||
if (msg?.attachments) {
|
if (msg?.attachments) {
|
||||||
|
|
||||||
let attach = Array.from(msg.attachments.values())
|
let attach = Array.from(msg.attachments.values())
|
||||||
for (let i = (useRanges && xi == scan_msg_begin ? ( scan_files_begin - (xi*10) ) : 0); i < (useRanges && xi == scan_msg_end ? ( scan_files_end - (xi*10) + 1 ) : attach.length); i++) {
|
for (
|
||||||
|
let i =
|
||||||
|
useRanges && xi == scan_msg_begin
|
||||||
|
? scan_files_begin - xi * 10
|
||||||
|
: 0;
|
||||||
|
i <
|
||||||
|
(useRanges && xi == scan_msg_end
|
||||||
|
? scan_files_end - xi * 10 + 1
|
||||||
|
: attach.length);
|
||||||
|
i++
|
||||||
|
) {
|
||||||
attachments.push(attach[i])
|
attachments.push(attach[i])
|
||||||
atSIB.push(attach[i].size)
|
atSIB.push(attach[i].size)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
if (!file.sizeInBytes)
|
||||||
|
file_updates.sizeInBytes = atSIB.reduce((a, b) => a + b, 0)
|
||||||
}
|
|
||||||
|
|
||||||
if (!file.sizeInBytes) file_updates.sizeInBytes = atSIB.reduce((a,b) => a+b, 0);
|
|
||||||
if (!file.chunkSize) file_updates.chunkSize = atSIB[0]
|
if (!file.chunkSize) file_updates.chunkSize = atSIB[0]
|
||||||
if (Object.keys(file_updates).length) { // if file_updates not empty
|
if (Object.keys(file_updates).length) {
|
||||||
|
// if file_updates not empty
|
||||||
// i gotta do these weird workarounds, ts is weird sometimes
|
// i gotta do these weird workarounds, ts is weird sometimes
|
||||||
// originally i was gonna do key is keyof FilePointer but for some reason
|
// originally i was gonna do key is keyof FilePointer but for some reason
|
||||||
// it ended up making typeof file[key] never??? so
|
// it ended up making typeof file[key] never??? so
|
||||||
// its 10pm and chinese people suck at being quiet so i just wanna get this over with
|
// its 10pm and chinese people suck at being quiet so i just wanna get this over with
|
||||||
// chinese is the worst language in terms of volume lmao
|
// chinese is the worst language in terms of volume lmao
|
||||||
let valid_fp_keys = ["sizeInBytes", "chunkSize"]
|
let valid_fp_keys = ["sizeInBytes", "chunkSize"]
|
||||||
let isValidFilePointerKey = (key: string): key is "sizeInBytes" | "chunkSize" => valid_fp_keys.includes(key)
|
let isValidFilePointerKey = (
|
||||||
|
key: string
|
||||||
|
): key is "sizeInBytes" | "chunkSize" =>
|
||||||
|
valid_fp_keys.includes(key)
|
||||||
|
|
||||||
for (let [key, value] of Object.entries(file_updates)) {
|
for (let [key, value] of Object.entries(file_updates)) {
|
||||||
if (isValidFilePointerKey(key)) file[key] = value
|
if (isValidFilePointerKey(key)) file[key] = value
|
||||||
}
|
}
|
||||||
|
|
||||||
writeFile(process.cwd()+"/.data/files.json",JSON.stringify(this.files),(err) => {})
|
// The original was a callback so I don't think I'm supposed to `await` this -Jack
|
||||||
|
writeFile(
|
||||||
|
process.cwd() + "/.data/files.json",
|
||||||
|
JSON.stringify(
|
||||||
|
this.files,
|
||||||
|
null,
|
||||||
|
process.env.NODE_ENV === "development" ? 4 : undefined
|
||||||
|
)
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
let position = 0;
|
let position = 0
|
||||||
|
|
||||||
let getNextChunk = async () => {
|
let getNextChunk = async () => {
|
||||||
let scanning_chunk = attachments[position]
|
let scanning_chunk = attachments[position]
|
||||||
|
@ -371,25 +387,45 @@ export default class Files {
|
||||||
return null
|
return null
|
||||||
}
|
}
|
||||||
|
|
||||||
let d = await axios.get(
|
let d = await axios
|
||||||
scanning_chunk.url,
|
.get(scanning_chunk.url, {
|
||||||
{
|
|
||||||
responseType: "arraybuffer",
|
responseType: "arraybuffer",
|
||||||
headers: {
|
headers: {
|
||||||
...(useRanges ? {
|
...(useRanges
|
||||||
"Range": `bytes=${position == 0 && range && file.chunkSize ? range.start-(scan_files_begin*file.chunkSize) : "0"}-${position == attachments.length-1 && range && file.chunkSize ? range.end-(scan_files_end*file.chunkSize) : ""}`
|
? {
|
||||||
} : {})
|
Range: `bytes=${
|
||||||
|
position == 0 &&
|
||||||
|
range &&
|
||||||
|
file.chunkSize
|
||||||
|
? range.start -
|
||||||
|
scan_files_begin *
|
||||||
|
file.chunkSize
|
||||||
|
: "0"
|
||||||
|
}-${
|
||||||
|
position == attachments.length - 1 &&
|
||||||
|
range &&
|
||||||
|
file.chunkSize
|
||||||
|
? range.end -
|
||||||
|
scan_files_end * file.chunkSize
|
||||||
|
: ""
|
||||||
|
}`,
|
||||||
}
|
}
|
||||||
}
|
: {}),
|
||||||
).catch((e:Error) => {console.error(e)})
|
},
|
||||||
|
})
|
||||||
|
.catch((e: Error) => {
|
||||||
|
console.error(e)
|
||||||
|
})
|
||||||
|
|
||||||
position++;
|
position++
|
||||||
|
|
||||||
if (d) {
|
if (d) {
|
||||||
return d.data
|
return d.data
|
||||||
} else {
|
} else {
|
||||||
reject({status:500,message:"internal server error"})
|
throw {
|
||||||
return "__ERR"
|
status: 500,
|
||||||
|
message: "internal server error",
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -402,7 +438,10 @@ export default class Files {
|
||||||
if (!lastChunkSent) return
|
if (!lastChunkSent) return
|
||||||
lastChunkSent = false
|
lastChunkSent = false
|
||||||
getNextChunk().then(async (nextChunk) => {
|
getNextChunk().then(async (nextChunk) => {
|
||||||
if (nextChunk == "__ERR") {this.destroy(new Error("file read error")); return}
|
if (nextChunk == "__ERR") {
|
||||||
|
this.destroy(new Error("file read error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
let response = this.push(nextChunk)
|
let response = this.push(nextChunk)
|
||||||
|
|
||||||
if (!nextChunk) return // EOF
|
if (!nextChunk) return // EOF
|
||||||
|
@ -414,15 +453,13 @@ export default class Files {
|
||||||
}
|
}
|
||||||
lastChunkSent = true
|
lastChunkSent = true
|
||||||
})
|
})
|
||||||
}
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
resolve(dataStream)
|
return dataStream
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
reject({status:404,message:"not found"})
|
throw { status: 404, message: "not found" }
|
||||||
}
|
}
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -430,33 +467,41 @@ export default class Files {
|
||||||
* @param uploadId Target file's ID
|
* @param uploadId Target file's ID
|
||||||
* @param noWrite Whether or not the change should be written to disk. Enable for bulk deletes
|
* @param noWrite Whether or not the change should be written to disk. Enable for bulk deletes
|
||||||
*/
|
*/
|
||||||
unlink(uploadId:string, noWrite: boolean = false):Promise<void> {
|
async unlink(uploadId: string, noWrite: boolean = false): Promise<void> {
|
||||||
return new Promise(async (resolve,reject) => {
|
let tmp = this.files[uploadId]
|
||||||
let tmp = this.files[uploadId];
|
if (!tmp) {
|
||||||
if (!tmp) {resolve(); return}
|
return
|
||||||
|
}
|
||||||
if (tmp.owner) {
|
if (tmp.owner) {
|
||||||
let id = files.deindex(tmp.owner,uploadId,noWrite);
|
let id = files.deindex(tmp.owner, uploadId, noWrite)
|
||||||
if (id) await id
|
if (id) await id
|
||||||
}
|
}
|
||||||
// this code deletes the files from discord, btw
|
// this code deletes the files from discord, btw
|
||||||
// if need be, replace with job queue system
|
// if need be, replace with job queue system
|
||||||
|
|
||||||
if (!this.uploadChannel) {reject(); return}
|
if (!this.uploadChannel) {
|
||||||
|
return
|
||||||
|
}
|
||||||
for (let x of tmp.messageids) {
|
for (let x of tmp.messageids) {
|
||||||
this.uploadChannel.messages.delete(x).catch(err => console.error(err))
|
this.uploadChannel.messages
|
||||||
|
.delete(x)
|
||||||
|
.catch((err) => console.error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
delete this.files[uploadId];
|
delete this.files[uploadId]
|
||||||
if (noWrite) {resolve(); return}
|
if (noWrite) {
|
||||||
writeFile(process.cwd()+"/.data/files.json",JSON.stringify(this.files),(err) => {
|
return
|
||||||
if (err) {
|
}
|
||||||
|
return writeFile(
|
||||||
|
process.cwd() + "/.data/files.json",
|
||||||
|
JSON.stringify(
|
||||||
|
this.files,
|
||||||
|
null,
|
||||||
|
process.env.NODE_ENV === "development" ? 4 : undefined
|
||||||
|
)
|
||||||
|
).catch((err) => {
|
||||||
this.files[uploadId] = tmp // !! this may not work, since tmp is a link to this.files[uploadId]?
|
this.files[uploadId] = tmp // !! this may not work, since tmp is a link to this.files[uploadId]?
|
||||||
reject()
|
throw err
|
||||||
} else {
|
|
||||||
resolve()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -468,5 +513,4 @@ export default class Files {
|
||||||
getFilePointer(uploadId: string): FilePointer {
|
getFilePointer(uploadId: string): FilePointer {
|
||||||
return this.files[uploadId]
|
return this.files[uploadId]
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,47 +1,52 @@
|
||||||
import bodyParser from "body-parser";
|
import bodyParser from "body-parser"
|
||||||
import express, { Router } from "express";
|
import express, { Router } from "express"
|
||||||
import * as Accounts from "../../../lib/accounts";
|
import * as Accounts from "../../../lib/accounts"
|
||||||
import * as auth from "../../../lib/auth";
|
import * as auth from "../../../lib/auth"
|
||||||
import axios, { AxiosResponse } from "axios"
|
import axios, { AxiosResponse } from "axios"
|
||||||
import { type Range } from "range-parser";
|
import { type Range } from "range-parser"
|
||||||
import multer, { memoryStorage } from "multer"
|
import multer, { memoryStorage } from "multer"
|
||||||
|
|
||||||
import ServeError from "../../../lib/errors";
|
import ServeError from "../../../lib/errors"
|
||||||
import Files from "../../../lib/files";
|
import Files from "../../../lib/files"
|
||||||
import { getAccount, requiresPermissions } from "../../../lib/middleware";
|
import { getAccount, requiresPermissions } from "../../../lib/middleware"
|
||||||
|
|
||||||
let parser = bodyParser.json({
|
let parser = bodyParser.json({
|
||||||
type: ["text/plain","application/json"]
|
type: ["text/plain", "application/json"],
|
||||||
})
|
})
|
||||||
|
|
||||||
export let primaryApi = Router();
|
export let primaryApi = Router()
|
||||||
|
|
||||||
const multerSetup = multer({ storage: memoryStorage() })
|
const multerSetup = multer({ storage: memoryStorage() })
|
||||||
|
|
||||||
let config = require(`${process.cwd()}/config.json`)
|
let config = require(`${process.cwd()}/config.json`)
|
||||||
|
|
||||||
primaryApi.use(getAccount);
|
primaryApi.use(getAccount)
|
||||||
|
|
||||||
module.exports = function (files: Files) {
|
module.exports = function (files: Files) {
|
||||||
|
primaryApi.get(
|
||||||
primaryApi.get(["/file/:fileId", "/cpt/:fileId/*", "/:fileId"], async (req:express.Request,res:express.Response) => {
|
["/file/:fileId", "/cpt/:fileId/*", "/:fileId"],
|
||||||
|
async (req: express.Request, res: express.Response) => {
|
||||||
let acc = res.locals.acc as Accounts.Account
|
let acc = res.locals.acc as Accounts.Account
|
||||||
|
|
||||||
let file = files.getFilePointer(req.params.fileId)
|
let file = files.getFilePointer(req.params.fileId)
|
||||||
res.setHeader("Access-Control-Allow-Origin", "*")
|
res.setHeader("Access-Control-Allow-Origin", "*")
|
||||||
res.setHeader("Content-Security-Policy", "sandbox allow-scripts")
|
res.setHeader("Content-Security-Policy", "sandbox allow-scripts")
|
||||||
if (req.query.attachment == "1") res.setHeader("Content-Disposition", "attachment")
|
if (req.query.attachment == "1")
|
||||||
|
res.setHeader("Content-Disposition", "attachment")
|
||||||
|
|
||||||
if (file) {
|
if (file) {
|
||||||
|
|
||||||
if (file.visibility == "private") {
|
if (file.visibility == "private") {
|
||||||
if (acc?.id != file.owner) {
|
if (acc?.id != file.owner) {
|
||||||
ServeError(res, 403, "you do not own this file")
|
ServeError(res, 403, "you do not own this file")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if (auth.getType(auth.tokenFor(req)) == "App" && auth.getPermissions(auth.tokenFor(req))?.includes("private")) {
|
if (
|
||||||
|
auth.getType(auth.tokenFor(req)) == "App" &&
|
||||||
|
auth
|
||||||
|
.getPermissions(auth.tokenFor(req))
|
||||||
|
?.includes("private")
|
||||||
|
) {
|
||||||
ServeError(res, 403, "insufficient permissions")
|
ServeError(res, 403, "insufficient permissions")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -56,57 +61,66 @@ module.exports = function(files: Files) {
|
||||||
if (file.chunkSize) {
|
if (file.chunkSize) {
|
||||||
let rng = req.range(file.sizeInBytes)
|
let rng = req.range(file.sizeInBytes)
|
||||||
if (rng) {
|
if (rng) {
|
||||||
|
|
||||||
// error handling
|
// error handling
|
||||||
if (typeof rng == "number") {
|
if (typeof rng == "number") {
|
||||||
res.status(rng == -1 ? 416 : 400).send()
|
res.status(rng == -1 ? 416 : 400).send()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if (rng.type != "bytes") {
|
if (rng.type != "bytes") {
|
||||||
res.status(400).send();
|
res.status(400).send()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// set ranges var
|
// set ranges var
|
||||||
let rngs = Array.from(rng)
|
let rngs = Array.from(rng)
|
||||||
if (rngs.length != 1) { res.status(400).send(); return }
|
if (rngs.length != 1) {
|
||||||
|
res.status(400).send()
|
||||||
|
return
|
||||||
|
}
|
||||||
range = rngs[0]
|
range = rngs[0]
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// supports ranges
|
// supports ranges
|
||||||
|
|
||||||
|
files
|
||||||
files.readFileStream(req.params.fileId, range).then(async stream => {
|
.readFileStream(req.params.fileId, range)
|
||||||
|
.then(async (stream) => {
|
||||||
if (range) {
|
if (range) {
|
||||||
res.status(206)
|
res.status(206)
|
||||||
res.header("Content-Length", (range.end-range.start + 1).toString())
|
res.header(
|
||||||
res.header("Content-Range", `bytes ${range.start}-${range.end}/${file.sizeInBytes}`)
|
"Content-Length",
|
||||||
|
(range.end - range.start + 1).toString()
|
||||||
|
)
|
||||||
|
res.header(
|
||||||
|
"Content-Range",
|
||||||
|
`bytes ${range.start}-${range.end}/${file.sizeInBytes}`
|
||||||
|
)
|
||||||
}
|
}
|
||||||
stream.pipe(res)
|
stream.pipe(res)
|
||||||
|
})
|
||||||
}).catch((err) => {
|
.catch((err) => {
|
||||||
ServeError(res, err.status, err.message)
|
ServeError(res, err.status, err.message)
|
||||||
})
|
})
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
ServeError(res, 404, "file not found")
|
ServeError(res, 404, "file not found")
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
})
|
primaryApi.head(
|
||||||
|
["/file/:fileId", "/cpt/:fileId/*", "/:fileId"],
|
||||||
primaryApi.head(["/file/:fileId", "/cpt/:fileId/*", "/:fileId"], (req: express.Request, res:express.Response) => {
|
(req: express.Request, res: express.Response) => {
|
||||||
let file = files.getFilePointer(req.params.fileId)
|
let file = files.getFilePointer(req.params.fileId)
|
||||||
|
|
||||||
if (
|
if (
|
||||||
file.visibility == "private"
|
file.visibility == "private" &&
|
||||||
&& (
|
(res.locals.acc?.id != file.owner ||
|
||||||
res.locals.acc?.id != file.owner
|
(auth.getType(auth.tokenFor(req)) == "App" &&
|
||||||
|| (auth.getType(auth.tokenFor(req)) == "App" && auth.getPermissions(auth.tokenFor(req))?.includes("private"))
|
auth
|
||||||
)
|
.getPermissions(auth.tokenFor(req))
|
||||||
|
?.includes("private")))
|
||||||
) {
|
) {
|
||||||
res.status(403).send()
|
res.status(403).send()
|
||||||
return
|
return
|
||||||
|
@ -115,7 +129,8 @@ module.exports = function(files: Files) {
|
||||||
res.setHeader("Access-Control-Allow-Origin", "*")
|
res.setHeader("Access-Control-Allow-Origin", "*")
|
||||||
res.setHeader("Content-Security-Policy", "sandbox allow-scripts")
|
res.setHeader("Content-Security-Policy", "sandbox allow-scripts")
|
||||||
|
|
||||||
if (req.query.attachment == "1") res.setHeader("Content-Disposition", "attachment")
|
if (req.query.attachment == "1")
|
||||||
|
res.setHeader("Content-Disposition", "attachment")
|
||||||
|
|
||||||
if (!file) {
|
if (!file) {
|
||||||
res.status(404)
|
res.status(404)
|
||||||
|
@ -130,12 +145,16 @@ module.exports = function(files: Files) {
|
||||||
}
|
}
|
||||||
res.send()
|
res.send()
|
||||||
}
|
}
|
||||||
})
|
}
|
||||||
|
)
|
||||||
|
|
||||||
// upload handlers
|
// upload handlers
|
||||||
|
|
||||||
primaryApi.post("/upload", requiresPermissions("upload"), multerSetup.single('file'), async (req,res) => {
|
primaryApi.post(
|
||||||
|
"/upload",
|
||||||
|
requiresPermissions("upload"),
|
||||||
|
multerSetup.single("file"),
|
||||||
|
async (req, res) => {
|
||||||
let acc = res.locals.acc as Accounts.Account
|
let acc = res.locals.acc as Accounts.Account
|
||||||
|
|
||||||
if (req.file) {
|
if (req.file) {
|
||||||
|
@ -146,16 +165,20 @@ module.exports = function(files: Files) {
|
||||||
params = JSON.parse(prm)
|
params = JSON.parse(prm)
|
||||||
}
|
}
|
||||||
|
|
||||||
files.uploadFile({
|
files
|
||||||
|
.uploadFile(
|
||||||
|
{
|
||||||
owner: acc?.id,
|
owner: acc?.id,
|
||||||
|
|
||||||
uploadId: params.uploadId,
|
uploadId: params.uploadId,
|
||||||
name:req.file.originalname,
|
filename: req.file.originalname,
|
||||||
mime:req.file.mimetype
|
mime: req.file.mimetype,
|
||||||
},req.file.buffer)
|
},
|
||||||
|
req.file.buffer
|
||||||
|
)
|
||||||
.then((uID) => res.send(uID))
|
.then((uID) => res.send(uID))
|
||||||
.catch((stat) => {
|
.catch((stat) => {
|
||||||
res.status(stat.status);
|
res.status(stat.status)
|
||||||
res.send(`[err] ${stat.message}`)
|
res.send(`[err] ${stat.message}`)
|
||||||
})
|
})
|
||||||
} catch {
|
} catch {
|
||||||
|
@ -166,29 +189,40 @@ module.exports = function(files: Files) {
|
||||||
res.status(400)
|
res.status(400)
|
||||||
res.send("[err] bad request")
|
res.send("[err] bad request")
|
||||||
}
|
}
|
||||||
})
|
}
|
||||||
|
)
|
||||||
primaryApi.post("/clone", requiresPermissions("upload"), bodyParser.json({type: ["text/plain","application/json"]}) ,(req,res) => {
|
|
||||||
|
|
||||||
|
primaryApi.post(
|
||||||
|
"/clone",
|
||||||
|
requiresPermissions("upload"),
|
||||||
|
bodyParser.json({ type: ["text/plain", "application/json"] }),
|
||||||
|
(req, res) => {
|
||||||
let acc = res.locals.acc as Accounts.Account
|
let acc = res.locals.acc as Accounts.Account
|
||||||
|
|
||||||
try {
|
try {
|
||||||
axios.get(req.body.url,{responseType:"arraybuffer"}).then((data:AxiosResponse) => {
|
axios
|
||||||
|
.get(req.body.url, { responseType: "arraybuffer" })
|
||||||
files.uploadFile({
|
.then((data: AxiosResponse) => {
|
||||||
|
files
|
||||||
|
.uploadFile(
|
||||||
|
{
|
||||||
owner: acc?.id,
|
owner: acc?.id,
|
||||||
|
filename:
|
||||||
name:req.body.url.split("/")[req.body.url.split("/").length-1] || "generic",
|
req.body.url.split("/")[
|
||||||
|
req.body.url.split("/").length - 1
|
||||||
|
] || "generic",
|
||||||
mime: data.headers["content-type"],
|
mime: data.headers["content-type"],
|
||||||
uploadId:req.body.uploadId
|
uploadId: req.body.uploadId,
|
||||||
},Buffer.from(data.data))
|
},
|
||||||
|
Buffer.from(data.data)
|
||||||
|
)
|
||||||
.then((uID) => res.send(uID))
|
.then((uID) => res.send(uID))
|
||||||
.catch((stat) => {
|
.catch((stat) => {
|
||||||
res.status(stat.status);
|
res.status(stat.status)
|
||||||
res.send(`[err] ${stat.message}`)
|
res.send(`[err] ${stat.message}`)
|
||||||
})
|
})
|
||||||
|
})
|
||||||
}).catch((err) => {
|
.catch((err) => {
|
||||||
console.log(err)
|
console.log(err)
|
||||||
res.status(400)
|
res.status(400)
|
||||||
res.send(`[err] failed to fetch data`)
|
res.send(`[err] failed to fetch data`)
|
||||||
|
@ -197,7 +231,8 @@ module.exports = function(files: Files) {
|
||||||
res.status(500)
|
res.status(500)
|
||||||
res.send("[err] an error occured")
|
res.send("[err] an error occured")
|
||||||
}
|
}
|
||||||
})
|
}
|
||||||
|
)
|
||||||
|
|
||||||
return primaryApi
|
return primaryApi
|
||||||
}
|
}
|
Loading…
Reference in a new issue