From c25a5230d93ce36d248f49e2733e88b9a1084182 Mon Sep 17 00:00:00 2001 From: Chip Wasson Date: Wed, 26 Jun 2024 16:51:03 -0600 Subject: [PATCH] Add proxy with file rewrite --- src/file/file.controller.ts | 113 +++++++++++++++--------- src/file/file.service.ts | 168 +++++++++++++++++++++++------------- src/minio/minio.service.ts | 18 +++- 3 files changed, 199 insertions(+), 100 deletions(-) diff --git a/src/file/file.controller.ts b/src/file/file.controller.ts index 1782d37..00b7f67 100644 --- a/src/file/file.controller.ts +++ b/src/file/file.controller.ts @@ -1,57 +1,90 @@ -import { Controller, Get, Param, Redirect, Render, UploadedFile, UseInterceptors } from '@nestjs/common'; +import { + Controller, + Get, + Param, + Redirect, + Render, + Res, + UploadedFile, + UseInterceptors, +} from '@nestjs/common'; import { FileService } from './file.service'; import { Post } from '@nestjs/common'; import { UploadedObjectInfo } from 'minio'; import { FileInterceptor } from '@nestjs/platform-express'; import { ApiConsumes, ApiTags } from '@nestjs/swagger'; import { InjectMetric } from '@willsoto/nestjs-prometheus'; +import { Response } from 'express'; @Controller('file') @ApiTags('file') export class FileController { - constructor(private readonly fileService: FileService) { } + constructor(private readonly fileService: FileService) {} - @Get() - @Render('file/upload') - generateUploadForm() { - return {} - } + @Get() + @Render('file/upload') + generateUploadForm() { + return {}; + } - @Get(':key') - @Redirect('', 302) - async getFile(@Param('key') key: string) { - const url = await this.fileService.generatePresignedUrl(key) - return { url } - } + @Get(':key/direct') + @Redirect('', 302) + async getFile(@Param('key') key: string) { + const url = await this.fileService.generatePresignedUrl(key); + return { url }; + } - @Post('upload') - @UseInterceptors(FileInterceptor('file')) - @ApiConsumes('multipart/form-data') - @Render('file/upload-result') - async handleFileUpload(@UploadedFile() file: Express.Multer.File): Promise { - const upload = await this.fileService.handleFileUpload(file); - return { - ...upload, - expireTime: new Date(upload.expireAt * 1000).toLocaleString(), - } + @Get(':key') + async getFileWithProxy(@Param('key') key: string, @Res() res: Response) { + if (!(await this.fileService.fileExists(key))) { + return res.status(404).send('File not found or expired'); } + const buffer = await this.fileService.getFileFromS3(key); + const metadata = await this.fileService.getFileMetadataFromS3(key); + const { originalfilename, 'content-type': contentType } = metadata.metaData; + return res + .header('Content-Type', contentType) + .header( + 'Content-Disposition', + `attachment; filename="${originalfilename ?? key}"`, + ) + .send(buffer); + } - @Post('upload.txt') - @UseInterceptors(FileInterceptor('file')) - @ApiConsumes('multipart/form-data') - async handleFileSilentUpload(@UploadedFile() file: Express.Multer.File): Promise { - const upload = await this.fileService.handleFileUpload(file); - return `https://api.us.dev/file/${upload.key}` - } + @Post('upload') + @UseInterceptors(FileInterceptor('file')) + @ApiConsumes('multipart/form-data') + @Render('file/upload-result') + async handleFileUpload( + @UploadedFile() file: Express.Multer.File, + ): Promise { + const upload = await this.fileService.handleFileUpload(file); + return { + ...upload, + expireTime: new Date(upload.expireAt * 1000).toLocaleString(), + }; + } - @Post('upload.json') - @UseInterceptors(FileInterceptor('file')) - @ApiConsumes('multipart/form-data') - async handleFileUploadJson(@UploadedFile() file: Express.Multer.File): Promise { - const upload = await this.fileService.handleFileUpload(file); - return { - ...upload, - expireTime: new Date(upload.expireAt * 1000).toLocaleString(), - } - } + @Post('upload.txt') + @UseInterceptors(FileInterceptor('file')) + @ApiConsumes('multipart/form-data') + async handleFileSilentUpload( + @UploadedFile() file: Express.Multer.File, + ): Promise { + const upload = await this.fileService.handleFileUpload(file); + return `https://api.us.dev/file/${upload.key}`; + } + + @Post('upload.json') + @UseInterceptors(FileInterceptor('file')) + @ApiConsumes('multipart/form-data') + async handleFileUploadJson( + @UploadedFile() file: Express.Multer.File, + ): Promise { + const upload = await this.fileService.handleFileUpload(file); + return { + ...upload, + expireTime: new Date(upload.expireAt * 1000).toLocaleString(), + }; + } } diff --git a/src/file/file.service.ts b/src/file/file.service.ts index ab1bf0b..4d6fc00 100644 --- a/src/file/file.service.ts +++ b/src/file/file.service.ts @@ -1,73 +1,123 @@ import { Injectable, Logger } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { Cron } from '@nestjs/schedule'; -import { UploadedObjectInfo } from 'minio'; +import { ItemBucketMetadata, UploadedObjectInfo } from 'minio'; import { MinioService } from 'src/minio/minio.service'; - @Injectable() export class FileService { - private readonly logger = new Logger(FileService.name); - private readonly bucketName; - private readonly filePrefix = 'file'; - private expirationTime: number; + private readonly logger = new Logger(FileService.name); + private readonly bucketName; + private readonly filePrefix = 'file'; + private expirationTime: number; - constructor( - private readonly minioService: MinioService, - private readonly configService: ConfigService, - ) { - this.expirationTime = this.configService.get('file.defaultTtl', 30 * 24 * 60 * 60); - this.bucketName = this.configService.get('file.bucketName', 'api.us.dev-files'); - try { - this.minioService.listBucketObjects(this.bucketName); - } catch (e) { - this.logger.log(`Error with bucket ${this.bucketName}: ${e.message}`); - } - this.deleteExpiredFiles() + constructor( + private readonly minioService: MinioService, + private readonly configService: ConfigService, + ) { + this.expirationTime = this.configService.get( + 'file.defaultTtl', + 30 * 24 * 60 * 60, + ); + this.bucketName = this.configService.get( + 'file.bucketName', + 'api.us.dev-files', + ); + try { + this.minioService.listBucketObjects(this.bucketName); + } catch (e) { + this.logger.log(`Error with bucket ${this.bucketName}: ${e.message}`); } + this.deleteExpiredFiles(); + } - private generateRandomKey(): string { - return Math.random().toString(36).substring(2); + private generateRandomKey(): string { + return Math.random().toString(36).substring(2); + } + + private pathFromKey(key: string): string { + return [this.filePrefix, key].join('/'); + } + + async handleFileUpload(file: Express.Multer.File): Promise<{ + uploadResult: UploadedObjectInfo; + expireAt: number; + key: string; + originalFilename: string; + }> { + const expireAt = Date.now() / 1000 + this.expirationTime; + const key = this.generateRandomKey(); + const uploadResult = await this.minioService.uploadBuffer( + this.bucketName, + this.pathFromKey(key), + file.buffer, + { + expireAt, + originalFilename: file.originalname, + 'content-type': file.mimetype, + }, + ); + return { + uploadResult, + key, + expireAt, + originalFilename: file.originalname, + }; + } + + async getFileFromS3(key: string): Promise { + return this.minioService.getBuffer(this.bucketName, this.pathFromKey(key)); + } + + async getFileMetadataFromS3(key: string): Promise { + return this.minioService.getObjectMetadata( + this.bucketName, + this.pathFromKey(key), + ); + } + + async fileExists(key: string): Promise { + return this.minioService.objectExists( + this.bucketName, + this.pathFromKey(key), + ); + } + + @Cron('0 0 * * *') + private async deleteExpiredFiles(): Promise { + this.logger.debug('Running cron job to delete expired files'); + const now = Date.now() / 1000; + const objectNames = await this.minioService.listBucketObjects( + this.bucketName, + this.filePrefix, + true, + ); + for (const objectName of objectNames) { + this.logger.debug(`Checking object ${objectName}`); + const objectInfo = await this.minioService.getObjectMetadata( + this.bucketName, + objectName, + ); + if (objectInfo.metaData.expireat < now) { + this.logger.debug(`Deleting object ${objectName}`); + await this.minioService.deleteObject(this.bucketName, objectName); + } } + } - async handleFileUpload(file: Express.Multer.File): Promise<{ uploadResult: UploadedObjectInfo, expireAt: number, key: string, originalFilename: string }> { - const expireAt = (Date.now() / 1000) + this.expirationTime; - const key = this.generateRandomKey(); - const uploadResult = await this.minioService.uploadBuffer(this.bucketName, [this.filePrefix, key].join('/'), file.buffer, { - expireAt, - originalFilename: file.originalname, - 'content-type': file.mimetype, - }); - return { - uploadResult, - key, - expireAt, - originalFilename: file.originalname, - } + async generatePresignedUrl(key: string): Promise { + const objectPath = [this.filePrefix, key].join('/'); + const metadata = await this.minioService.getObjectMetadata( + this.bucketName, + objectPath, + ); + if (metadata.expireAt < Date.now() / 1000) { + throw new Error('Object has expired'); } - - @Cron('0 0 * * *') - private async deleteExpiredFiles(): Promise { - this.logger.debug('Running cron job to delete expired files'); - const now = Date.now() / 1000; - const objectNames = await this.minioService.listBucketObjects(this.bucketName, this.filePrefix, true); - for (const objectName of objectNames) { - this.logger.debug(`Checking object ${objectName}`) - const objectInfo = await this.minioService.getObjectMetadata(this.bucketName, objectName); - if (objectInfo.metaData.expireat < now) { - this.logger.debug(`Deleting object ${objectName}`); - await this.minioService.deleteObject(this.bucketName, objectName); - } - } - } - - async generatePresignedUrl(key: string): Promise { - const objectPath = [this.filePrefix, key].join('/'); - const metadata = await this.minioService.getObjectMetadata(this.bucketName, objectPath); - if (metadata.expireAt < Date.now() / 1000) { - throw new Error('Object has expired'); - } - return await this.minioService.generatePresignedUrl(this.bucketName, objectPath, 10); - } - + return await this.minioService.generatePresignedUrl( + this.bucketName, + objectPath, + 10, + ); + } } diff --git a/src/minio/minio.service.ts b/src/minio/minio.service.ts index 6b3f060..5e0fa37 100644 --- a/src/minio/minio.service.ts +++ b/src/minio/minio.service.ts @@ -91,7 +91,11 @@ export class MinioService { prefix?: string, recursive: boolean = false, ): Promise { - const objectStream = await this.client.listObjectsV2(bucketName, prefix, recursive); + const objectStream = await this.client.listObjectsV2( + bucketName, + prefix, + recursive, + ); const objects = await new Promise( (resolve, reject) => { const objects: ItemBucketMetadata[] = []; @@ -128,4 +132,16 @@ export class MinioService { ): Promise { return await this.client.presignedGetObject(bucketName, objectName, expiry); } + + public async objectExists( + bucketName: string, + objectName: string, + ): Promise { + try { + await this.client.statObject(bucketName, objectName); + return true; + } catch (e) { + return false; + } + } }