From 7c40650b9a7c3ba455f670c8df6b387599aace4f Mon Sep 17 00:00:00 2001
From: Chip Wasson <chip@wasson.io>
Date: Sun, 19 Nov 2023 18:36:29 -0700
Subject: [PATCH] Add minio and kv modules

---
 src/app.module.ts               |   4 +
 src/kv/kv.controller.spec.ts    |  18 +++++
 src/kv/kv.controller.ts         |  94 ++++++++++++++++++++++
 src/kv/kv.module.ts             |  10 +++
 src/kv/kv.service.spec.ts       |  18 +++++
 src/kv/kv.service.ts            | 135 ++++++++++++++++++++++++++++++++
 src/minio/minio.module.ts       |   7 ++
 src/minio/minio.service.spec.ts |  18 +++++
 src/minio/minio.service.ts      |  91 +++++++++++++++++++++
 9 files changed, 395 insertions(+)
 create mode 100644 src/kv/kv.controller.spec.ts
 create mode 100644 src/kv/kv.controller.ts
 create mode 100644 src/kv/kv.module.ts
 create mode 100644 src/kv/kv.service.spec.ts
 create mode 100644 src/kv/kv.service.ts
 create mode 100644 src/minio/minio.module.ts
 create mode 100644 src/minio/minio.service.spec.ts
 create mode 100644 src/minio/minio.service.ts

diff --git a/src/app.module.ts b/src/app.module.ts
index 34e13ca..afc9cbc 100644
--- a/src/app.module.ts
+++ b/src/app.module.ts
@@ -14,6 +14,8 @@ import { IrcbotService } from './ircbot/ircbot.service';
 import { DinosaurwetModule } from './dinosaurwet/dinosaurwet.module';
 import { OgScraperModule } from './ogscraper/ogscraper.module';
 import { PrometheusModule } from '@willsoto/nestjs-prometheus';
+import { MinioModule } from './minio/minio.module';
+import { KvModule } from './kv/kv.module';
 
 @Module({
   imports: [
@@ -35,6 +37,8 @@ import { PrometheusModule } from '@willsoto/nestjs-prometheus';
     IrcbotModule,
     DinosaurwetModule,
     OgScraperModule,
+    MinioModule,
+    KvModule,
   ],
   controllers: [AppController],
   providers: [AppService],
diff --git a/src/kv/kv.controller.spec.ts b/src/kv/kv.controller.spec.ts
new file mode 100644
index 0000000..5af12e1
--- /dev/null
+++ b/src/kv/kv.controller.spec.ts
@@ -0,0 +1,18 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { KvController } from './KvController';
+
+describe('KvController', () => {
+  let controller: KvController;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      controllers: [KvController],
+    }).compile();
+
+    controller = module.get<KvController>(KvController);
+  });
+
+  it('should be defined', () => {
+    expect(controller).toBeDefined();
+  });
+});
diff --git a/src/kv/kv.controller.ts b/src/kv/kv.controller.ts
new file mode 100644
index 0000000..36aeb31
--- /dev/null
+++ b/src/kv/kv.controller.ts
@@ -0,0 +1,94 @@
+import {
+  Controller,
+  Get,
+  Post,
+  Req,
+  Body,
+  Param,
+  HttpException,
+} from '@nestjs/common';
+import { KvService } from './kv.service';
+import { Request } from 'express';
+
+@Controller('kv')
+export class KvController {
+  constructor(private readonly kvService: KvService) {}
+
+  @Get(':namespace/:key/metadata')
+  async getMetadata(
+    @Param('namespace') namespace: string,
+    @Param('key') key: string,
+    @Req() request: Request,
+  ): Promise<object> {
+    if (!request.headers['x-secret-key']) {
+      throw new HttpException("Missing 'X-Secret-Key' header", 403);
+    }
+    try {
+      return await this.kvService.queryObjectMetadata(
+        namespace,
+        key,
+        request.headers['x-secret-key'] as string,
+      );
+    } catch (e) {
+      throw new HttpException(e.message, 403);
+    }
+  }
+
+  @Get(':namespace/:key')
+  async get(
+    @Param('namespace') namespace: string,
+    @Param('key') key: string,
+    @Req() request: Request,
+  ): Promise<string> {
+    if (request.headers['x-secret-key']) {
+      try {
+        return await this.kvService.getWithSecretKey(
+          namespace,
+          key,
+          request.headers['x-secret-key'] as string,
+        );
+      } catch (e) {
+        throw new HttpException(e.message, 403);
+      }
+    }
+    try {
+      return await this.kvService.get(namespace, key);
+    } catch (e) {
+      throw new HttpException(e.message, 403);
+    }
+  }
+
+  @Post(':namespace/:key')
+  async create(
+    @Param('namespace') namespace: string,
+    @Param('key') key: string,
+    @Req() request: Request,
+    @Body() body: object,
+  ) {
+    const secretKey =
+      (request.headers['x-secret-key'] as string | undefined) ?? '';
+    const publicFlag = request.headers['x-public'] === 'true';
+    if (secretKey) {
+      return await this.kvService.set(
+        namespace,
+        key,
+        JSON.stringify(body),
+        secretKey,
+        { public: publicFlag },
+      );
+    }
+    return 'No secret key provided within X-Secret-Key header';
+  }
+
+  @Post(':namespace')
+  async claimNamespace(
+    @Param('namespace') namespace: string,
+    @Req() request: Request,
+  ) {
+    try {
+      return await this.kvService.claimNamespace(namespace);
+    } catch (e) {
+      throw new HttpException(e.message, 403);
+    }
+  }
+}
diff --git a/src/kv/kv.module.ts b/src/kv/kv.module.ts
new file mode 100644
index 0000000..143ca04
--- /dev/null
+++ b/src/kv/kv.module.ts
@@ -0,0 +1,10 @@
+import { Module } from '@nestjs/common';
+import { KvService } from './kv.service';
+import { KvController } from './kv.controller';
+import { MinioService } from 'src/minio/minio.service';
+
+@Module({
+  providers: [KvService, MinioService],
+  controllers: [KvController],
+})
+export class KvModule {}
diff --git a/src/kv/kv.service.spec.ts b/src/kv/kv.service.spec.ts
new file mode 100644
index 0000000..49960d6
--- /dev/null
+++ b/src/kv/kv.service.spec.ts
@@ -0,0 +1,18 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { KvService } from './kv.service';
+
+describe('KvService', () => {
+  let service: KvService;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [KvService],
+    }).compile();
+
+    service = module.get<KvService>(KvService);
+  });
+
+  it('should be defined', () => {
+    expect(service).toBeDefined();
+  });
+});
diff --git a/src/kv/kv.service.ts b/src/kv/kv.service.ts
new file mode 100644
index 0000000..39cd7d8
--- /dev/null
+++ b/src/kv/kv.service.ts
@@ -0,0 +1,135 @@
+import { Injectable } from '@nestjs/common';
+import { UploadedObjectInfo } from 'minio';
+import { MinioService } from 'src/minio/minio.service';
+
+@Injectable()
+export class KvService {
+  private readonly kvBucketName = 'api.us.dev';
+  private readonly kvPrefix = 'kv';
+  private readonly kvMetadataFileName = 'metadata.json';
+  private readonly kvMetadataPath = `${this.kvPrefix}/${this.kvMetadataFileName}`;
+
+  constructor(private readonly minioService: MinioService) {}
+
+  public generateFilePath(namespace: string, key: string): string {
+    return `${this.kvPrefix}/${namespace}/${key}`;
+  }
+
+  private async getMetadataFile(): Promise<Buffer> {
+    return await this.minioService.getBuffer(
+      this.kvBucketName,
+      this.kvMetadataPath,
+    );
+  }
+
+  private async setMetadataFile(buffer: Buffer): Promise<void> {
+    await this.minioService.uploadBuffer(
+      this.kvBucketName,
+      this.kvMetadataPath,
+      buffer,
+    );
+  }
+
+  public async queryObjectMetadata(
+    namespace: string,
+    key: string,
+    secretKey: string,
+  ): Promise<object> {
+    const metadata = await this.getMetadataFile().then((buffer) =>
+      JSON.parse(buffer.toString()),
+    );
+    if (metadata[namespace]?.secretKey !== secretKey) {
+      throw new Error('Secret key does not match');
+    }
+    const objectMetadata = await this.minioService.getObjectMetadata(
+      this.kvBucketName,
+      this.generateFilePath(namespace, key),
+    );
+    return objectMetadata?.metaData;
+  }
+
+  public async get(namespace: string, key: string): Promise<string> {
+    const objectMetadata = await this.minioService.getObjectMetadata(
+      this.kvBucketName,
+      this.generateFilePath(namespace, key),
+    );
+    console.log(objectMetadata);
+    if (objectMetadata?.metaData.public !== 'true') {
+      throw new Error('Object is not public');
+    }
+    return await this.minioService
+      .getBuffer(this.kvBucketName, this.generateFilePath(namespace, key))
+      .then((buffer) => buffer.toString());
+  }
+
+  public async getWithSecretKey(
+    namespace: string,
+    key: string,
+    secretKey: string,
+  ): Promise<string> {
+    const metadata = await this.getMetadataFile().then((buffer) =>
+      JSON.parse(buffer.toString()),
+    );
+    if (metadata[namespace]?.secretKey !== secretKey) {
+      throw new Error('Secret key does not match');
+    }
+    return await this.minioService
+      .getBuffer(this.kvBucketName, this.generateFilePath(namespace, key))
+      .then((buffer) => buffer.toString());
+  }
+
+  public async claimNamespace(
+    namespace: string,
+    extraData: object = {},
+  ): Promise<void> {
+    const metadata = await this.getMetadataFile().then((buffer) =>
+      JSON.parse(buffer.toString()),
+    );
+    if (metadata[namespace]?.secretKey) {
+      throw new Error('Namespace already claimed');
+    }
+    metadata[namespace] = {
+      ...extraData,
+      secretKey: Math.random().toString(36).substring(2),
+      claimed: new Date().toISOString(),
+    };
+    await this.setMetadataFile(Buffer.from(JSON.stringify(metadata)));
+    return metadata[namespace];
+  }
+
+  public async set(
+    namespace: string,
+    key: string,
+    value: string,
+    secretKey: string,
+    options: {
+      public: boolean;
+    } = {
+      public: false,
+    },
+  ): Promise<UploadedObjectInfo> {
+    const metadata = await this.getMetadataFile().then((buffer) =>
+      JSON.parse(buffer.toString()),
+    );
+    if (metadata[namespace]?.secretKey !== secretKey) {
+      throw new Error('Incorrect secret key for namespace');
+    }
+    return await this.minioService.uploadBuffer(
+      this.kvBucketName,
+      this.generateFilePath(namespace, key),
+      Buffer.from(value),
+      {
+        set: new Date().toISOString(),
+        namespace,
+        ...options,
+      },
+    );
+  }
+
+  public async delete(namespace: string, key: string): Promise<void> {
+    await this.minioService.deleteObject(
+      this.kvBucketName,
+      this.generateFilePath(namespace, key),
+    );
+  }
+}
diff --git a/src/minio/minio.module.ts b/src/minio/minio.module.ts
new file mode 100644
index 0000000..23cbe4f
--- /dev/null
+++ b/src/minio/minio.module.ts
@@ -0,0 +1,7 @@
+import { Module } from '@nestjs/common';
+import { MinioService } from './minio.service';
+
+@Module({
+  providers: [MinioService]
+})
+export class MinioModule {}
diff --git a/src/minio/minio.service.spec.ts b/src/minio/minio.service.spec.ts
new file mode 100644
index 0000000..8cc5a96
--- /dev/null
+++ b/src/minio/minio.service.spec.ts
@@ -0,0 +1,18 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { MinioService } from './minio.service';
+
+describe('MinioService', () => {
+  let service: MinioService;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [MinioService],
+    }).compile();
+
+    service = module.get<MinioService>(MinioService);
+  });
+
+  it('should be defined', () => {
+    expect(service).toBeDefined();
+  });
+});
diff --git a/src/minio/minio.service.ts b/src/minio/minio.service.ts
new file mode 100644
index 0000000..cfa324a
--- /dev/null
+++ b/src/minio/minio.service.ts
@@ -0,0 +1,91 @@
+import { Injectable } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import { Client, ItemBucketMetadata, UploadedObjectInfo } from 'minio';
+
+@Injectable()
+export class MinioService {
+  private readonly client: Client;
+  public readonly defaultBucketName: string;
+
+  constructor(private readonly configService: ConfigService) {
+    this.client = new Client({
+      endPoint: this.configService.get<string>('S3_ENDPOINT', 's3.hooli.co'),
+      port: this.configService.get<number>('S3_PORT', 443),
+      useSSL: true,
+      accessKey: this.configService.get<string>('S3_ACCESS_KEY', ''),
+      secretKey: this.configService.get<string>('S3_SECRET_KEY', ''),
+    });
+    this.defaultBucketName = this.configService.get<string>('S3_BUCKET', '');
+  }
+
+  public async uploadFile(
+    bucketName: string,
+    objectName: string,
+    filePath: string,
+    metadata?: ItemBucketMetadata,
+  ): Promise<UploadedObjectInfo> {
+    return await this.client.fPutObject(
+      bucketName,
+      objectName,
+      filePath,
+      metadata,
+    );
+  }
+
+  public async uploadBuffer(
+    bucketName: string,
+    objectName: string,
+    buffer: Buffer,
+    metadata?: ItemBucketMetadata,
+  ): Promise<UploadedObjectInfo> {
+    return await this.client.putObject(
+      bucketName,
+      objectName,
+      buffer,
+      metadata,
+    );
+  }
+
+  public async getBuffer(
+    bucketName: string,
+    objectName: string,
+  ): Promise<Buffer> {
+    const objectStream = await this.client.getObject(bucketName, objectName);
+    return await new Promise((resolve, reject) => {
+      const chunks: Buffer[] = [];
+      objectStream.on('data', (chunk) => chunks.push(chunk));
+      objectStream.on('end', () => resolve(Buffer.concat(chunks)));
+      objectStream.on('error', (err) => reject(err));
+    });
+  }
+
+  public async getObjectMetadata(
+    bucketName: string,
+    objectName: string,
+  ): Promise<ItemBucketMetadata> {
+    return await this.client.statObject(bucketName, objectName);
+  }
+
+  public async listBucketObjects(
+    bucketName: string,
+    prefix?: string,
+  ): Promise<string[]> {
+    const objectStream = await this.client.listObjects(bucketName, prefix);
+    const objects = await new Promise<ItemBucketMetadata[]>(
+      (resolve, reject) => {
+        const objects: ItemBucketMetadata[] = [];
+        objectStream.on('data', (object) => objects.push(object));
+        objectStream.on('end', () => resolve(objects));
+        objectStream.on('error', (err) => reject(err));
+      },
+    );
+    return objects.map((object) => object.name);
+  }
+
+  public async deleteObject(
+    bucketName: string,
+    objectName: string,
+  ): Promise<void> {
+    await this.client.removeObject(bucketName, objectName);
+  }
+}