Fix redis setup

2023-11-20 18:53:57 -07:00
parent a94ec100a5
commit 6ecdd5a0b8
6 changed files with 65 additions and 41 deletions
--- a/package.json
+++ b/package.json
@@ -34,7 +34,7 @@
    "bcrypt": "^5.1.1",
    "bull": "^4.11.5",
    "cache-manager": "^5.3.1",
-    "cache-manager-redis-store": "^3.0.1",
+    "cache-manager-redis-yet": "^4.1.2",
    "hbs": "^4.2.0",
    "minio": "^7.1.3",
    "open-graph-scraper": "^6.3.0",
--- a/src/app.module.ts
+++ b/src/app.module.ts
@@ -16,9 +16,9 @@ import { PrometheusModule } from '@willsoto/nestjs-prometheus';
 import { MinioModule } from './minio/minio.module';
 import { KvModule } from './kv/kv.module';
 import { BullModule } from '@nestjs/bull';
-import type { RedisClientOptions } from 'redis';
-import { redisStore } from 'cache-manager-redis-store';
 import { ScheduleModule } from '@nestjs/schedule';
+import { RedisClientOptions } from 'redis';
+import { redisStore } from 'cache-manager-redis-yet';

@Module({
  imports: [
@@ -27,31 +27,34 @@ import { ScheduleModule } from '@nestjs/schedule';
      isGlobal: true,
      load: [configuration],
    }),
-    // CacheModule.registerAsync<RedisClientOptions>({
-    //   isGlobal: true,
-    //   imports: [ConfigModule],
-    //   inject: [ConfigService],
-    //   useFactory: async (configService: ConfigService) => ({
-    //     store: redisStore as unknown as CacheStoreFactory,
-    //     ttl: configService.get<number>('CACHE_TTL', 5),
-    //     // Store-specific configuration:
-    //     password: configService.get<string>('redis.pass', ''),
-    //     host: configService.get<string>('redis.host', 'localhost'),
-    //     port: configService.get<number>('redis.port', 6379),
-    //     db: configService.get<number>('redis.db', 1),
-    //   }),
-    // }),
+    CacheModule.registerAsync<RedisClientOptions>({
+      isGlobal: true,
+      inject: [ConfigService],
+      imports: [ConfigModule],
+      useFactory: async (configService: ConfigService) => ({
+        store: redisStore,
+        socket: {
+          host: configService.get<string>('redis.host'),
+          port: configService.get<number>('redis.port'),
+        },
+        database: configService.get<number>('redis.db'),
+        password: configService.get<string>('redis.password'),
+      }),
+    }),
    BullModule.forRootAsync({
      imports: [ConfigModule],
      inject: [ConfigService],
-      useFactory: async (configService: ConfigService) => ({
+      useFactory: async (configService: ConfigService) => {
+        const config = {
          redis: {
-          password: configService.get<string>('redis.pass', ''),
-          host: configService.get<string>('redis.host', 'localhost'),
-          port: configService.get<number>('redis.port', 6379),
-          db: configService.get<number>('redis.db', 1),
+            password: configService.get<string>('redis.password'),
+            host: configService.get<string>('redis.host'),
+            port: configService.get<number>('redis.port'),
+            db: configService.get<number>('redis.db'),
+          },
+        };
+        return config;
      },
-      }),
    }),
    PrometheusModule.register({
      defaultLabels: {
--- a/src/config/configuration.ts
+++ b/src/config/configuration.ts
@@ -12,7 +12,7 @@ export default () => ({
    password: process.env.IRC_PASSWORD ?? '',
  },
  redis: {
-    host: process.env.REDIS_HOST ?? 's3.hooli.co',
+    host: process.env.REDIS_HOST ?? 'redis-master',
    port: parseInt(process.env.REDIS_PORT ?? '6379'),
    password: process.env.REDIS_PASS ?? '',
    db: parseInt(process.env.REDIS_DB ?? '1'),
--- a/src/kv/kv.module.ts
+++ b/src/kv/kv.module.ts
@@ -2,9 +2,15 @@ import { Module } from '@nestjs/common';
 import { KvService } from './kv.service';
 import { KvController } from './kv.controller';
 import { MinioService } from 'src/minio/minio.service';
+import { BullModule } from '@nestjs/bull';

@Module({
  providers: [KvService, MinioService],
+  imports: [
+    BullModule.registerQueue({
+      name: 'kv',
+    }),
+  ],
  controllers: [KvController],
 })
 export class KvModule {}
--- a/src/kv/kv.service.ts
+++ b/src/kv/kv.service.ts
@@ -1,4 +1,6 @@
+import { InjectQueue } from '@nestjs/bull';
 import { Injectable, Logger } from '@nestjs/common';
+import { Queue } from 'bull';
 import { UploadedObjectInfo } from 'minio';
 import { MinioService } from 'src/minio/minio.service';

@@ -10,7 +12,10 @@ export class KvService {
  private readonly kvMetadataPath = `${this.kvPrefix}/${this.kvMetadataFileName}`;
  private readonly logger: Logger = new Logger(KvService.name);

-  constructor(private readonly minioService: MinioService) {}
+  constructor(
+    private readonly minioService: MinioService,
+    @InjectQueue('kv') private kvProcessingQueue: Queue,
+  ) {}

  public generateFilePath(namespace: string, key: string): string {
    return `${this.kvPrefix}/${namespace}/${key}`;
@@ -114,9 +119,11 @@ export class KvService {
      ...extraData,
      secretKey: this.generateSecretKey(),
      claimed: new Date().toISOString(),
+      namespace,
    };
    await this.setMetadataFile(Buffer.from(JSON.stringify(metadata)));
    this.logger.verbose(`Claimed namespace ${namespace}`);
+    this.kvProcessingQueue.add('namespaceModeration', metadata[namespace]);
    return metadata[namespace];
  }

@@ -139,7 +146,7 @@ export class KvService {
      throw new Error('Incorrect secret key for namespace');
    }
    this.logger.verbose(`Setting ${namespace}/${key}`);
-    return await this.minioService.uploadBuffer(
+    const uploadResult = await this.minioService.uploadBuffer(
      this.kvBucketName,
      this.generateFilePath(namespace, key),
      Buffer.from(value),
@@ -149,6 +156,7 @@ export class KvService {
        ...options,
      },
    );
+    return uploadResult;
  }

  public async delete(namespace: string, key: string): Promise<void> {
--- a/yarn.lock
+++ b/yarn.lock
@@ -924,12 +924,12 @@
    picocolors "^1.0.0"
    tslib "^2.6.0"

-"@redis/bloom@1.2.0":
+"@redis/bloom@1.2.0", "@redis/bloom@^1.2.0":
  version "1.2.0"
  resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-1.2.0.tgz#d3fd6d3c0af3ef92f26767b56414a370c7b63b71"
  integrity sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==

-"@redis/client@1.5.12":
+"@redis/client@1.5.12", "@redis/client@^1.5.8":
  version "1.5.12"
  resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.5.12.tgz#4c387727992152aea443b869de0ebb697f899187"
  integrity sha512-/ZjE18HRzMd80eXIIUIPcH81UoZpwulbo8FmbElrjPqH0QC0SeIKu1BOU49bO5trM5g895kAjhvalt5h77q+4A==
@@ -938,22 +938,22 @@
    generic-pool "3.9.0"
    yallist "4.0.0"

-"@redis/graph@1.1.1":
+"@redis/graph@1.1.1", "@redis/graph@^1.1.0":
  version "1.1.1"
  resolved "https://registry.yarnpkg.com/@redis/graph/-/graph-1.1.1.tgz#8c10df2df7f7d02741866751764031a957a170ea"
  integrity sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==

-"@redis/json@1.0.6":
+"@redis/json@1.0.6", "@redis/json@^1.0.4":
  version "1.0.6"
  resolved "https://registry.yarnpkg.com/@redis/json/-/json-1.0.6.tgz#b7a7725bbb907765d84c99d55eac3fcf772e180e"
  integrity sha512-rcZO3bfQbm2zPRpqo82XbW8zg4G/w4W3tI7X8Mqleq9goQjAGLL7q/1n1ZX4dXEAmORVZ4s1+uKLaUOg7LrUhw==

-"@redis/search@1.1.6":
+"@redis/search@1.1.6", "@redis/search@^1.1.3":
  version "1.1.6"
  resolved "https://registry.yarnpkg.com/@redis/search/-/search-1.1.6.tgz#33bcdd791d9ed88ab6910243a355d85a7fedf756"
  integrity sha512-mZXCxbTYKBQ3M2lZnEddwEAks0Kc7nauire8q20oA0oA/LoA+E/b5Y5KZn232ztPb1FkIGqo12vh3Lf+Vw5iTw==

-"@redis/time-series@1.0.5":
+"@redis/time-series@1.0.5", "@redis/time-series@^1.0.4":
  version "1.0.5"
  resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.5.tgz#a6d70ef7a0e71e083ea09b967df0a0ed742bc6ad"
  integrity sha512-IFjIgTusQym2B5IZJG3XKr5llka7ey84fw/NOYqESP5WUfQs9zz1ww/9+qoz4ka/S6KcGBodzlCeZ5UImKbscg==
@@ -1967,14 +1967,21 @@ bytes@3.1.2:
  resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.2.tgz#8b0beeb98605adf1b128fa4386403c009e0221a5"
  integrity sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==

-cache-manager-redis-store@^3.0.1:
-  version "3.0.1"
-  resolved "https://registry.yarnpkg.com/cache-manager-redis-store/-/cache-manager-redis-store-3.0.1.tgz#8eeb211212763d04cef4058666182d624f714299"
-  integrity sha512-o560kw+dFqusC9lQJhcm6L2F2fMKobJ5af+FoR2PdnMVdpQ3f3Bz6qzvObTGyvoazQJxjQNWgMQeChP4vRTuXQ==
+cache-manager-redis-yet@^4.1.2:
+  version "4.1.2"
+  resolved "https://registry.yarnpkg.com/cache-manager-redis-yet/-/cache-manager-redis-yet-4.1.2.tgz#fa04df1a979a42585393a7a9918168978a7211ec"
+  integrity sha512-pM2K1ZlOv8gQpE1Z5mcDrfLj5CsNKVRiYua/SZ12j7LEDgfDeFVntI6JSgIw0siFSR/9P/FpG30scI3frHwibA==
  dependencies:
-    redis "^4.3.1"
+    "@redis/bloom" "^1.2.0"
+    "@redis/client" "^1.5.8"
+    "@redis/graph" "^1.1.0"
+    "@redis/json" "^1.0.4"
+    "@redis/search" "^1.1.3"
+    "@redis/time-series" "^1.0.4"
+    cache-manager "^5.2.2"
+    redis "^4.6.7"

-cache-manager@^5.3.1:
+cache-manager@^5.2.2, cache-manager@^5.3.1:
  version "5.3.1"
  resolved "https://registry.yarnpkg.com/cache-manager/-/cache-manager-5.3.1.tgz#80e4edd593b2d7f7b2b2199dc6fbb4748432bc47"
  integrity sha512-9HP6nc1ZqyZgcVEpy5XS2ns9MYE6cPEM6InA1wQhR6M7GviJzLH2NTFYnf3NEfRmLE351NCSkDo2VISX8dlG+w==
@@ -5033,7 +5040,7 @@ redis-parser@^3.0.0:
  dependencies:
    redis-errors "^1.0.0"

-redis@^4.3.1, redis@^4.6.11:
+redis@^4.6.11, redis@^4.6.7:
  version "4.6.11"
  resolved "https://registry.yarnpkg.com/redis/-/redis-4.6.11.tgz#fad85e104545228f212259fd557c3e4f8eafcd3d"
  integrity sha512-kg1Lt4NZLYkAjPOj/WcyIGWfZfnyfKo1Wg9YKVSlzhFwxpFIl3LYI8BWy1Ab963LLDsTz2+OwdsesHKljB3WMQ==