From 6ecdd5a0b8cd6e4c86244366c0b1ee85e7fcd2f8 Mon Sep 17 00:00:00 2001 From: Chip Wasson Date: Mon, 20 Nov 2023 18:53:57 -0700 Subject: [PATCH] Fix redis setup --- package.json | 2 +- src/app.module.ts | 51 ++++++++++++++++++++----------------- src/config/configuration.ts | 2 +- src/kv/kv.module.ts | 6 +++++ src/kv/kv.service.ts | 12 +++++++-- yarn.lock | 33 ++++++++++++++---------- 6 files changed, 65 insertions(+), 41 deletions(-) diff --git a/package.json b/package.json index 1f64dc2..952c39b 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "bcrypt": "^5.1.1", "bull": "^4.11.5", "cache-manager": "^5.3.1", - "cache-manager-redis-store": "^3.0.1", + "cache-manager-redis-yet": "^4.1.2", "hbs": "^4.2.0", "minio": "^7.1.3", "open-graph-scraper": "^6.3.0", diff --git a/src/app.module.ts b/src/app.module.ts index d280aa4..fd6ab96 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -16,9 +16,9 @@ import { PrometheusModule } from '@willsoto/nestjs-prometheus'; import { MinioModule } from './minio/minio.module'; import { KvModule } from './kv/kv.module'; import { BullModule } from '@nestjs/bull'; -import type { RedisClientOptions } from 'redis'; -import { redisStore } from 'cache-manager-redis-store'; import { ScheduleModule } from '@nestjs/schedule'; +import { RedisClientOptions } from 'redis'; +import { redisStore } from 'cache-manager-redis-yet'; @Module({ imports: [ @@ -27,31 +27,34 @@ import { ScheduleModule } from '@nestjs/schedule'; isGlobal: true, load: [configuration], }), - // CacheModule.registerAsync({ - // isGlobal: true, - // imports: [ConfigModule], - // inject: [ConfigService], - // useFactory: async (configService: ConfigService) => ({ - // store: redisStore as unknown as CacheStoreFactory, - // ttl: configService.get('CACHE_TTL', 5), - // // Store-specific configuration: - // password: configService.get('redis.pass', ''), - // host: configService.get('redis.host', 'localhost'), - // port: configService.get('redis.port', 6379), - // db: configService.get('redis.db', 1), - // }), - // }), + CacheModule.registerAsync({ + isGlobal: true, + inject: [ConfigService], + imports: [ConfigModule], + useFactory: async (configService: ConfigService) => ({ + store: redisStore, + socket: { + host: configService.get('redis.host'), + port: configService.get('redis.port'), + }, + database: configService.get('redis.db'), + password: configService.get('redis.password'), + }), + }), BullModule.forRootAsync({ imports: [ConfigModule], inject: [ConfigService], - useFactory: async (configService: ConfigService) => ({ - redis: { - password: configService.get('redis.pass', ''), - host: configService.get('redis.host', 'localhost'), - port: configService.get('redis.port', 6379), - db: configService.get('redis.db', 1), - }, - }), + useFactory: async (configService: ConfigService) => { + const config = { + redis: { + password: configService.get('redis.password'), + host: configService.get('redis.host'), + port: configService.get('redis.port'), + db: configService.get('redis.db'), + }, + }; + return config; + }, }), PrometheusModule.register({ defaultLabels: { diff --git a/src/config/configuration.ts b/src/config/configuration.ts index cc62f84..a303a7b 100644 --- a/src/config/configuration.ts +++ b/src/config/configuration.ts @@ -12,7 +12,7 @@ export default () => ({ password: process.env.IRC_PASSWORD ?? '', }, redis: { - host: process.env.REDIS_HOST ?? 's3.hooli.co', + host: process.env.REDIS_HOST ?? 'redis-master', port: parseInt(process.env.REDIS_PORT ?? '6379'), password: process.env.REDIS_PASS ?? '', db: parseInt(process.env.REDIS_DB ?? '1'), diff --git a/src/kv/kv.module.ts b/src/kv/kv.module.ts index 143ca04..f739e0e 100644 --- a/src/kv/kv.module.ts +++ b/src/kv/kv.module.ts @@ -2,9 +2,15 @@ import { Module } from '@nestjs/common'; import { KvService } from './kv.service'; import { KvController } from './kv.controller'; import { MinioService } from 'src/minio/minio.service'; +import { BullModule } from '@nestjs/bull'; @Module({ providers: [KvService, MinioService], + imports: [ + BullModule.registerQueue({ + name: 'kv', + }), + ], controllers: [KvController], }) export class KvModule {} diff --git a/src/kv/kv.service.ts b/src/kv/kv.service.ts index 1cefa0a..b560ab1 100644 --- a/src/kv/kv.service.ts +++ b/src/kv/kv.service.ts @@ -1,4 +1,6 @@ +import { InjectQueue } from '@nestjs/bull'; import { Injectable, Logger } from '@nestjs/common'; +import { Queue } from 'bull'; import { UploadedObjectInfo } from 'minio'; import { MinioService } from 'src/minio/minio.service'; @@ -10,7 +12,10 @@ export class KvService { private readonly kvMetadataPath = `${this.kvPrefix}/${this.kvMetadataFileName}`; private readonly logger: Logger = new Logger(KvService.name); - constructor(private readonly minioService: MinioService) {} + constructor( + private readonly minioService: MinioService, + @InjectQueue('kv') private kvProcessingQueue: Queue, + ) {} public generateFilePath(namespace: string, key: string): string { return `${this.kvPrefix}/${namespace}/${key}`; @@ -114,9 +119,11 @@ export class KvService { ...extraData, secretKey: this.generateSecretKey(), claimed: new Date().toISOString(), + namespace, }; await this.setMetadataFile(Buffer.from(JSON.stringify(metadata))); this.logger.verbose(`Claimed namespace ${namespace}`); + this.kvProcessingQueue.add('namespaceModeration', metadata[namespace]); return metadata[namespace]; } @@ -139,7 +146,7 @@ export class KvService { throw new Error('Incorrect secret key for namespace'); } this.logger.verbose(`Setting ${namespace}/${key}`); - return await this.minioService.uploadBuffer( + const uploadResult = await this.minioService.uploadBuffer( this.kvBucketName, this.generateFilePath(namespace, key), Buffer.from(value), @@ -149,6 +156,7 @@ export class KvService { ...options, }, ); + return uploadResult; } public async delete(namespace: string, key: string): Promise { diff --git a/yarn.lock b/yarn.lock index 94ce436..a068deb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -924,12 +924,12 @@ picocolors "^1.0.0" tslib "^2.6.0" -"@redis/bloom@1.2.0": +"@redis/bloom@1.2.0", "@redis/bloom@^1.2.0": version "1.2.0" resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-1.2.0.tgz#d3fd6d3c0af3ef92f26767b56414a370c7b63b71" integrity sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg== -"@redis/client@1.5.12": +"@redis/client@1.5.12", "@redis/client@^1.5.8": version "1.5.12" resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.5.12.tgz#4c387727992152aea443b869de0ebb697f899187" integrity sha512-/ZjE18HRzMd80eXIIUIPcH81UoZpwulbo8FmbElrjPqH0QC0SeIKu1BOU49bO5trM5g895kAjhvalt5h77q+4A== @@ -938,22 +938,22 @@ generic-pool "3.9.0" yallist "4.0.0" -"@redis/graph@1.1.1": +"@redis/graph@1.1.1", "@redis/graph@^1.1.0": version "1.1.1" resolved "https://registry.yarnpkg.com/@redis/graph/-/graph-1.1.1.tgz#8c10df2df7f7d02741866751764031a957a170ea" integrity sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw== -"@redis/json@1.0.6": +"@redis/json@1.0.6", "@redis/json@^1.0.4": version "1.0.6" resolved "https://registry.yarnpkg.com/@redis/json/-/json-1.0.6.tgz#b7a7725bbb907765d84c99d55eac3fcf772e180e" integrity sha512-rcZO3bfQbm2zPRpqo82XbW8zg4G/w4W3tI7X8Mqleq9goQjAGLL7q/1n1ZX4dXEAmORVZ4s1+uKLaUOg7LrUhw== -"@redis/search@1.1.6": +"@redis/search@1.1.6", "@redis/search@^1.1.3": version "1.1.6" resolved "https://registry.yarnpkg.com/@redis/search/-/search-1.1.6.tgz#33bcdd791d9ed88ab6910243a355d85a7fedf756" integrity sha512-mZXCxbTYKBQ3M2lZnEddwEAks0Kc7nauire8q20oA0oA/LoA+E/b5Y5KZn232ztPb1FkIGqo12vh3Lf+Vw5iTw== -"@redis/time-series@1.0.5": +"@redis/time-series@1.0.5", "@redis/time-series@^1.0.4": version "1.0.5" resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.5.tgz#a6d70ef7a0e71e083ea09b967df0a0ed742bc6ad" integrity sha512-IFjIgTusQym2B5IZJG3XKr5llka7ey84fw/NOYqESP5WUfQs9zz1ww/9+qoz4ka/S6KcGBodzlCeZ5UImKbscg== @@ -1967,14 +1967,21 @@ bytes@3.1.2: resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.2.tgz#8b0beeb98605adf1b128fa4386403c009e0221a5" integrity sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg== -cache-manager-redis-store@^3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/cache-manager-redis-store/-/cache-manager-redis-store-3.0.1.tgz#8eeb211212763d04cef4058666182d624f714299" - integrity sha512-o560kw+dFqusC9lQJhcm6L2F2fMKobJ5af+FoR2PdnMVdpQ3f3Bz6qzvObTGyvoazQJxjQNWgMQeChP4vRTuXQ== +cache-manager-redis-yet@^4.1.2: + version "4.1.2" + resolved "https://registry.yarnpkg.com/cache-manager-redis-yet/-/cache-manager-redis-yet-4.1.2.tgz#fa04df1a979a42585393a7a9918168978a7211ec" + integrity sha512-pM2K1ZlOv8gQpE1Z5mcDrfLj5CsNKVRiYua/SZ12j7LEDgfDeFVntI6JSgIw0siFSR/9P/FpG30scI3frHwibA== dependencies: - redis "^4.3.1" + "@redis/bloom" "^1.2.0" + "@redis/client" "^1.5.8" + "@redis/graph" "^1.1.0" + "@redis/json" "^1.0.4" + "@redis/search" "^1.1.3" + "@redis/time-series" "^1.0.4" + cache-manager "^5.2.2" + redis "^4.6.7" -cache-manager@^5.3.1: +cache-manager@^5.2.2, cache-manager@^5.3.1: version "5.3.1" resolved "https://registry.yarnpkg.com/cache-manager/-/cache-manager-5.3.1.tgz#80e4edd593b2d7f7b2b2199dc6fbb4748432bc47" integrity sha512-9HP6nc1ZqyZgcVEpy5XS2ns9MYE6cPEM6InA1wQhR6M7GviJzLH2NTFYnf3NEfRmLE351NCSkDo2VISX8dlG+w== @@ -5033,7 +5040,7 @@ redis-parser@^3.0.0: dependencies: redis-errors "^1.0.0" -redis@^4.3.1, redis@^4.6.11: +redis@^4.6.11, redis@^4.6.7: version "4.6.11" resolved "https://registry.yarnpkg.com/redis/-/redis-4.6.11.tgz#fad85e104545228f212259fd557c3e4f8eafcd3d" integrity sha512-kg1Lt4NZLYkAjPOj/WcyIGWfZfnyfKo1Wg9YKVSlzhFwxpFIl3LYI8BWy1Ab963LLDsTz2+OwdsesHKljB3WMQ==