chip/us-api
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Fix redis setup

Browse Source
This commit is contained in:
Chip Wasson
2023-11-20 18:53:57 -07:00
parent a94ec100a5
commit 6ecdd5a0b8
6 changed files with 65 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
package.json
View File

@@ -34,7 +34,7 @@
"bcrypt": "^5.1.1",
"bull": "^4.11.5",
"cache-manager": "^5.3.1",
"cache-manager-redis-store": "^3.0.1",
"cache-manager-redis-yet": "^4.1.2",
"hbs": "^4.2.0",
"minio": "^7.1.3",
"open-graph-scraper": "^6.3.0",

51
src/app.module.ts
View File

@@ -16,9 +16,9 @@ import { PrometheusModule } from '@willsoto/nestjs-prometheus';
import { MinioModule } from './minio/minio.module';
import { KvModule } from './kv/kv.module';
import { BullModule } from '@nestjs/bull';
import type { RedisClientOptions } from 'redis';
import { redisStore } from 'cache-manager-redis-store';
import { ScheduleModule } from '@nestjs/schedule';
import { RedisClientOptions } from 'redis';
import { redisStore } from 'cache-manager-redis-yet';
@Module({
imports: [
@@ -27,31 +27,34 @@ import { ScheduleModule } from '@nestjs/schedule';
isGlobal: true,
load: [configuration],
}),
// CacheModule.registerAsync<RedisClientOptions>({
// isGlobal: true,
// imports: [ConfigModule],
// inject: [ConfigService],
// useFactory: async (configService: ConfigService) => ({
// store: redisStore as unknown as CacheStoreFactory,
// ttl: configService.get<number>('CACHE_TTL', 5),
// // Store-specific configuration:
// password: configService.get<string>('redis.pass', ''),
// host: configService.get<string>('redis.host', 'localhost'),
// port: configService.get<number>('redis.port', 6379),
// db: configService.get<number>('redis.db', 1),
// }),
// }),
CacheModule.registerAsync<RedisClientOptions>({
isGlobal: true,
inject: [ConfigService],
imports: [ConfigModule],
useFactory: async (configService: ConfigService) => ({
store: redisStore,
socket: {
host: configService.get<string>('redis.host'),
port: configService.get<number>('redis.port'),
},
database: configService.get<number>('redis.db'),
password: configService.get<string>('redis.password'),
}),
}),
BullModule.forRootAsync({
imports: [ConfigModule],
inject: [ConfigService],
useFactory: async (configService: ConfigService) => ({
redis: {
password: configService.get<string>('redis.pass', ''),
host: configService.get<string>('redis.host', 'localhost'),
port: configService.get<number>('redis.port', 6379),
db: configService.get<number>('redis.db', 1),
},
}),
useFactory: async (configService: ConfigService) => {
const config = {
redis: {
password: configService.get<string>('redis.password'),
host: configService.get<string>('redis.host'),
port: configService.get<number>('redis.port'),
db: configService.get<number>('redis.db'),
},
};
return config;
},
}),
PrometheusModule.register({
defaultLabels: {

2
src/config/configuration.ts
View File

@@ -12,7 +12,7 @@ export default () => ({
password: process.env.IRC_PASSWORD ?? '',
},
redis: {
host: process.env.REDIS_HOST ?? 's3.hooli.co',
host: process.env.REDIS_HOST ?? 'redis-master',
port: parseInt(process.env.REDIS_PORT ?? '6379'),
password: process.env.REDIS_PASS ?? '',
db: parseInt(process.env.REDIS_DB ?? '1'),

6
src/kv/kv.module.ts
View File

@@ -2,9 +2,15 @@ import { Module } from '@nestjs/common';
import { KvService } from './kv.service';
import { KvController } from './kv.controller';
import { MinioService } from 'src/minio/minio.service';
import { BullModule } from '@nestjs/bull';
@Module({
providers: [KvService, MinioService],
imports: [
BullModule.registerQueue({
name: 'kv',
}),
],
controllers: [KvController],
})
export class KvModule {}

12
src/kv/kv.service.ts
View File

@@ -1,4 +1,6 @@
import { InjectQueue } from '@nestjs/bull';
import { Injectable, Logger } from '@nestjs/common';
import { Queue } from 'bull';
import { UploadedObjectInfo } from 'minio';
import { MinioService } from 'src/minio/minio.service';
@@ -10,7 +12,10 @@ export class KvService {
private readonly kvMetadataPath = `${this.kvPrefix}/${this.kvMetadataFileName}`;
private readonly logger: Logger = new Logger(KvService.name);
constructor(private readonly minioService: MinioService) {}
constructor(
private readonly minioService: MinioService,
@InjectQueue('kv') private kvProcessingQueue: Queue,
) {}
public generateFilePath(namespace: string, key: string): string {
return `${this.kvPrefix}/${namespace}/${key}`;
@@ -114,9 +119,11 @@ export class KvService {
...extraData,
secretKey: this.generateSecretKey(),
claimed: new Date().toISOString(),
namespace,
};
await this.setMetadataFile(Buffer.from(JSON.stringify(metadata)));
this.logger.verbose(`Claimed namespace ${namespace}`);
this.kvProcessingQueue.add('namespaceModeration', metadata[namespace]);
return metadata[namespace];
}
@@ -139,7 +146,7 @@ export class KvService {
throw new Error('Incorrect secret key for namespace');
}
this.logger.verbose(`Setting ${namespace}/${key}`);
return await this.minioService.uploadBuffer(
const uploadResult = await this.minioService.uploadBuffer(
this.kvBucketName,
this.generateFilePath(namespace, key),
Buffer.from(value),
@@ -149,6 +156,7 @@ export class KvService {
...options,
},
);
return uploadResult;
}
public async delete(namespace: string, key: string): Promise<void> {

33
yarn.lock
View File

@@ -924,12 +924,12 @@
picocolors "^1.0.0"
tslib "^2.6.0"
"@redis/bloom@1.2.0":
"@redis/bloom@1.2.0", "@redis/bloom@^1.2.0":
version "1.2.0"
resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-1.2.0.tgz#d3fd6d3c0af3ef92f26767b56414a370c7b63b71"
integrity sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==
"@redis/client@1.5.12":
"@redis/client@1.5.12", "@redis/client@^1.5.8":
version "1.5.12"
resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.5.12.tgz#4c387727992152aea443b869de0ebb697f899187"
integrity sha512-/ZjE18HRzMd80eXIIUIPcH81UoZpwulbo8FmbElrjPqH0QC0SeIKu1BOU49bO5trM5g895kAjhvalt5h77q+4A==
@@ -938,22 +938,22 @@
generic-pool "3.9.0"
yallist "4.0.0"
"@redis/graph@1.1.1":
"@redis/graph@1.1.1", "@redis/graph@^1.1.0":
version "1.1.1"
resolved "https://registry.yarnpkg.com/@redis/graph/-/graph-1.1.1.tgz#8c10df2df7f7d02741866751764031a957a170ea"
integrity sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==
"@redis/json@1.0.6":
"@redis/json@1.0.6", "@redis/json@^1.0.4":
version "1.0.6"
resolved "https://registry.yarnpkg.com/@redis/json/-/json-1.0.6.tgz#b7a7725bbb907765d84c99d55eac3fcf772e180e"
integrity sha512-rcZO3bfQbm2zPRpqo82XbW8zg4G/w4W3tI7X8Mqleq9goQjAGLL7q/1n1ZX4dXEAmORVZ4s1+uKLaUOg7LrUhw==
"@redis/search@1.1.6":
"@redis/search@1.1.6", "@redis/search@^1.1.3":
version "1.1.6"
resolved "https://registry.yarnpkg.com/@redis/search/-/search-1.1.6.tgz#33bcdd791d9ed88ab6910243a355d85a7fedf756"
integrity sha512-mZXCxbTYKBQ3M2lZnEddwEAks0Kc7nauire8q20oA0oA/LoA+E/b5Y5KZn232ztPb1FkIGqo12vh3Lf+Vw5iTw==
"@redis/time-series@1.0.5":
"@redis/time-series@1.0.5", "@redis/time-series@^1.0.4":
version "1.0.5"
resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.5.tgz#a6d70ef7a0e71e083ea09b967df0a0ed742bc6ad"
integrity sha512-IFjIgTusQym2B5IZJG3XKr5llka7ey84fw/NOYqESP5WUfQs9zz1ww/9+qoz4ka/S6KcGBodzlCeZ5UImKbscg==
@@ -1967,14 +1967,21 @@ bytes@3.1.2:
resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.2.tgz#8b0beeb98605adf1b128fa4386403c009e0221a5"
integrity sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==
cache-manager-redis-store@^3.0.1:
version "3.0.1"
resolved "https://registry.yarnpkg.com/cache-manager-redis-store/-/cache-manager-redis-store-3.0.1.tgz#8eeb211212763d04cef4058666182d624f714299"
integrity sha512-o560kw+dFqusC9lQJhcm6L2F2fMKobJ5af+FoR2PdnMVdpQ3f3Bz6qzvObTGyvoazQJxjQNWgMQeChP4vRTuXQ==
cache-manager-redis-yet@^4.1.2:
version "4.1.2"
resolved "https://registry.yarnpkg.com/cache-manager-redis-yet/-/cache-manager-redis-yet-4.1.2.tgz#fa04df1a979a42585393a7a9918168978a7211ec"
integrity sha512-pM2K1ZlOv8gQpE1Z5mcDrfLj5CsNKVRiYua/SZ12j7LEDgfDeFVntI6JSgIw0siFSR/9P/FpG30scI3frHwibA==
dependencies:
redis "^4.3.1"
"@redis/bloom" "^1.2.0"
"@redis/client" "^1.5.8"
"@redis/graph" "^1.1.0"
"@redis/json" "^1.0.4"
"@redis/search" "^1.1.3"
"@redis/time-series" "^1.0.4"
cache-manager "^5.2.2"
redis "^4.6.7"
cache-manager@^5.3.1:
cache-manager@^5.2.2, cache-manager@^5.3.1:
version "5.3.1"
resolved "https://registry.yarnpkg.com/cache-manager/-/cache-manager-5.3.1.tgz#80e4edd593b2d7f7b2b2199dc6fbb4748432bc47"
integrity sha512-9HP6nc1ZqyZgcVEpy5XS2ns9MYE6cPEM6InA1wQhR6M7GviJzLH2NTFYnf3NEfRmLE351NCSkDo2VISX8dlG+w==
@@ -5033,7 +5040,7 @@ redis-parser@^3.0.0:
dependencies:
redis-errors "^1.0.0"
redis@^4.3.1, redis@^4.6.11:
redis@^4.6.11, redis@^4.6.7:
version "4.6.11"
resolved "https://registry.yarnpkg.com/redis/-/redis-4.6.11.tgz#fad85e104545228f212259fd557c3e4f8eafcd3d"
integrity sha512-kg1Lt4NZLYkAjPOj/WcyIGWfZfnyfKo1Wg9YKVSlzhFwxpFIl3LYI8BWy1Ab963LLDsTz2+OwdsesHKljB3WMQ==