Add initial jobs implementation

2024-04-11 22:28:56 -06:00
parent 73c91a7c63
commit aa1277fafd
6 changed files with 280 additions and 0 deletions
--- a/src/jobs/jobs.service.ts
+++ b/src/jobs/jobs.service.ts
@@ -0,0 +1,156 @@
+import { Inject, Injectable } from '@nestjs/common';
+import { MinioService } from 'src/minio/minio.service';
+import Redis from 'ioredis'
+import { InjectRedis } from '@liaoliaots/nestjs-redis';
+import { CACHE_MANAGER } from '@nestjs/cache-manager';
+import { Cache } from 'cache-manager';
+
+@Injectable()
+export class JobsService {
+    constructor(
+        private readonly minioService: MinioService,
+        @InjectRedis() private readonly redis: Redis,
+        @Inject(CACHE_MANAGER) private readonly cacheManager: Cache,
+    ) { }
+
+    private jobNameBuilder(jobName: string) {
+        return `job:${jobName}`;
+    }
+
+    private todoListNameBuilder(jobName: string) {
+        return `todo:${jobName}`;
+    }
+
+    private doneListNameBuilder(jobName: string) {
+        return `done:${jobName}`;
+    }
+
+    private claimedListNameBuilder(jobName: string) {
+        return `claimed:${jobName}`;
+    }
+
+    private completeCountNameBuilder(jobName: string, claimer: string) {
+        return `complete:${jobName}:${claimer}`;
+    }
+
+    private claimerCountNameBuilder(jobName: string, claimer: string) {
+        return `claim:${jobName}:${claimer}`;
+    }
+
+    private async getCompleteCounts(jobName: string) {
+        const keys = await this.redis.keys(`complete:${jobName}:*`);
+        const counts = await Promise.all(keys.map(async key => {
+            const count = await this.redis.get(key);
+            if (!count) {
+                return null;
+            }
+            return { claimer: key.split(':')[2], count: parseInt(count) };
+        }));
+        return counts.reduce((acc: any, val: any) => {
+            if (!val) {
+                return acc;
+            }
+            acc[val.claimer] = val.count;
+            return acc;
+        }, {})
+    }
+
+    private async getClaimCounts(jobName: string) {
+        const keys = await this.redis.keys(`claim:${jobName}:*`);
+        const counts = await Promise.all(keys.map(async key => {
+            const count = await this.redis.get(key);
+            if (!count) {
+                return null;
+            }
+            return { claimer: key.split(':')[2], count: parseInt(count) };
+        }));
+        return counts.reduce((acc: any, val: any) => {
+            if (!val) {
+                return acc;
+            }
+            acc[val.claimer] = val.count;
+            return acc;
+        }, {});
+    }
+
+    async getLeaderboard(jobName: string) {
+        const cachedLeaderboard = await this.cacheManager.get(`leaderboard:${jobName}`);
+        if (cachedLeaderboard) {
+            return cachedLeaderboard;
+        }
+        const completeCounts = await this.getCompleteCounts(jobName);
+        const claimCounts = await this.getClaimCounts(jobName);
+        this.cacheManager.set(`leaderboard:${jobName}`, { completeCounts, claimCounts }, 200);
+        return { completeCounts, claimCounts };
+    }
+
+
+    async addItemsToJob(jobName: string, items: string[]) {
+        await this.redis.rpush(this.todoListNameBuilder(jobName), ...items);
+    }
+
+    async claimJobItem(jobName: string, claimer: string): Promise<string | null> {
+        const jobItem = await this.redis.brpoplpush(this.todoListNameBuilder(jobName), this.claimedListNameBuilder(jobName), 10);
+        if (jobItem) {
+            await this.redis.rpush(this.jobNameBuilder(jobName), JSON.stringify({ item: jobItem, client: claimer }));
+        }
+        await this.redis.incr(this.claimerCountNameBuilder(jobName, claimer));
+        return jobItem;
+    }
+
+    async completeJobItem(jobName: string, jobItem: string, completer: string, data: any) {
+        await this.redis.lrem(this.claimedListNameBuilder(jobName), 1, jobItem);
+        await this.redis.lrem(this.todoListNameBuilder(jobName), 1, JSON.stringify({ item: jobItem, client: completer }));
+        await this.redis.rpush(this.doneListNameBuilder(jobName), JSON.stringify({ item: jobItem, client: completer, data }));
+        await this.redis.decr(this.claimerCountNameBuilder(jobName, completer));
+        await this.redis.incr(this.completeCountNameBuilder(jobName, completer));
+    }
+
+    async getTodoItems(jobName: string) {
+        return this.redis.lrange(this.todoListNameBuilder(jobName), 0, -1);
+    }
+
+    async getTodoItemCount(jobName: string) {
+        return this.redis.llen(this.todoListNameBuilder(jobName));
+    }
+
+    async getClaimedItems(jobName: string) {
+        return this.redis.lrange(this.claimedListNameBuilder(jobName), 0, -1);
+    }
+
+    async getClaimedItemCount(jobName: string) {
+        return this.redis.llen(this.claimedListNameBuilder(jobName));
+    }
+
+    async getDoneItems(jobName: string) {
+        return this.redis.lrange(this.doneListNameBuilder(jobName), 0, -1);
+    }
+
+    async getDoneItemCount(jobName: string) {
+        return this.redis.llen(this.doneListNameBuilder(jobName));
+    }
+
+    async getJobs() {
+        return this.redis.keys('job:*');
+    }
+
+    async registerJob(jobName: string, metadata: any) {
+        await this.redis.set(this.jobNameBuilder(jobName), JSON.stringify(metadata));
+    }
+
+    async getJobMetadata(jobName: string): Promise<any | null> {
+        const result = await this.redis.get(this.jobNameBuilder(jobName))
+        if (!result) {
+            return null;
+        }
+        return JSON.parse(result)
+    }
+
+    async resetClaimedItems(jobName: string) {
+        const claimedItems = await this.getClaimedItems(jobName);
+        for (const claimedItem of claimedItems) {
+            await this.redis.rpoplpush(this.claimedListNameBuilder(jobName), this.todoListNameBuilder(jobName));
+        }
+    }
+
+}