Add assembly ai conversion

.dockerignore

@@ -33,3 +33,6 @@ lerna-debug.log*
 !.vscode/tasks.json
 !.vscode/launch.json
 !.vscode/extensions.json
+
+# fly.io
+fly.toml

.env.example

@@ -1,7 +1,8 @@
-JWT_SECRET=#jBhALXeYXC3$#e9
+NODE_ENV=development
+JWT_SECRET=jBhALXeYXC3$
 RAPID_API_KEY=
-IRC_SERVER=irc.libera.chat
-IRC_CHANNEL="##usdev-dev"
+IRC_SERVER=
+IRC_CHANNEL=
 
 REDIS_HOST=localhost
 REDIS_PASS=password
@@ -12,3 +13,7 @@ S3_USE_SSL=false
 S3_ACCESS_KEY="localminio"
 S3_SECRET_KEY="localminio"
 S3_BUCKET="devbucket"
+
+FOCO_LIVE_AIRTABLE_APIKEY=
+
+MAILGUN_SEND_KEY_HOOLI=

.github/workflows/fly-deploy.yml

@@ -0,0 +1,18 @@
+# See https://fly.io/docs/app-guides/continuous-deployment-with-github-actions/
+
+name: Fly Deploy
+on:
+  push:
+    branches:
+      - main
+jobs:
+  deploy:
+    name: Deploy app
+    runs-on: ubuntu-latest
+    concurrency: deploy-group    # optional: ensure only one action runs at a time
+    steps:
+      - uses: actions/checkout@v4
+      - uses: superfly/flyctl-actions/setup-flyctl@master
+      - run: flyctl deploy --remote-only
+        env:
+          FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}

.gitignore

@@ -37,3 +37,6 @@ lerna-debug.log*
 !.vscode/extensions.json
 
 data/
+
+# Ignore Bruno for now
+api.us.dev/

README.md

@@ -16,13 +16,22 @@ cp .env.example .env
 yarn
 # Start local services
 docker compose up -d
-# Copy default data into local minio
-cp -r default/* data/minio/devbucket
 # Start Application
 yarn start:dev
+# Log in with development user (admin) and dev password (password)
+curl --request POST \
+  --url http://localhost:3000/auth/login \
+  --header 'Content-Type: application/json' \
+  --data '{
+	"username":"admin",
+	"password":"password"
+}'
 ```
 
-Visit http://localhost:3000/api
+The Login command will return a JSON object with `access_token`. You can then
+head over to http://localhost:3000/api, click the "Authorize" button, and enter
+the `access_token`. This will then be applied to all of the endpoints that
+require auth on the API page.
 
 ## Configuration
 

docker-compose.yaml

@@ -21,8 +21,6 @@ services:
       - db_net
     ports:
       - 6379:6379
-    command: >
-      --requirepass ${REDIS_PASS}
     env_file:
       - .env
 networks:

fly.toml

@@ -0,0 +1,44 @@
+# fly.toml app configuration file generated for us-api on 2024-09-01T10:29:03-06:00
+#
+# See https://fly.io/docs/reference/configuration/ for information about how to use this file.
+#
+
+app = 'us-api'
+primary_region = 'den'
+
+[build]
+
+[http_service]
+  internal_port = 3000
+  force_https = true
+  auto_stop_machines = 'stop'
+  auto_start_machines = true
+  min_machines_running = 1
+  processes = ['app']
+
+[[vm]]
+  size = 'shared-cpu-2x'
+
+# [mounts]
+#   source = "usapi_data"
+#   destination = "/data"
+#   initial_size = "20gb"
+#   auto_extend_size_increment = "10gb"
+#   auto_extend_size_threshold = 80
+#   auto_extend_size_limit = "200gb"
+
+[env]
+DATA_ROOT_PATH = "/data"
+FILE_DEFAULT_TTL = "600"
+S3_ENDPOINT="s3.us-west-002.backblazeb2.com"
+S3_PORT="443"
+S3_USE_SSL="true"
+S3_BUCKET="api-us-dev"
+REDIS_DB="0"
+NODE_ENV="production"
+JUNK_DRAWER_BUCKET_NAME="api-us-dev"
+JUNK_DRAWER_ROOT_PATH="junk-drawer"
+
+[metrics]
+port = 3000
+path = "/metrics"

package.json

@@ -20,6 +20,7 @@
     "test:e2e": "jest --config ./test/jest-e2e.json"
   },
   "dependencies": {
+    "@liaoliaots/nestjs-redis": "^9.0.5",
     "@nestjs/bull": "^10.0.1",
     "@nestjs/cache-manager": "^2.1.1",
     "@nestjs/common": "^10.0.0",
@@ -30,13 +31,18 @@
     "@nestjs/schedule": "^4.0.0",
     "@nestjs/swagger": "^7.1.11",
     "@willsoto/nestjs-prometheus": "^6.0.0",
+    "airtable": "^0.12.2",
     "axios": "^1.5.0",
     "bcrypt": "^5.1.1",
     "bull": "^4.11.5",
     "cache-manager": "^5.3.1",
     "cache-manager-redis-yet": "^4.1.2",
+    "fp-ts": "^2.16.3",
+    "haversine-ts": "^1.2.0",
     "hbs": "^4.2.0",
-    "minio": "^7.1.3",
+    "ioredis": "^5.3.2",
+    "mailgun.js": "^10.2.3",
+    "minio": "^8.0.1",
     "open-graph-scraper": "^6.3.0",
     "prom-client": "^15.0.0",
     "ramda": "^0.29.0",
@@ -44,6 +50,7 @@
     "reflect-metadata": "^0.1.13",
     "rxjs": "^7.8.1",
     "slate-irc": "^0.9.3",
+    "uuid": "^10.0.0",
     "xml": "^1.0.1"
   },
   "devDependencies": {
@@ -59,6 +66,7 @@
     "@types/ramda": "^0.29.3",
     "@types/slate-irc": "^0.0.29",
     "@types/supertest": "^2.0.12",
+    "@types/uuid": "^10.0.0",
     "@types/xml": "^1.0.9",
     "@typescript-eslint/eslint-plugin": "^6.0.0",
     "@typescript-eslint/parser": "^6.0.0",

src/adsb-exchange/adsb-exchange.controller.ts

@@ -0,0 +1,116 @@
+import {
+  BadRequestException,
+  Controller,
+  Get,
+  Logger,
+  Param,
+  UseInterceptors,
+} from '@nestjs/common';
+import { ExtendedAircraft, LiteAircraft } from './types';
+import { AdsbExchangeService, ValidRadius } from './adsb-exchange.service';
+import { CacheInterceptor, CacheTTL } from '@nestjs/cache-manager';
+import { ApiParam, ApiTags } from '@nestjs/swagger';
+
+@Controller('adsb-exchange')
+@UseInterceptors(CacheInterceptor)
+@CacheTTL(60)
+@ApiTags('adsb-exchange')
+export class AdsbExchangeController {
+  private readonly logger: Logger = new Logger(AdsbExchangeController.name);
+  constructor(private readonly adsbExchangeService: AdsbExchangeService) {}
+
+  @Get('radius/lat/:lat/long/:long/dist/:dist')
+  @ApiParam({
+    name: 'lat',
+    type: Number,
+    example: 40.6043,
+    required: true,
+    description: 'Latitude of the center point',
+  })
+  @ApiParam({
+    name: 'long',
+    type: Number,
+    example: -105.01919,
+    required: true,
+    description: 'Longitude of the center point, note that West is negative',
+  })
+  @ApiParam({
+    name: 'dist',
+    type: Number,
+    example: 10,
+    required: true,
+    description:
+      'Radius in nautical miles. Must be one of 1, 5, 10, 25, 50, 100, 250',
+  })
+  async adsbExchangeAircraftWithinRadius(
+    @Param('lat') lat: number,
+    @Param('long') long: number,
+    @Param('dist') dist: number,
+  ): Promise<ExtendedAircraft[]> {
+    dist = Number(dist);
+    lat = Number(lat);
+    long = Number(long);
+    this.logger.verbose(
+      `Requesting aircraft within ${dist} nautical miles of ${lat}, ${long}`,
+    );
+    if (!this.adsbExchangeService.isValidRadius(dist)) {
+      throw new BadRequestException(`Invalid radius: ${dist}`);
+    }
+    return this.adsbExchangeService.extendAircraft(
+      await this.adsbExchangeService.adsbExchangeAircraftWithinRadius(dist, {
+        lat,
+        long,
+      }),
+      { lat, long },
+    );
+  }
+
+  @Get('radius/lat/:lat/long/:long/dist/:dist/lite')
+  @ApiParam({
+    name: 'lat',
+    type: Number,
+    example: 40.6043,
+    required: true,
+    description: 'Latitude of the center point',
+  })
+  @ApiParam({
+    name: 'long',
+    type: Number,
+    example: -105.01919,
+    required: true,
+    description: 'Longitude of the center point, note that West is negative',
+  })
+  @ApiParam({
+    name: 'dist',
+    type: Number,
+    example: 10,
+    required: true,
+    description:
+      'Radius in nautical miles. Must be one of 1, 5, 10, 25, 50, 100, 250',
+  })
+  async adsbExchangeAircraftWithinRadiusLite(
+    @Param('lat') lat: number,
+    @Param('long') long: number,
+    @Param('dist') dist: number,
+  ): Promise<LiteAircraft[]> {
+    dist = Number(dist);
+    lat = Number(lat);
+    long = Number(long);
+    this.logger.verbose(
+      `Requesting aircraft within ${dist} nautical miles of ${lat}, ${long}`,
+    );
+    if (!this.adsbExchangeService.isValidRadius(dist)) {
+      throw new BadRequestException(`Invalid radius: ${dist}`);
+    }
+    const aircraft = await this.adsbExchangeService.extendAircraft(
+      await this.adsbExchangeService.adsbExchangeAircraftWithinRadius(dist, {
+        lat,
+        long,
+      }),
+      { lat, long },
+    );
+    return this.adsbExchangeService
+      .liteAircraft(aircraft)
+      .filter((aircraft) => aircraft.registration !== undefined);
+  }
+}

src/adsb-exchange/adsb-exchange.module.ts

@@ -0,0 +1,11 @@
+import { Module } from '@nestjs/common';
+import { AdsbExchangeService } from './adsb-exchange.service';
+import { AdsbExchangeController } from './adsb-exchange.controller';
+import { CacheModule } from '@nestjs/cache-manager';
+
+@Module({
+  imports: [CacheModule.register()],
+  providers: [AdsbExchangeService],
+  controllers: [AdsbExchangeController],
+})
+export class AdsbExchangeModule {}

src/adsb-exchange/adsb-exchange.service.ts

@@ -0,0 +1,81 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { AdsbExchange, Bearing, ExtendedAircraft } from './types';
+import { DDPoint, Haversine, UnitOfDistance } from 'haversine-ts';
+import axios from 'axios';
+import { ConfigService } from '@nestjs/config';
+
+const ValidRadii = [1, 5, 10, 25, 50, 100, 250] as const;
+export type ValidRadius = (typeof ValidRadii)[number];
+
+@Injectable()
+export class AdsbExchangeService {
+  private readonly logger: Logger = new Logger(AdsbExchangeService.name);
+
+  constructor(private readonly configService: ConfigService) {}
+
+  isValidRadius(radius: number): radius is ValidRadius {
+    return ValidRadii.includes(radius as ValidRadius);
+  }
+
+  bearingToAircraft(
+    aircraft: AdsbExchange.Aircraft,
+    location: AdsbExchange.Location,
+  ): Bearing {
+    const locationPoint = new DDPoint(location.lat, location.long);
+    const aircraftPoint = new DDPoint(aircraft.lat, aircraft.lon);
+
+    const haversine = new Haversine(UnitOfDistance.Mile);
+    const bearing = haversine.getBearing(locationPoint, aircraftPoint);
+    return {
+      bearing: bearing.start,
+      distance: haversine.getDistance(locationPoint, aircraftPoint),
+    };
+  }
+
+  async adsbExchangeAircraftWithinRadius(
+    radius: ValidRadius,
+    location: AdsbExchange.Location,
+  ): Promise<AdsbExchange.Aircraft[]> {
+    this.logger.verbose(
+      `Requesting aircraft within ${radius} nautical miles of ${location.lat}, ${location.long}`,
+    );
+    const options = {
+      method: 'GET',
+      url: `https://adsbexchange-com1.p.rapidapi.com/v2/lat/${location.lat}/lon/${location.long}/dist/${radius}/`,
+      headers: {
+        'x-rapidapi-key': this.configService.get<string>('rapidApiKey'),
+        'x-rapidapi-host': 'adsbexchange-com1.p.rapidapi.com',
+      },
+    };
+
+    try {
+      const response =
+        await axios.request<AdsbExchange.AdsbExchangeRadiusResponse>(options);
+      return response.data.ac;
+    } catch (error) {
+      this.logger.error(error);
+      return [];
+    }
+  }
+
+  extendAircraft(
+    aircraft: AdsbExchange.Aircraft[],
+    location: AdsbExchange.Location,
+  ): ExtendedAircraft[] {
+    return aircraft.map((a) => ({
+      ...a,
+      bearing: this.bearingToAircraft(a, location),
+    }));
+  }
+
+  liteAircraft = (aircraft: ExtendedAircraft[]) =>
+    aircraft.map((a) => ({
+      flight: a.flight,
+      registration: a.r,
+      type: a.t,
+      altitude: a.alt_baro,
+      speed: a.gs,
+      track: a.track,
+      bearing: a.bearing,
+    }));
+}

src/adsb-exchange/types.ts

@@ -0,0 +1,189 @@
+export namespace AdsbExchange {
+  export const AircraftMessageTypes = [
+    'adsb_icao',
+    'adsb_icao_nt',
+    'adsr_icao',
+    'tisb_icao',
+    'adsc',
+    'mlat',
+    'other',
+    'mode_s',
+    'adsb_other',
+    'adsr_other',
+    'tisb_other',
+    'tisb_trackfile',
+  ] as const;
+
+  export type AircraftMessageType = (typeof AircraftMessageTypes)[number];
+
+  export interface Location {
+    lat: number;
+    long: number;
+  }
+
+  export interface Aircraft {
+    /**
+     * The 24-bit ICAO identifier of the aircraft
+     */
+    hex: string;
+    /**
+     * Type of underlying message/best source of data
+     */
+    type: AircraftMessageType | string;
+    /**
+     * Flight callsign or aircraft registration as 8 characters
+     */
+    flight: string;
+    /**
+     * Aircraft registration pulled from database
+     */
+    r: string;
+    /**
+     * Aircraft type pulled from database
+     */
+    t: string;
+    /**
+     * Barometric altitude in feet or "ground"
+     */
+    alt_baro: number | 'ground';
+    /**
+     * Geometric (GNSS/INS) altitude in feet
+     */
+    alt_geom: number;
+    /**
+     * Groundspeed in knots
+     */
+    gs: number;
+    /**
+     * True track over ground in degrees
+     */
+    track: number;
+    /**
+     * Rate of change of geometric altitude in feet per minute
+     */
+    geom_rate: number;
+    /**
+     * Mode A squawk code encoded as 4 octal digits
+     */
+    squawk: string;
+    /**
+     * Emergency status
+     */
+    emergency: string;
+    /**
+     * Emitter category to identify particular aircraft
+     * or vehicle classes. https://www.adsbexchange.com/emitter-category-ads-b-do-260b-2-2-3-2-5-2/
+     */
+    category: string;
+    /**
+     * Aircraft location in decimal degrees
+     */
+    lat: number;
+    /**
+     * Aircraft location in decimal degrees
+     */
+    lon: number;
+    /**
+     * Navigation integrity category
+     */
+    nic: number;
+    /**
+     * Radius of containment in meters
+     */
+    rc: number;
+    /**
+     * How long ago in seconds the aircraft was last seen
+     */
+    seen_pos: number;
+    /**
+     * ADS-B version
+     */
+    version: number;
+    /**
+     * Navigation integrity category for barometric altitude
+     */
+    nic_baro: number;
+    /**
+     * Navigation accuracy for position
+     */
+    nac_p: number;
+    /**
+     * Navigation accuracy for velocity
+     */
+    nac_v: number;
+    /**
+     * Source integrity level
+     */
+    sil: number;
+    /**
+     * Interpretation of SIL
+     */
+    sil_type: 'unknown' | 'perhour' | 'persample';
+    /**
+     * Geometric vertical accuracy
+     */
+    gva: number;
+    /**
+     * System design assurance level
+     */
+    sda: number;
+    /**
+     * Flight status alert bit
+     */
+    alert: number;
+    /**
+     * Flight status special position indicator
+     */
+    spi: number;
+    mlat: [];
+    tisb: [];
+    /**
+     * Total number of mode S messages received
+     * for this aircraft
+     */
+    messages: number;
+    /**
+     * How long ago in seconds a message was last received
+     * for this aircraft
+     */
+    seen: number;
+    /**
+     * Recent average signal strength in dbFS
+     */
+    rssi: number;
+  }
+
+  export interface AdsbExchangeRadiusResponse {
+    ac: Aircraft[];
+    msg: string;
+    now: number;
+    total: number;
+    ctime: number;
+    ptime: number;
+  }
+}
+
+export interface Bearing {
+  /**
+   * Bearing in degrees
+   */
+  bearing: number;
+  /**
+   * Distance in miles
+   */
+  distance: number;
+}
+
+export interface LiteAircraft {
+  flight?: string;
+  registration?: string;
+  type: string;
+  altitude: number | 'ground';
+  speed: number;
+  track: number;
+  bearing: Bearing;
+}
+
+export type ExtendedAircraft = AdsbExchange.Aircraft & {
+  bearing: Bearing;
+};

src/app.controller.spec.ts

@@ -16,7 +16,7 @@ describe('AppController', () => {
 
   describe('root', () => {
     it('should return "Hello World!"', () => {
-      expect(appController.getHello()).toBe('Hello World!');
+      expect(appController.getHello()).toBe("Hello World! This is Chip's generalized API for fetching information and things. You can contact me on mastodon @chip@talking.dev.");
     });
   });
 });

src/app.controller.ts

@@ -1,4 +1,4 @@
-import { Controller, Get } from '@nestjs/common';
+import { Controller, Get, Req } from '@nestjs/common';
 import { AppService } from './app.service';
 
 @Controller()
@@ -9,4 +9,9 @@ export class AppController {
   getHello(): string {
     return this.appService.getHello();
   }
+
+  @Get('headers')
+  getHeaders(@Req() req: Request) {
+    return req.headers;
+  }
 }

src/app.module.ts

@@ -8,10 +8,8 @@ import { UsersModule } from './users/users.module';
 import { ConfigModule, ConfigService } from '@nestjs/config';
 import { DomainrproxyModule } from './domainrproxy/domainrproxy.module';
 import configuration from './config/configuration';
-import { CacheModule, CacheStoreFactory } from '@nestjs/cache-manager';
+import { CacheInterceptor, CacheModule } from '@nestjs/cache-manager';
 import { IrcbotModule } from './ircbot/ircbot.module';
-import { DinosaurwetModule } from './dinosaurwet/dinosaurwet.module';
-import { OgScraperModule } from './ogscraper/ogscraper.module';
 import { PrometheusModule } from '@willsoto/nestjs-prometheus';
 import { MinioModule } from './minio/minio.module';
 import { KvModule } from './kv/kv.module';
@@ -20,6 +18,20 @@ import { ScheduleModule } from '@nestjs/schedule';
 import { RedisClientOptions } from 'redis';
 import { redisStore } from 'cache-manager-redis-yet';
 import { RedirectsModule } from './redirects/redirects.module';
+import { FileModule } from './file/file.module';
+import { FocoLiveModule } from './foco-live/foco-live.module';
+import { PowModule } from './pow/pow.module';
+import { FocoCoffeeModule } from './fococoffee/fococoffee.module';
+import { JobsModule } from './jobs/jobs.module';
+import { RedisModule, RedisModuleOptions } from '@liaoliaots/nestjs-redis';
+import { AdsbExchangeModule } from './adsb-exchange/adsb-exchange.module';
+import { APP_INTERCEPTOR } from '@nestjs/core';
+import { JunkDrawerModule } from './junk-drawer/junk-drawer.module';
+import { EmailModule } from './email/email.module';
+import { ContactModule } from './contact/contact.module';
+import { HoardingModule } from './hoarding/hoarding.module';
+import { NamesModule } from './names/names.module';
+import { AssemblyAiModule } from './assembly-ai/assembly-ai.module';
 
 @Module({
   imports: [
@@ -28,54 +40,85 @@ import { RedirectsModule } from './redirects/redirects.module';
       isGlobal: true,
       load: [configuration],
     }),
-    CacheModule.registerAsync<RedisClientOptions>({
-      isGlobal: true,
-      inject: [ConfigService],
+    RedisModule.forRootAsync({
       imports: [ConfigModule],
-      useFactory: async (configService: ConfigService) => ({
-        store: redisStore,
-        socket: {
-          host: configService.get<string>('redis.host'),
-          port: configService.get<number>('redis.port'),
+      inject: [ConfigService],
+      useFactory: async (
+        configService: ConfigService,
+      ): Promise<RedisModuleOptions> => {
+        return {
+          config: {
+            url: configService.get<string>('redis.url'),
+            db: configService.get<number>('redis.db'),
+            family: 6,
+          },
+        };
       },
-        database: configService.get<number>('redis.db'),
-        password: configService.get<string>('redis.password'),
-      }),
     }),
+    // CacheModule.registerAsync<RedisClientOptions>({
+    //   isGlobal: true,
+    //   inject: [ConfigService],
+    //   imports: [ConfigModule],
+    //   useFactory: async (configService: ConfigService) => ({
+    //     store: redisStore,
+    //     url: configService.get<string>('redis.url'),
+    //     family: 6,
+    //     database: configService.get<number>('redis.db'),
+    //   }),
+    // }),
+    CacheModule.register({ isGlobal: true }),
     BullModule.forRootAsync({
       imports: [ConfigModule],
       inject: [ConfigService],
       useFactory: async (configService: ConfigService) => {
         const config = {
           redis: {
-            password: configService.get<string>('redis.password'),
-            host: configService.get<string>('redis.host'),
-            port: configService.get<number>('redis.port'),
+            url: configService.get<string>('redis.url'),
             db: configService.get<number>('redis.db'),
+            family: 6,
           },
         };
         return config;
       },
     }),
     PrometheusModule.register({
+      defaultMetrics: {
+        enabled: false,
+      },
       defaultLabels: {
         app: 'us.dev api',
       },
     }),
-    CacheModule.register({ isGlobal: true }),
     ParkioModule,
     IswordModule,
     AuthModule,
     UsersModule,
     DomainrproxyModule,
     IrcbotModule,
-    DinosaurwetModule,
-    OgScraperModule,
     MinioModule,
     KvModule,
     RedirectsModule,
+    FileModule,
+    FocoLiveModule,
+    PowModule,
+    FocoCoffeeModule,
+    JobsModule,
+    AdsbExchangeModule,
+    JunkDrawerModule,
+    EmailModule,
+    ContactModule,
+    HoardingModule,
+    NamesModule,
+    AssemblyAiModule,
   ],
   controllers: [AppController],
-  providers: [AppService],
+  providers: [
+    AppService,
+    {
+      provide: APP_INTERCEPTOR,
+      useClass: CacheInterceptor,
+    },
+  ],
+  exports: [PrometheusModule],
 })
-export class AppModule { }
+export class AppModule {}

src/assembly-ai/assembly-ai.controller.ts

@@ -0,0 +1,37 @@
+import { Body, Controller, Post } from '@nestjs/common';
+import { ApiResponse } from '@nestjs/swagger';
+
+interface Utterance {
+  speaker: string;
+  text: string;
+  start: number;
+  end: number;
+  confidence: number;
+  words: Array<{
+    start: number;
+    end: number;
+    text: string;
+    confidence: number;
+    speaker: string;
+  }>;
+}
+
+@Controller('assembly-ai')
+export class AssemblyAiController {
+  @Post('utterance-to-script')
+  @ApiResponse({
+    status: 200,
+    description: 'Converts utterances to a script format',
+    type: String,
+  })
+  async utteranceToScript(
+    @Body() body: { utterances: Utterance[] } | Utterance[],
+  ) {
+    const utterances = Array.isArray(body) ? body : body.utterances;
+    return utterances.map(
+      (utterance) => `
+${utterance.speaker}: 
+${utterance.text}`,
+    );
+  }
+}

src/assembly-ai/assembly-ai.module.ts

@@ -0,0 +1,7 @@
+import { Module } from '@nestjs/common';
+import { AssemblyAiController } from './assembly-ai.controller';
+
+@Module({
+  controllers: [AssemblyAiController]
+})
+export class AssemblyAiModule {}

src/auth/auth.controller.ts

@@ -10,7 +10,35 @@ import {
 } from '@nestjs/common';
 import { AuthService } from './auth.service';
 import { AuthGuard } from './auth.guard';
-import { ApiBearerAuth, ApiTags } from '@nestjs/swagger';
+import { ApiBearerAuth, ApiBody, ApiProperty, ApiTags } from '@nestjs/swagger';
+
+class HashDto {
+  @ApiProperty({
+    description: 'The password to hash',
+    default: 'password',
+  })
+  password: string;
+
+  @ApiProperty({
+    description: 'The number of bcrypt hashing rounds',
+    default: 10,
+  })
+  rounds?: number;
+}
+
+class LoginDto {
+  @ApiProperty({
+    description: 'The username to authenticate',
+    default: 'admin',
+  })
+  username: string;
+
+  @ApiProperty({
+    description: 'The password to authenticate',
+    default: 'password',
+  })
+  password: string;
+}
 
 @ApiTags('auth')
 @Controller('auth')
@@ -19,14 +47,17 @@ export class AuthController {
 
   @HttpCode(HttpStatus.OK)
   @Post('login')
+  @ApiBody({ type: LoginDto })
   signIn(@Body() signInDto: Record<string, any>) {
     return this.authService.signIn(signInDto.username, signInDto.password);
   }
 
   @UseGuards(AuthGuard)
   @Post('hash')
-  hash(@Body() hashDto: { pass: string; rounds?: number }) {
-    return this.authService.hash(hashDto.pass, hashDto.rounds);
+  @ApiBody({ type: HashDto })
+  @ApiBearerAuth()
+  hash(@Body() hashDto: HashDto) {
+    return this.authService.hash(hashDto.password, hashDto.rounds);
   }
 
   @UseGuards(AuthGuard)

src/config/configuration.ts

@@ -1,24 +1,58 @@
+require('dotenv').config();
+
 export default () => ({
   port: parseInt(process.env.PORT ?? '', 10) || 3000,
+  isProduction: process.env.NODE_ENV === 'production',
   // UserAgent should be added to calls made to third party apis
   userAgent: 'api.us.dev/@chip@talking.dev',
   rapidApiKey: process.env.RAPID_API_KEY || '',
   irc: {
-    enabled: process.env.IRC_SERVER !== undefined,
+    enabled:
+      process.env.IRC_SERVER === '' || process.env.IRC_SERVER === undefined
+        ? false
+        : true,
     server: process.env.IRC_SERVER,
     tls: process.env.IRC_TLS === 'true',
     port: parseInt(process.env.IRC_PORT ?? '6697'),
     channel: process.env.IRC_CHANNEL ?? '#usdev',
     password: process.env.IRC_PASSWORD ?? '',
     nick:
-      process.env.IRC_NICK ?? process.env.NODE_ENV === 'production'
+      (process.env.IRC_NICK ?? process.env.NODE_ENV === 'production')
         ? 'us-bot'
         : 'us-dev',
   },
   redis: {
-    host: process.env.REDIS_HOST ?? 'redis-master',
-    port: parseInt(process.env.REDIS_PORT ?? '6379'),
-    password: process.env.REDIS_PASS ?? '',
+    url: process.env.REDIS_URL ?? '',
     db: parseInt(process.env.REDIS_DB ?? '1'),
   },
+  file: {
+    bucketName:
+      process.env.FILE_BUCKET_NAME ??
+      process.env.S3_BUCKET ??
+      'api.us.dev-files',
+    // default file ttl in seconds
+    defaultTtl: parseInt(
+      process.env.FILE_DEFAULT_TTL ?? (30 * 24 * 60 * 60).toString(),
+    ),
+  },
+  s3: {
+    bucketName: process.env.S3_BUCKET ?? 'api.us.dev',
+  },
+  focoLive: {
+    airtable: {
+      apiKey: process.env.FOCO_LIVE_AIRTABLE_APIKEY ?? '',
+    },
+  },
+  junkDrawer: {
+    bucketName: process.env.JUNK_DRAWER_BUCKET_NAME ?? 'junk-drawer',
+    rootPath: process.env.JUNK_DRAWER_ROOT_PATH ?? '',
+  },
+  mailgun: {
+    hooliKey: process.env.MAILGUN_SEND_KEY_HOOLI ?? '',
+  },
+  thirdPartyServices: {
+    lichess: {
+      token: process.env.SVC_LICHESS_TOKEN ?? '',
+    },
+  },
 });

src/contact/contact.controller.ts

@@ -0,0 +1,85 @@
+import {
+  Body,
+  Controller,
+  Get,
+  NotFoundException,
+  Param,
+  Post,
+  UseInterceptors,
+} from '@nestjs/common';
+import { FileInterceptor } from '@nestjs/platform-express';
+import { ApiConsumes, ApiParam, ApiQuery, ApiTags } from '@nestjs/swagger';
+import { EmailService } from 'src/email/email.service';
+import { PowService } from 'src/pow/pow.service';
+
+interface Destination {
+  email: string;
+  powChallenge: boolean;
+}
+
+type Destinations = {
+  [key: string]: Destination;
+};
+
+const destinations: Destinations = {
+  focolive: {
+    // email: 'hello@fortcollinslive.com',
+    email: 'mailtest@chip.bz',
+    powChallenge: true,
+  },
+};
+
+@Controller('contact')
+@ApiTags('contact')
+export class ContactController {
+  constructor(
+    private readonly powService: PowService,
+    private readonly emailService: EmailService,
+  ) {}
+
+  @Get(':destination/challenge')
+  @ApiParam({
+    name: 'destination',
+    required: true,
+    description: 'The destination to send the email to, the contact "class"',
+  })
+  async getChallenge(@Param('destination') destination: string) {
+    const dest = destinations[destination];
+    if (!dest) {
+      throw new NotFoundException();
+    }
+    if (dest.powChallenge) {
+      return {
+        challenge: await this.powService.generateChallenge(),
+        difficulty: this.powService.getDifficulty(),
+      };
+    }
+  }
+  @Post(':destination')
+  @ApiConsumes('multipart/form-data')
+  @UseInterceptors(FileInterceptor('file'))
+  async sendEmail(
+    @Param('destination') destination: string,
+    @Body('challenge') challenge: string,
+    @Body('proof') proof: string,
+    @Body('subject') subject: string,
+    @Body('text') text: string,
+  ) {
+    const dest = destinations[destination];
+    if (!dest) {
+      throw new NotFoundException();
+    }
+    if (dest.powChallenge) {
+      if (!(await this.powService.verifyChallenge(challenge, proof))) {
+        throw new NotFoundException();
+      }
+    }
+    const result = await this.emailService.sendEmail(
+      [dest.email],
+      subject,
+      text,
+    );
+    this.powService.markChallengeAsComplete(challenge);
+    return result;
+  }
+}

src/contact/contact.module.ts

@@ -0,0 +1,12 @@
+import { Module } from '@nestjs/common';
+import { ContactController } from './contact.controller';
+import { EmailService } from 'src/email/email.service';
+import { PowService } from 'src/pow/pow.service';
+import { PowModule } from 'src/pow/pow.module';
+
+@Module({
+  imports: [PowModule],
+  controllers: [ContactController],
+  providers: [EmailService],
+})
+export class ContactModule {}

src/dinosaurwet/dinosaurwet.controller.ts

@@ -1,188 +0,0 @@
-import { Controller, Get, Res } from '@nestjs/common';
-import { Response } from 'express';
-import { identity, prop } from 'ramda';
-import * as xml from 'xml';
-import { InjectMetric } from '@willsoto/nestjs-prometheus';
-import { Gauge, Histogram } from 'prom-client';
-import { ApiTags } from '@nestjs/swagger';
-
-const episodeFiles: [number, string][] = [
-  [27544345, '01-Into-the-Big-Wet.mp3'],
-  [30724178, '02-Ocean-Rumours.mp3'],
-  [30752181, '03-Step-Two.mp3'],
-  [28793207, '04-Red-Ladder-Revelations.mp3'],
-  [29684714, '05-What-to-do-with-a-Problem-Like-Daveda.mp3'],
-  [29387545, '06-Hair-Bussy.mp3'],
-  [30789798, '07-Monkey-Monkey-Cock-a-Roach.mp3'],
-  [29449403, '08-SS-Jungle-and-Blood.mp3'],
-  [29554728, '09-A-Bad-Day-in-Sogland.mp3'],
-  [31728952, '10-A-Loathed-Trip-Down-Memory-Lane.mp3'],
-  [29711881, '11-Dog-Egg-Consumption.mp3'],
-  [31212772, '12-Meg-and-Bobs-Wedding-Bussy.mp3'],
-  [30951548, '13-100-Genuine-Pussy-Hound.mp3'],
-  [33766503, '14-One-Perfect-Answer.mp3'],
-  [29643754, '15-Succulent-Pig-Excitement.mp3'],
-  [29797145, '16-Welcome-to-Jackson-Island.mp3'],
-  [37926451, '17-Infinite-Meat.mp3'],
-  [33863888, '18-The-Jackson-Triumvirate.mp3'],
-  [30256482, '19-A-Lucky-Day-to-be-a-Jackson.mp3'],
-  [40947042, '20-The-Joels-Triumphant.mp3'],
-];
-
-const startDate = new Date('2023-10-16');
-
-type Episode = {
-  filename: string;
-  length: number;
-  url: string;
-  title: string;
-  description: string;
-  fakeReleaseDate: Date;
-};
-
-const dinosaurImage =
-  'https://imaginary.apps.hooli.co/enlarge?height=1400&width=1400&url=https://www.sanspantsradio.com/wp-content/uploads/2023/03/Dino-Wet-600x600.jpg';
-
-const episodes: Episode[] = episodeFiles.map((file, i) => {
-  const [length, filename] = file;
-  return {
-    filename,
-    length,
-    url: `https://s3.apps.hnl.byoi.net/dinosaurpark/DinosaurWet/${filename}`,
-    title: filename.replace('.mp3', '').replace(/-/g, ' ').trim(),
-    description:
-      'A bootleg-ish of Dinosar Wet, making the downloaded files more like and RSS feed',
-    fakeReleaseDate: new Date(
-      new Date(startDate).setDate(startDate.getDate() + i * 7),
-    ),
-  };
-});
-
-const buildFeeds = (
-  title: string,
-  episodes: Episode[],
-  pubDateFn: (index: number) => Date = () => startDate,
-) => {
-  const episodeItems = episodes
-    .filter((episode: Episode, index: number) => pubDateFn(index) <= new Date())
-    .map(({ title, url, length, fakeReleaseDate }, index) => ({
-      item: [
-        { title },
-        { guid: url },
-        { pubDate: pubDateFn(index).toUTCString() },
-        {
-          'itunes:image': {
-            _attr: {
-              href: dinosaurImage,
-            },
-          },
-        },
-        { 'itunes:episode': index + 1 },
-        {
-          enclosure: [
-            {
-              _attr: {
-                url,
-                type: 'audio/mpeg',
-                length,
-              },
-            },
-          ],
-        },
-      ],
-    }));
-  const feedObject = {
-    rss: [
-      {
-        _attr: {
-          version: '2.0',
-          'xmlns:atom': 'http://www.w3.org/2005/Atom',
-          'xmlns:itunes': 'http://www.itunes.com/dtds/podcast-1.0.dtd',
-          'xmlns:content': 'http://purl.org/rss/1.0/modules/content/',
-          'xmlns:podcast': 'https://podcastindex.org/namespace/1.0',
-        },
-      },
-      {
-        channel: [
-          {
-            'atom:link': {
-              _attr: {
-                href: 'https://api.us.dev/dinosaurwet',
-                rel: 'self',
-                type: 'application/rss+xml',
-              },
-            },
-          },
-          {
-            title,
-          },
-          {
-            link: 'https://sanspantsradio.com',
-          },
-          {
-            'itunes:image': {
-              _attr: {
-                href: dinosaurImage,
-              },
-            },
-          },
-          {
-            description:
-              'A recreation of Dinosaur Wet as an RSS feed for listeners-with-licenses enjoyment.',
-          },
-          { language: 'en-US' },
-          ...episodeItems,
-        ],
-      },
-    ],
-  };
-
-  const feed = '<?xml version="1.0" encoding="UTF-8"?>' + xml(feedObject);
-  return { feed, episodeItems };
-};
-
-@ApiTags('dinosaurwet')
-@Controller('dinosaurwet')
-export class DinosaurwetController {
-  constructor(
-    @InjectMetric('weekly_count') public weeklyCount: Gauge<string>,
-    @InjectMetric('daily_count') public dailyCount: Gauge<string>,
-    @InjectMetric('rss_query_count') public queryCount: Gauge<string>,
-  ) {}
-
-  @Get('')
-  getAllAtOnce(@Res() response: Response) {
-    this.queryCount.inc();
-    response.header('Content-Type', 'application/xml');
-    const feed = buildFeeds('Dinosaur Wet - All At Once', episodes);
-    return response.send(feed);
-  }
-
-  @Get('daily')
-  getDaily(@Res() response: Response) {
-    this.queryCount.inc();
-    response.header('Content-Type', 'application/xml');
-    const feed = buildFeeds(
-      'Dinosaur Wet - Daily',
-      episodes,
-      (index: number) =>
-        new Date(new Date(startDate).setDate(startDate.getDate() + index)),
-    );
-    this.dailyCount.set(feed.episodeItems.length);
-    return response.send(feed.feed);
-  }
-
-  @Get('weekly')
-  getWeekly(@Res() response: Response) {
-    this.queryCount.inc();
-    response.header('Content-Type', 'application/xml');
-    const feed = buildFeeds(
-      'Dinosaur Wet - Weekly',
-      episodes,
-      (index: number) =>
-        new Date(new Date(startDate).setDate(startDate.getDate() + index * 7)),
-    );
-    this.weeklyCount.set(feed.episodeItems.length);
-    return response.send(feed.feed);
-  }
-}

src/dinosaurwet/dinosaurwet.module.ts

@@ -1,33 +0,0 @@
-import { Module } from '@nestjs/common';
-import { DinosaurwetController } from './dinosaurwet.controller';
-import {
-  PrometheusModule,
-  makeGaugeProvider,
-} from '@willsoto/nestjs-prometheus';
-
-@Module({
-  imports: [
-    PrometheusModule.register({
-      customMetricPrefix: 'dinosaurwet',
-      defaultMetrics: {
-        enabled: false,
-      },
-    }),
-  ],
-  controllers: [DinosaurwetController],
-  providers: [
-    makeGaugeProvider({
-      name: 'daily_count',
-      help: 'The current daily Dinosaur Wet episode count',
-    }),
-    makeGaugeProvider({
-      name: 'weekly_count',
-      help: 'The current weekly Dinosaur Wet episode count',
-    }),
-    makeGaugeProvider({
-      name: 'rss_query_count',
-      help: 'Total RSS endpoint queries',
-    }),
-  ],
-})
-export class DinosaurwetModule {}

src/domainrproxy/domainrproxy.controller.ts

@@ -9,7 +9,7 @@ import {
 } from '@nestjs/swagger';
 import { DomainrParsedStatusResult } from './domainrproxy.service';
 
-@ApiTags('Domainr')
+@ApiTags('domainr')
 @Controller('domainrproxy')
 export class DomainrproxyController {
   constructor(private readonly proxyService: DomainrproxyService) {}

src/email/email.module.ts

@@ -0,0 +1,8 @@
+import { Module } from '@nestjs/common';
+import { EmailService } from './email.service';
+
+@Module({
+  providers: [EmailService],
+  exports: [EmailService],
+})
+export class EmailModule {}

src/email/email.service.ts

@@ -0,0 +1,36 @@
+import { Injectable } from '@nestjs/common';
+import Mailgun from 'mailgun.js';
+import * as formdata from 'form-data';
+import { IMailgunClient } from 'mailgun.js/Interfaces';
+import { ConfigService } from '@nestjs/config';
+
+@Injectable()
+export class EmailService {
+  private readonly mailgun: IMailgunClient | undefined;
+  private static readonly emailFromDomain = 'hello.hooli.co';
+
+  constructor(private readonly configService: ConfigService) {
+    const mailgun = new Mailgun(formdata);
+    const hooliKey = this.configService.get<string>('mailgun.hooliKey');
+    if (!hooliKey) {
+      return;
+    }
+    this.mailgun = mailgun.client({
+      username: 'api',
+      key: hooliKey,
+    });
+  }
+
+  async sendEmail(to: string[], subject: string, text: string) {
+    if (!this.mailgun) {
+      return;
+    }
+    return this.mailgun.messages.create(EmailService.emailFromDomain, {
+      from: `HooliMail <hooli-mail@${EmailService.emailFromDomain}>`,
+      to,
+      subject,
+      text,
+      html: text,
+    });
+  }
+}

src/file/file.controller.ts

@@ -0,0 +1,89 @@
+import {
+  Controller,
+  Get,
+  Param,
+  Redirect,
+  Render,
+  Res,
+  UploadedFile,
+  UseInterceptors,
+} from '@nestjs/common';
+import { FileService } from './file.service';
+import { Post } from '@nestjs/common';
+import { FileInterceptor } from '@nestjs/platform-express';
+import { ApiConsumes, ApiTags } from '@nestjs/swagger';
+import { InjectMetric } from '@willsoto/nestjs-prometheus';
+import { Response } from 'express';
+
+@Controller('file')
+@ApiTags('file')
+export class FileController {
+  constructor(private readonly fileService: FileService) {}
+
+  @Get()
+  @Render('file/upload')
+  generateUploadForm() {
+    return {};
+  }
+
+  @Get(':key/direct')
+  @Redirect('', 302)
+  async getFile(@Param('key') key: string) {
+    const url = await this.fileService.generatePresignedUrl(key);
+    return { url };
+  }
+
+  @Get(':key')
+  async getFileWithProxy(@Param('key') key: string, @Res() res: Response) {
+    if (!(await this.fileService.fileExists(key))) {
+      return res.status(404).send('File not found or expired');
+    }
+    const buffer = await this.fileService.getFileFromS3(key);
+    const metadata = await this.fileService.getFileMetadataFromS3(key);
+    const { originalfilename, 'content-type': contentType } = metadata.metaData;
+    return res
+      .header('Content-Type', contentType)
+      .header(
+        'Content-Disposition',
+        `attachment; filename="${originalfilename ?? key}"`,
+      )
+      .send(buffer);
+  }
+
+  @Post('upload')
+  @UseInterceptors(FileInterceptor('file'))
+  @ApiConsumes('multipart/form-data')
+  @Render('file/upload-result')
+  async handleFileUpload(
+    @UploadedFile() file: Express.Multer.File,
+  ): Promise<any> {
+    const upload = await this.fileService.handleFileUpload(file);
+    return {
+      ...upload,
+      expireTime: new Date(upload.expireAt * 1000).toLocaleString(),
+    };
+  }
+
+  @Post('upload.txt')
+  @UseInterceptors(FileInterceptor('file'))
+  @ApiConsumes('multipart/form-data')
+  async handleFileSilentUpload(
+    @UploadedFile() file: Express.Multer.File,
+  ): Promise<string> {
+    const upload = await this.fileService.handleFileUpload(file);
+    return `https://api.us.dev/file/${upload.key}`;
+  }
+
+  @Post('upload.json')
+  @UseInterceptors(FileInterceptor('file'))
+  @ApiConsumes('multipart/form-data')
+  async handleFileUploadJson(
+    @UploadedFile() file: Express.Multer.File,
+  ): Promise<any> {
+    const upload = await this.fileService.handleFileUpload(file);
+    return {
+      ...upload,
+      expireTime: new Date(upload.expireAt * 1000).toLocaleString(),
+    };
+  }
+}

src/file/file.module.ts

@@ -0,0 +1,10 @@
+import { Module } from '@nestjs/common';
+import { FileService } from './file.service';
+import { FileController } from './file.controller';
+import { MinioService } from 'src/minio/minio.service';
+
+@Module({
+  providers: [FileService, MinioService],
+  controllers: [FileController]
+})
+export class FileModule { }

src/file/file.service.ts

@@ -0,0 +1,124 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import { Cron } from '@nestjs/schedule';
+import { ItemBucketMetadata } from 'minio';
+import { UploadedObjectInfo } from 'minio/dist/main/internal/type';
+import { MinioService } from 'src/minio/minio.service';
+
+@Injectable()
+export class FileService {
+  private readonly logger = new Logger(FileService.name);
+  private readonly bucketName;
+  private readonly filePrefix = 'file';
+  private expirationTime: number;
+
+  constructor(
+    private readonly minioService: MinioService,
+    private readonly configService: ConfigService,
+  ) {
+    this.expirationTime = this.configService.get(
+      'file.defaultTtl',
+      30 * 24 * 60 * 60,
+    );
+    this.bucketName = this.configService.get(
+      'file.bucketName',
+      'api.us.dev-files',
+    );
+    try {
+      this.minioService.listBucketObjects(this.bucketName);
+    } catch (e) {
+      this.logger.log(`Error with bucket ${this.bucketName}: ${e.message}`);
+    }
+    this.deleteExpiredFiles();
+  }
+
+  private generateRandomKey(): string {
+    return Math.random().toString(36).substring(2);
+  }
+
+  private pathFromKey(key: string): string {
+    return [this.filePrefix, key].join('/');
+  }
+
+  async handleFileUpload(file: Express.Multer.File): Promise<{
+    uploadResult: UploadedObjectInfo;
+    expireAt: number;
+    key: string;
+    originalFilename: string;
+  }> {
+    const expireAt = Date.now() / 1000 + this.expirationTime;
+    const key = this.generateRandomKey();
+    const uploadResult = await this.minioService.uploadBuffer(
+      this.bucketName,
+      this.pathFromKey(key),
+      file.buffer,
+      {
+        expireAt,
+        originalFilename: file.originalname,
+        'content-type': file.mimetype,
+      },
+    );
+    return {
+      uploadResult,
+      key,
+      expireAt,
+      originalFilename: file.originalname,
+    };
+  }
+
+  async getFileFromS3(key: string): Promise<Buffer> {
+    return this.minioService.getBuffer(this.bucketName, this.pathFromKey(key));
+  }
+
+  async getFileMetadataFromS3(key: string): Promise<ItemBucketMetadata> {
+    return this.minioService.getObjectMetadata(
+      this.bucketName,
+      this.pathFromKey(key),
+    );
+  }
+
+  async fileExists(key: string): Promise<boolean> {
+    return this.minioService.objectExists(
+      this.bucketName,
+      this.pathFromKey(key),
+    );
+  }
+
+  @Cron('0 0 * * *')
+  private async deleteExpiredFiles(): Promise<void> {
+    this.logger.debug('Running cron job to delete expired files');
+    const now = Date.now() / 1000;
+    const objectNames = await this.minioService.listBucketObjects(
+      this.bucketName,
+      this.filePrefix,
+      true,
+    );
+    for (const objectName of objectNames) {
+      this.logger.debug(`Checking object ${objectName}`);
+      const objectInfo = await this.minioService.getObjectMetadata(
+        this.bucketName,
+        objectName,
+      );
+      if (objectInfo.metaData.expireat < now) {
+        this.logger.debug(`Deleting object ${objectName}`);
+        await this.minioService.deleteObject(this.bucketName, objectName);
+      }
+    }
+  }
+
+  async generatePresignedUrl(key: string): Promise<string> {
+    const objectPath = [this.filePrefix, key].join('/');
+    const metadata = await this.minioService.getObjectMetadata(
+      this.bucketName,
+      objectPath,
+    );
+    if (metadata.expireAt < Date.now() / 1000) {
+      throw new Error('Object has expired');
+    }
+    return await this.minioService.generatePresignedUrl(
+      this.bucketName,
+      objectPath,
+      10,
+    );
+  }
+}

src/foco-live/foco-live.controller.ts

@@ -0,0 +1,32 @@
+import { Controller, Get, Logger, Query } from '@nestjs/common';
+import { FocoLiveService } from './foco-live.service';
+import { ApiQuery, ApiTags } from '@nestjs/swagger';
+import { InjectMetric } from '@willsoto/nestjs-prometheus';
+import { Gauge } from 'prom-client';
+
+@ApiTags('foco-live')
+@Controller('foco-live')
+export class FocoLiveController {
+    private readonly logger = new Logger(FocoLiveController.name);
+
+    constructor(
+        private readonly focoLiveService: FocoLiveService,
+        @InjectMetric('query_count') public queryCount: Gauge<string>,
+    ) { }
+
+    @Get('events')
+    @ApiQuery({ name: 'venue', required: false })
+    @ApiQuery({ name: 'before', required: false, description: "Filter events that are before the provided date, any date format" })
+    @ApiQuery({ name: 'after', required: false, description: "Filter events that are after the provided date, any date format" })
+    async getEvents(@Query('venue') venue?: string, @Query('before') before?: string, @Query('after') after?: string) {
+        this.queryCount.inc();
+        this.logger.verbose(`GET /foco-live/events?venue=${venue}&before=${before}&after=${after}`);
+        return this.focoLiveService.getEvents({ venue, before: before ? new Date(before) : undefined, after: after ? new Date(after) : undefined });
+    }
+
+    @Get('venues')
+    async getVenues() {
+        this.queryCount.inc();
+        return this.focoLiveService.getAllVenuesCached();
+    }
+}

src/foco-live/foco-live.module.ts

@@ -0,0 +1,33 @@
+import { Module } from '@nestjs/common';
+import { FocoLiveController } from './foco-live.controller';
+import { FocoLiveService } from './foco-live.service';
+import { PrometheusModule, makeGaugeProvider } from '@willsoto/nestjs-prometheus';
+import { CacheModule } from '@nestjs/cache-manager';
+
+@Module({
+  imports: [
+    CacheModule.register(),
+    PrometheusModule.register({
+      customMetricPrefix: 'foco_live',
+      defaultMetrics: {
+        enabled: false,
+      },
+    })
+  ],
+  controllers: [FocoLiveController],
+  providers: [FocoLiveService,
+    makeGaugeProvider({
+      name: 'event_count',
+      help: 'The current number of events in the Foco Live database at last query time',
+    }),
+    makeGaugeProvider({
+      name: 'event_cache_misses',
+      help: 'The total number of cache misses for the Foco Live events',
+    }),
+    makeGaugeProvider({
+      name: 'query_count',
+      help: 'The total number of queries to the Foco Live API',
+    }),
+  ]
+})
+export class FocoLiveModule { }

src/foco-live/foco-live.service.ts

@@ -0,0 +1,152 @@
+import { CACHE_MANAGER } from '@nestjs/cache-manager';
+import { Inject, Injectable, Logger } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import { Cron } from '@nestjs/schedule';
+import { InjectMetric } from '@willsoto/nestjs-prometheus';
+import * as Airtable from 'airtable';
+import { AirtableBase } from 'airtable/lib/airtable_base';
+import { all } from 'axios';
+import { Cache } from 'cache-manager';
+import { Gauge } from 'prom-client';
+import { filter, pipe } from 'ramda';
+
+const tables = {
+  venues: 'tblRi4wDorKqNJJbs',
+  events: 'tbl4RZ75QF5WefE7L',
+};
+
+const compareDates = (a: any, b: any) =>
+  new Date(a.Date).getTime() - new Date(b.Date).getTime();
+
+const beforeFilter = (before?: Date) => (a: Event) =>
+  before ? new Date(a.Date) <= before : true;
+const afterFilter = (after?: Date) => (a: Event) =>
+  after ? new Date(a.Date) >= after : true;
+
+export interface Event {
+  Date: string;
+  'Music Start Time': string;
+  'Bar or Venue Name': string;
+  'Band/DJ/Musician Name': string;
+  'Cost Select': string;
+  Cost: string;
+  'Date Select': string;
+  '"Specials" at Venue': string;
+  Genre?: string;
+  id: string;
+}
+
+export interface Venue {
+  id: string;
+  'Bar or Venue Name': string;
+  'Street Address': string;
+  City: string;
+  'Zip Code': number;
+  State: string;
+  'Phone Number': string;
+  Website: string;
+  'Has Calendar Of Events': string;
+  'Facebook Page': string;
+  Instagram: string;
+  'Twitter Account': string;
+}
+
+const cacheKeys = {
+  allEvents: 'foco_live_events',
+  allVenues: 'foco_live_venues',
+};
+
+@Injectable()
+export class FocoLiveService {
+  private readonly airtableBase: AirtableBase;
+  private readonly logger = new Logger(FocoLiveService.name);
+
+  constructor(
+    private readonly config: ConfigService,
+    @InjectMetric('event_count') public eventCount: Gauge<string>,
+    @InjectMetric('event_cache_misses') public cacheMisses: Gauge<string>,
+    @Inject(CACHE_MANAGER) private cacheManager: Cache,
+  ) {
+    this.airtableBase = new Airtable({
+      apiKey: config.get('focoLive.airtable.apiKey'),
+    }).base('app1SjPrn5qrhr59J');
+  }
+
+  async getAllEvents(): Promise<Event[]> {
+    return (
+      (
+        await this.airtableBase('Events')
+          .select({
+            view: 'Grid view',
+          })
+          .all()
+      ).map((record) => ({
+        id: record.id,
+        ...record.fields,
+      })) as any as Event[]
+    )
+      .sort(compareDates)
+      .reverse();
+  }
+
+  async getAllEventsCached(): Promise<Event[]> {
+    let events: Event[] | null | undefined = await this.cacheManager.get(
+      cacheKeys.allEvents,
+    );
+    if (!events) {
+      events = await this.getAllEvents();
+      this.cacheMisses.inc();
+      this.cacheManager.set(cacheKeys.allEvents, events, 10 * 60 * 1000);
+    }
+    this.eventCount.set(events.length);
+    return events;
+  }
+
+  async getAllVenues(): Promise<Venue[]> {
+    return (
+      (
+        await this.airtableBase('Venues')
+          .select({
+            view: 'Grid view',
+          })
+          .all()
+      ).map((record) => ({
+        id: record.id,
+        ...record.fields,
+      })) as any as Venue[]
+    )
+      .sort(compareDates)
+      .reverse();
+  }
+
+  async getAllVenuesCached(): Promise<Venue[]> {
+    return (
+      (await this.cacheManager.get(cacheKeys.allVenues)) ||
+      (await this.getAllVenues())
+    );
+  }
+
+  @Cron('0 */5 * * * *')
+  async refreshEvents() {
+    this.logger.verbose('Refreshing events cache.');
+    await this.getAllEventsCached();
+  }
+
+  async getEvents(
+    options: { venue?: string; before?: Date; after?: Date } = {},
+  ) {
+    const events = await this.getAllEventsCached();
+    const results = pipe(
+      filter(
+        (a: Event) =>
+          a['Bar or Venue Name'] === (options.venue ?? a['Bar or Venue Name']),
+      ),
+      filter(beforeFilter(options.before)),
+      filter(afterFilter(options.after)),
+    )(events);
+    this.logger.verbose(
+      `Returning ${results.length} events, ${events.length} total events in database.`,
+    );
+    return results;
+  }
+}

src/fococoffee/bindle.service.ts

@@ -0,0 +1,20 @@
+import { Injectable } from '@nestjs/common';
+import { ShopifyProduct, ShopifyProductWithHelpers, addShopifyHelperProperties, parseShopifyProduct } from './shopifyUtils';
+import axios from 'axios';
+import { map, pipe } from 'ramda';
+
+@Injectable()
+export class BindleService {
+    public readonly shopifyUrl = 'https://bindlecoffee.com/products.json'
+
+    public async fetchProducts(): Promise<ShopifyProductWithHelpers[]> {
+        const response = await axios.get(this.shopifyUrl)
+        if (response.status !== 200) {
+            throw new Error('Failed to fetch products')
+        }
+        return pipe(
+            map(parseShopifyProduct),
+            map(addShopifyHelperProperties({ shopBaseUrl: 'https://bindlecoffee.com' }))
+        )(response.data.products as ShopifyProduct<string>[])
+    }
+}

src/fococoffee/fococoffee.controller.ts

@@ -0,0 +1,50 @@
+import { Controller, Get, UseInterceptors } from '@nestjs/common';
+import { HarbingerService } from './harbinger.service';
+import { FocoCoffeeService } from './fococoffee.service';
+import { LimaService } from './lima.service';
+import { BindleService } from './bindle.service';
+import { ApiResponse, ApiTags } from '@nestjs/swagger';
+import { CacheInterceptor } from '@nestjs/cache-manager';
+
+@Controller('fococoffee')
+@ApiTags('fococoffee')
+@UseInterceptors(CacheInterceptor)
+export class FocoCoffeeController {
+
+    constructor(
+        private readonly focoCoffeeService: FocoCoffeeService,
+        private readonly harbingerService: HarbingerService,
+        private readonly limaService: LimaService,
+        private readonly bindleService: BindleService
+    ) { }
+
+    @Get('harbinger/products')
+    @ApiResponse({ status: 200, description: 'Returns the list of Harbinger products' })
+    async getHarbingerProducts() {
+        return this.harbingerService.fetchProducts()
+    }
+
+    @Get('lima/products')
+    @ApiResponse({ status: 200, description: 'Returns the list of Lima products' })
+    async getLimaProducts() {
+        return this.limaService.fetchProducts()
+    }
+
+    @Get('bindle/products')
+    @ApiResponse({ status: 200, description: 'Returns the list of Bindle products' })
+    async getBindleProducts() {
+        return this.bindleService.fetchProducts()
+    }
+
+    @Get('beans')
+    @ApiResponse({ status: 200, description: 'Returns the list of coffee bean products from all suppliers' })
+    async getBeans() {
+        return this.focoCoffeeService.getBeans()
+    }
+
+    @Get('products')
+    @ApiResponse({ status: 200, description: 'Returns the list of all products from all suppliers' })
+    async getAllProducts() {
+        return this.focoCoffeeService.getAllProducts()
+    }
+}

src/fococoffee/fococoffee.module.ts

@@ -0,0 +1,13 @@
+import { Module } from '@nestjs/common';
+import { FocoCoffeeController } from './fococoffee.controller';
+import { FocoCoffeeService } from './fococoffee.service';
+import { HarbingerService } from './harbinger.service';
+import { LimaService } from './lima.service';
+import { BindleService } from './bindle.service';
+
+@Module({
+  imports: [],
+  controllers: [FocoCoffeeController],
+  providers: [FocoCoffeeService, HarbingerService, LimaService, BindleService]
+})
+export class FocoCoffeeModule { }

src/fococoffee/fococoffee.service.ts

@@ -0,0 +1,39 @@
+import { Injectable } from '@nestjs/common';
+import { LimaService } from './lima.service';
+import { HarbingerService } from './harbinger.service';
+import { BindleService } from './bindle.service';
+
+@Injectable()
+export class FocoCoffeeService {
+    constructor(
+        private readonly limaService: LimaService,
+        private readonly harbingerService: HarbingerService,
+        private readonly bindleService: BindleService
+    ) { }
+
+    public async getAllProducts() {
+        const limaProducts = await this.limaService.fetchProducts()
+        const harbingerProducts = await this.harbingerService.fetchProducts()
+        const bindleProducts = await this.bindleService.fetchProducts()
+        return {
+            lima: limaProducts,
+            harbinger: harbingerProducts,
+            bindle: bindleProducts
+        }
+    }
+
+    public async getBeans() {
+        const limaBeans = await this.limaService.fetchProducts()
+        const harbingerBeans = await this.harbingerService.fetchProducts()
+        const bindleBeans = await this.bindleService.fetchProducts()
+        return {
+            lima: limaBeans
+                .filter(p => p.tags.includes("Coffee"))
+                .filter(p => !p.title.toLocaleLowerCase().includes("subscription"))
+                .filter(p => p.handle !== "lima-sample-pack")
+            ,
+            harbinger: harbingerBeans.filter(p => p.product_type === "Packaged Coffee"),
+            bindle: bindleBeans.filter(p => p.product_type === "Coffee, Roasted")
+        }
+    }
+}

src/fococoffee/harbinger.service.ts

@@ -0,0 +1,20 @@
+import { Injectable } from '@nestjs/common';
+import { ShopifyProduct, ShopifyProductWithHelpers, addShopifyHelperProperties, parseShopifyProduct } from './shopifyUtils';
+import axios from 'axios';
+import { map, pipe } from 'ramda';
+
+@Injectable()
+export class HarbingerService {
+    public readonly shopifyUrl = 'https://harbingercoffee.com/products.json'
+
+    public async fetchProducts(): Promise<ShopifyProductWithHelpers[]> {
+        const response = await axios.get(this.shopifyUrl)
+        if (response.status !== 200) {
+            throw new Error('Failed to fetch products')
+        }
+        return pipe(
+            map(parseShopifyProduct),
+            map(addShopifyHelperProperties({ shopBaseUrl: 'https://harbingercoffee.com' }))
+        )(response.data.products as ShopifyProduct<string>[])
+    }
+}

src/fococoffee/lima.service.ts

@@ -0,0 +1,20 @@
+import { Injectable } from '@nestjs/common';
+import { ShopifyProduct, ShopifyProductWithHelpers, addShopifyHelperProperties, parseShopifyProduct } from './shopifyUtils';
+import axios from 'axios';
+import { map, pipe } from 'ramda';
+
+@Injectable()
+export class LimaService {
+    public readonly shopifyUrl = 'https://www.limacoffeeroasters.com/products.json'
+
+    public async fetchProducts(): Promise<ShopifyProductWithHelpers[]> {
+        const response = await axios.get(this.shopifyUrl)
+        if (response.status !== 200) {
+            throw new Error('Failed to fetch products')
+        }
+        return pipe(
+            map(parseShopifyProduct),
+            map(addShopifyHelperProperties({ shopBaseUrl: 'https://www.limacoffeeroasters.com' }))
+        )(response.data.products as ShopifyProduct<string>[])
+    }
+}

src/fococoffee/shopifyUtils.ts

@@ -0,0 +1,96 @@
+import { pipe } from "ramda";
+
+export interface ShopifyVariant<T> {
+    id: number;
+    title: string;
+    price: string;
+    sku: string;
+    position: number;
+    option1?: string;
+    option2?: string;
+    option3?: string;
+    taxable: boolean;
+    grams: number;
+    available: boolean;
+    created_at: T;
+    updated_at: T;
+}
+
+export interface ShopifyImage<T> {
+    id: number;
+    created_at: T;
+    position: number;
+    updated_at: T;
+    product_id: number;
+    variant_ids: number[];
+    src: string;
+    width: number;
+    height: number;
+}
+
+export interface ShopifyOption {
+    name: string;
+    position: number;
+    values: string[];
+}
+
+export interface ShopifyProduct<T> {
+    id: number;
+    title: string;
+    handle: string;
+    body_html: string;
+    published_at: T;
+    created_at: T;
+    vendor: string;
+    product_type: string;
+    updated_at: T;
+    tags: string[];
+    variants: ShopifyVariant<T>[];
+    images: ShopifyImage<T>[];
+    options: ShopifyOption[];
+}
+
+
+export interface ShopifyProductWithHelpers extends Omit<ShopifyProduct<Date>, "images"> {
+    productUrl: string;
+    images: ({
+        imaginaryUrl: string;
+    } | ShopifyImage<Date>)[]
+}
+
+export interface helperData {
+    shopBaseUrl: string,
+}
+
+export const addShopifyHelperProperties = (data: helperData) => (productIn: ShopifyProduct<Date>): ShopifyProductWithHelpers => {
+    return {
+        productUrl: `${data.shopBaseUrl}/products/${productIn.handle}`,
+        ...productIn,
+        images: productIn.images.map((image) => ({
+            ...image,
+            imaginaryUrl: `https://imaginary.hooli.co/resize?width=250&height=250&url=${image.src}`
+        })),
+    }
+}
+
+export const parseShopifyProductDates = (productIn: ShopifyProduct<string | Date>): ShopifyProduct<Date> => {
+    return {
+        ...productIn,
+        published_at: new Date(productIn.published_at),
+        created_at: new Date(productIn.created_at),
+        updated_at: new Date(productIn.updated_at),
+        variants: productIn.variants.map((variant) => ({
+            ...variant,
+            created_at: new Date(variant.created_at),
+            updated_at: new Date(variant.updated_at),
+        })),
+        images: productIn.images.map((image) => ({
+            ...image,
+            created_at: new Date(image.created_at),
+            updated_at: new Date(image.updated_at),
+        })),
+    };
+}
+
+
+export const parseShopifyProduct = pipe(parseShopifyProductDates);

src/hoarding/hoarding.module.ts

@@ -0,0 +1,8 @@
+import { Module } from '@nestjs/common';
+import { HoardingService } from './hoarding.service';
+import { MinioService } from 'src/minio/minio.service';
+
+@Module({
+  providers: [HoardingService, MinioService],
+})
+export class HoardingModule {}

src/hoarding/hoarding.service.ts

@@ -0,0 +1,84 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import { Cron } from '@nestjs/schedule';
+import axios from 'axios';
+import { MinioService } from 'src/minio/minio.service';
+
+@Injectable()
+export class HoardingService {
+  private readonly logger: Logger = new Logger(HoardingService.name);
+  constructor(
+    public readonly minioService: MinioService,
+    private readonly configService: ConfigService,
+  ) {
+    this.hoardChessStats();
+  }
+
+  @Cron('0 0 1 * * *')
+  async hoardChessStats() {
+    try {
+      await this.hoardUserChessStats('sentientcrouton');
+    } catch (e) {
+      this.logger.error(e);
+    }
+    try {
+      await this.hoardUserChessStats('archy_type');
+    } catch (e) {
+      this.logger.error(e);
+    }
+    try {
+      await this.hoardLichessStats('sentientcrouton');
+    } catch (e) {
+      this.logger.error(e);
+    }
+    try {
+      await this.hoardLichessStats('archy_type');
+    } catch (e) {
+      this.logger.error(e);
+    }
+  }
+
+  async hoardLichessStats(user: string) {
+    if (!this.configService.get('thirdPartyServices.lichess.token')) {
+      this.logger.error('No lichess token found');
+      return;
+    }
+    const token = this.configService.get('thirdPartyServices.lichess.token');
+    this.logger.log(`Hoarding lichess stats for ${user}`);
+    const lichessState = (id: string) => `https://lichess.org/api/user/${id}`;
+    const stats = await axios.get(lichessState(user), {
+      headers: {
+        Authorization: `Bearer ${token}`,
+      },
+    });
+    await this.minioService.uploadBuffer(
+      this.minioService.defaultBucketName,
+      `hoarding/lichess-stats/${user}/${new Date().getTime()}-${user}.json`,
+      Buffer.from(
+        JSON.stringify({
+          time: new Date().getTime(),
+          data: stats.data,
+        }),
+      ),
+    );
+  }
+
+  async hoardUserChessStats(user: string) {
+    this.logger.log(`Hoarding chess.com stats for ${user}`);
+    const chessState = (id: string) =>
+      `https://api.chess.com/pub/player/${id}/stats`;
+    const stats = await axios.get(chessState(user));
+
+    const time = new Date().getTime();
+    await this.minioService.uploadBuffer(
+      this.minioService.defaultBucketName,
+      `hoarding/chess.com-stats/${user}/${time}-${user}.json`,
+      Buffer.from(
+        JSON.stringify({
+          time,
+          data: stats.data,
+        }),
+      ),
+    );
+  }
+}

src/ircbot/ircbot.service.ts

@@ -17,7 +17,10 @@ export class IrcbotService {
     public readonly configService: ConfigService,
     public readonly domainrProxy: DomainrproxyService,
   ) {
-    if (!this.configService.get<boolean>('irc.enabled')) return;
+    if (!this.configService.get<boolean>('irc.enabled')) {
+      this.logger.verbose('IRC disabled, not connecting');
+      return;
+    };
     const nick = this.configService.get<string>('irc.nick') || 'us-dev';
     const ircPassword = this.configService.get<string>('irc.password');
     this.socket = connect({
@@ -68,7 +71,7 @@ export class IrcbotService {
             this.client.send(channel, `Dunno what ${command} means`);
             return;
         }
-      } catch {}
+      } catch { }
     });
   }
 }

src/jobs/jobs.controller.ts

@@ -0,0 +1,190 @@
+import { Body, Controller, Get, Header, Headers, Param, Post, Render, Req, UploadedFile, UseInterceptors } from '@nestjs/common';
+import { JobMetadata, JobsService } from './jobs.service';
+import { ApiBearerAuth, ApiBody, ApiConsumes, ApiParam, ApiProperty, ApiTags } from '@nestjs/swagger';
+import { FileInterceptor } from '@nestjs/platform-express';
+
+
+class ClaimCompleteDto {
+    @ApiProperty({
+        description: 'Identity of the completer',
+        example: 'my-identity'
+    })
+    completer: string;
+
+    @ApiProperty({
+        description: 'Identity of the item to complete',
+        example: 'my-item'
+    })
+    item: string;
+
+    @ApiProperty({
+        description: 'Data to store with the completion',
+        example: { foo: 'bar' }
+    })
+    data: any;
+}
+
+@Controller('jobs')
+@ApiTags('jobs')
+export class JobsController {
+    constructor(
+        private readonly jobsService: JobsService,
+    ) { }
+
+    @Get(':jobName/stats.json')
+    @ApiParam({ name: 'jobName', required: true })
+    async getStats(@Param('jobName') jobName: string) {
+        return {
+            todoCount: await this.jobsService.getTodoItemCount(jobName),
+            claimedCount: await this.jobsService.getClaimedItemCount(jobName),
+            doneCount: await this.jobsService.getDoneItemCount(jobName),
+            doneScore: await this.jobsService.getDoneItemScoresTotal(jobName),
+        };
+    }
+
+    @Get(':jobName/stats')
+    @ApiParam({ name: 'jobName', required: true })
+    @Render('jobs/stats')
+    async getStatsPage(@Param('jobName') jobName: string) {
+        return {
+            jobName,
+            todoCount: await this.jobsService.getTodoItemCount(jobName),
+            claimedCount: await this.jobsService.getClaimedItemCount(jobName),
+            doneCount: await this.jobsService.getDoneItemCount(jobName),
+            doneScore: await this.jobsService.getDoneItemScoresTotal(jobName),
+        };
+    }
+
+    @Get(':jobName/todo.json')
+    @ApiParam({ name: 'jobName', required: true })
+    async getTodoItems(@Param('jobName') jobName: string) {
+        return this.jobsService.getTodoItems(jobName);
+    }
+
+    @Get(':jobName/done.json')
+    @ApiParam({ name: 'jobName', required: true })
+    async getDone(@Param('jobName') jobName: string) {
+        return this.jobsService.getDoneItems(jobName);
+    }
+
+    @Get(':jobName/claimed.json')
+    @ApiParam({ name: 'jobName', required: true })
+    async getClaimed(@Param('jobName') jobName: string) {
+        return this.jobsService.getClaimedItems(jobName);
+    }
+
+    @Get(':jobName/leaderboard.json')
+    @ApiParam({ name: 'jobName', required: true })
+    async getLeaderboard(@Param('jobName') jobName: string) {
+        return this.jobsService.getLeaderboard(jobName);
+    }
+
+    @Get(':jobName/leaderboard')
+    @ApiParam({ name: 'jobName', required: true })
+    @Render('jobs/leaderboard')
+    async getLeaderboardPage(@Param('jobName') jobName: string) {
+        const leaderboard = await this.jobsService.getLeaderboard(jobName);
+        type LeaderboardItem = { name: string, count: number }
+        return {
+            jobName,
+            claims: Object.keys(leaderboard.claimCounts).map(claimer => ({ name: claimer, count: leaderboard.claimCounts[claimer] })).sort((a: LeaderboardItem, b: LeaderboardItem) => b.count - a.count),
+            completes: Object.keys(leaderboard.completeCounts).map(completer => ({ name: completer, count: leaderboard.completeCounts[completer] })).sort((a: LeaderboardItem, b: LeaderboardItem) => b.count - a.count),
+        }
+    }
+
+    @Post(':jobName/register')
+    @ApiParam({ name: 'jobName', required: true })
+    @ApiBody({
+        description: "Job metadata", examples:
+        {
+            "example1": {
+                value: {
+                    name: 'my-job',
+                    description: 'My job description',
+                    tags: ['tag1', 'tag2'],
+                    createdBy: 'my-identity',
+                    createdAt: '2021-01-01T00:00:00Z',
+                    claimSecret: 'my-secret'
+                }
+            }
+        }
+    })
+    @ApiConsumes('application/json')
+    async claimJobItemJson(@Param('jobName') jobName: string, @Body() body: JobMetadata) {
+        return this.jobsService.registerJob(jobName, body);
+    }
+
+    @Get(':jobName.json')
+    @ApiParam({ name: 'jobName', required: true })
+    async getJobMetadata(@Param('jobName') jobName: string) {
+        const metadata = await this.jobsService.getPublicJobMetadata(jobName);
+        if (!metadata) {
+            return {
+                error: 'Job not registered',
+            };
+        }
+        return metadata;
+    }
+
+    @Post(':jobName/clear-todo')
+    @ApiParam({ name: 'jobName', required: true })
+    @ApiBearerAuth('x-claim-key')
+    async clearTodoItems(@Headers('x-claim-key') claimKey: string, @Param('jobName') jobName: string) {
+        return this.jobsService.clearTodoItems(jobName, claimKey);
+    }
+
+    @Get(':jobName/add')
+    @ApiParam({ name: 'jobName', required: true })
+    @Render('jobs/add')
+    async addItemsPage(@Param('jobName') jobName: string) {
+        return {
+            jobName,
+        };
+    }
+
+    @Post(':jobName/add')
+    @ApiParam({ name: 'jobName', required: true })
+    @ApiConsumes('text/plain')
+    @ApiBody({ type: String, description: "Items to add, one per line separated by newline characters" })
+    async addItemsToJob(@Param('jobName') jobName: string, @Body() items: string) {
+        return this.jobsService.addItemsToJob(jobName, items.split('\n'));
+    }
+
+    @Post(':jobName/addFile')
+    @UseInterceptors(FileInterceptor('file'))
+    @ApiParam({ name: 'jobName', required: true })
+    @ApiConsumes('multipart/form-data')
+    @ApiBody({ description: "File containing items to add, one per line separated by newline characters" })
+    async addItemsToJobFromFile(@Param('jobName') jobName: string, @UploadedFile() file: Express.Multer.File) {
+        return this.jobsService.addItemsToJob(
+            jobName,
+            file.buffer.toString().split('\n')
+        );
+    }
+
+    @Post(':jobName/claim')
+    @ApiParam({ name: 'jobName', required: true })
+    @ApiConsumes('text/plain')
+    @ApiBody({ type: String, description: "Claimer identity string" })
+    async claimJobItem(@Param('jobName') jobName: string, @Body() claimer: string) {
+        const item = await this.jobsService.claimJobItem(jobName, claimer);
+        return {
+            jobName,
+            claimer,
+            item
+        }
+    }
+
+    @Post(':jobName/complete')
+    @ApiParam({ name: 'jobName', required: true })
+    async completeJobItem(@Param('jobName') jobName: string, @Body() body: ClaimCompleteDto) {
+        return this.jobsService.completeJobItem(jobName, body.item, body.completer, body.data);
+    }
+
+    @Post(':jobName/reset-claimed')
+    @ApiParam({ name: 'jobName', required: true })
+    async resetClaimed(@Param('jobName') jobName: string) {
+        return this.jobsService.resetClaimedItems(jobName);
+    }
+
+}

src/jobs/jobs.module.ts

@@ -0,0 +1,10 @@
+import { Module } from '@nestjs/common';
+import { JobsService } from './jobs.service';
+import { JobsController } from './jobs.controller';
+import { MinioService } from 'src/minio/minio.service';
+
+@Module({
+  providers: [JobsService, MinioService],
+  controllers: [JobsController]
+})
+export class JobsModule { }

src/jobs/jobs.service.ts

@@ -0,0 +1,241 @@
+import { Inject, Injectable } from '@nestjs/common';
+import { MinioService } from 'src/minio/minio.service';
+import Redis from 'ioredis'
+import { InjectRedis } from '@liaoliaots/nestjs-redis';
+import { CACHE_MANAGER } from '@nestjs/cache-manager';
+import { Cache } from 'cache-manager';
+import { aperture, splitEvery } from 'ramda';
+
+export interface JobMetadata {
+    name: string;
+    description: string;
+    tags: string[];
+    createdBy: string;
+    createdAt: string;
+    claimSecret: string;
+}
+
+export const privateMetadataKeys = ['claimSecret'];
+
+export type PublicJobMetadata = Omit<JobMetadata, typeof privateMetadataKeys[number]>;
+
+type Leaderboard = {
+    completeCounts: { [claimer: string]: number },
+    claimCounts: { [claimer: string]: number }
+    doneScores: { [claimer: string]: number }
+}
+
+@Injectable()
+export class JobsService {
+    constructor(
+        private readonly minioService: MinioService,
+        @InjectRedis() private readonly redis: Redis,
+        @Inject(CACHE_MANAGER) private readonly cacheManager: Cache,
+    ) { }
+
+    public static cleanJobMetadata(metadata: JobMetadata): PublicJobMetadata {
+        return Object.fromEntries(Object.entries(metadata).filter(([key]) => !privateMetadataKeys.includes(key))) as PublicJobMetadata;
+    }
+
+    private jobNameBuilder(jobName: string) {
+        return `job:${jobName}`;
+    }
+
+    private todoListNameBuilder(jobName: string) {
+        return `todo:${jobName}`;
+    }
+
+    private doneListNameBuilder(jobName: string) {
+        return `done:${jobName}`;
+    }
+
+    private claimedListNameBuilder(jobName: string) {
+        return `claimed:${jobName}`;
+    }
+
+    private completeCountNameBuilder(jobName: string, claimer: string) {
+        return `complete:${jobName}:${claimer}`;
+    }
+
+    private claimerCountNameBuilder(jobName: string, claimer: string) {
+        return `claim:${jobName}:${claimer}`;
+    }
+
+    private claimerCountWildcardBuilder(jobName: string) {
+        return `claim:${jobName}:*`;
+    }
+
+    private async getCompleteCounts(jobName: string): Promise<{ [claimer: string]: number }> {
+        const keys = await this.redis.keys(`complete:${jobName}:*`);
+        const counts = await Promise.all(keys.map(async key => {
+            const count = await this.redis.get(key);
+            if (!count) {
+                return null;
+            }
+            return { claimer: key.split(':')[2], count: parseInt(count) };
+        }));
+        return counts.reduce((acc: any, val: any) => {
+            if (!val) {
+                return acc;
+            }
+            acc[val.claimer] = val.count;
+            return acc;
+        }, {})
+    }
+
+    private async getClaimCounts(jobName: string): Promise<{ [claimer: string]: number }> {
+        const keys = await this.redis.keys(`claim:${jobName}:*`);
+        const counts = await Promise.all(keys.map(async key => {
+            const count = await this.redis.get(key);
+            if (!count) {
+                return null;
+            }
+            return { claimer: key.split(':')[2], count: parseInt(count) };
+        }));
+        return counts.reduce((acc: any, val: any) => {
+            if (!val) {
+                return acc;
+            }
+            acc[val.claimer] = val.count;
+            return acc;
+        }, {});
+    }
+
+    async getLeaderboard(jobName: string): Promise<Leaderboard> {
+        const cachedLeaderboard = await this.cacheManager.get<Leaderboard>(`leaderboard:${jobName}`);
+        if (cachedLeaderboard) {
+            return cachedLeaderboard;
+        }
+        const completeCounts = await this.getCompleteCounts(jobName);
+        const claimCounts = await this.getClaimCounts(jobName);
+        const doneScores = await this.getDoneItemScores(jobName);
+        this.cacheManager.set(`leaderboard:${jobName}`, { completeCounts, claimCounts, doneScores }, 200);
+        return { completeCounts, claimCounts, doneScores };
+    }
+
+
+    async addItemsToJob(jobName: string, items: string[]) {
+        const splitSize = 500
+        for (const itemSubset of (items.length > splitSize ? splitEvery(splitSize, items) : [items])) {
+            await this.redis.rpush(this.todoListNameBuilder(jobName), ...itemSubset);
+        }
+    }
+
+    async claimJobItem(jobName: string, claimer: string): Promise<string | null> {
+        const jobItem = await this.redis.brpoplpush(this.todoListNameBuilder(jobName), this.claimedListNameBuilder(jobName), 10);
+        if (!jobItem) {
+            return null;
+        }
+        await this.redis.incr(this.claimerCountNameBuilder(jobName, claimer));
+        return jobItem;
+    }
+
+    async completeJobItem(jobName: string, jobItem: string, completer: string, data: any) {
+        const claimRemoveResult = await this.redis.lrem(this.claimedListNameBuilder(jobName), 1, jobItem);
+        if (claimRemoveResult === 0) {
+            return false;
+        }
+        await this.redis.rpush(this.doneListNameBuilder(jobName), JSON.stringify({ item: jobItem, client: completer, data }));
+        await this.redis.decr(this.claimerCountNameBuilder(jobName, completer));
+        await this.redis.incr(this.completeCountNameBuilder(jobName, completer));
+        return true
+    }
+
+    async getTodoItems(jobName: string) {
+        return this.redis.lrange(this.todoListNameBuilder(jobName), 0, -1);
+    }
+
+    async getTodoItemCount(jobName: string) {
+        return this.redis.llen(this.todoListNameBuilder(jobName));
+    }
+
+    async getClaimedItems(jobName: string) {
+        return this.redis.lrange(this.claimedListNameBuilder(jobName), 0, -1);
+    }
+
+    async getClaimedItemCount(jobName: string) {
+        return this.redis.llen(this.claimedListNameBuilder(jobName));
+    }
+
+    async getDoneItems(jobName: string) {
+        return this.redis.lrange(this.doneListNameBuilder(jobName), 0, -1);
+    }
+
+    async getDoneItemCount(jobName: string) {
+        return this.redis.llen(this.doneListNameBuilder(jobName));
+    }
+
+    async getDoneItemScoresTotal(jobName: string) {
+        const doneItems = await this.getDoneItems(jobName);
+        const scores = doneItems.map(item => JSON.parse(item).data?.score || 1);
+        return scores.reduce((acc: number, val: number) => acc + val, 0);
+    }
+
+    // Sum the "score" property of each item in the done list and
+    // accumulate based on claimer
+    async getDoneItemScores(jobName: string) {
+        const doneItems = await this.getDoneItems(jobName);
+        const scores = doneItems.map(item => JSON.parse(item).data?.score || 1);
+        const claimers = doneItems.map(item => JSON.parse(item).client);
+        const scoreMap = claimers.reduce((acc: any, val: any, idx: number) => {
+            acc[val] = (acc[val] || 0) + scores[idx];
+            return acc;
+        }, {});
+        return scoreMap;
+    }
+
+    async getJobs() {
+        return this.redis.keys('job:*');
+    }
+
+    async isJobRegistered(jobName: string) {
+        return this.redis.exists(this.jobNameBuilder(jobName));
+    }
+
+    async registerJob(jobName: string, metadata: JobMetadata) {
+        if (await this.isJobRegistered(jobName)) {
+            return false;
+        }
+        await this.redis.set(this.jobNameBuilder(jobName), JSON.stringify(metadata));
+        return true
+    }
+
+    async getJobMetadata(jobName: string): Promise<JobMetadata | null> {
+        const result = await this.redis.get(this.jobNameBuilder(jobName))
+        if (!result) {
+            return null;
+        }
+        return JSON.parse(result)
+    }
+
+    async getPublicJobMetadata(jobName: string): Promise<PublicJobMetadata | null> {
+        const metadata = await this.getJobMetadata(jobName);
+        if (!metadata) {
+            return null;
+        }
+        return JobsService.cleanJobMetadata(metadata);
+    }
+
+    async clearClaimerCounts(jobName: string) {
+        const keys = await this.redis.keys(this.claimerCountWildcardBuilder(jobName));
+        await Promise.all(keys.map(key => this.redis.del(key)));
+    }
+
+    async resetClaimedItems(jobName: string) {
+        const claimedItems = await this.getClaimedItems(jobName);
+        for (const claimedItem of claimedItems) {
+            await this.redis.rpoplpush(this.claimedListNameBuilder(jobName), this.todoListNameBuilder(jobName));
+        }
+        await this.clearClaimerCounts(jobName);
+    }
+
+    async clearTodoItems(jobName: string, claimKey: string) {
+        const metadata = await this.getJobMetadata(jobName);
+        if (metadata === null || metadata.claimSecret !== claimKey) {
+            return false;
+        }
+
+        await this.redis.del(this.todoListNameBuilder(jobName));
+    }
+
+}

src/junk-drawer/junk-drawer.controller.ts

@@ -0,0 +1,123 @@
+import {
+  Body,
+  Controller,
+  Get,
+  Param,
+  Post,
+  Redirect,
+  Render,
+  Req,
+  Res,
+  UploadedFiles,
+  UseInterceptors,
+} from '@nestjs/common';
+import { JunkDrawerService } from './junk-drawer.service';
+import { ApiConsumes, ApiTags } from '@nestjs/swagger';
+import { FilesInterceptor } from '@nestjs/platform-express';
+import { generateUniqueSlug } from 'src/utils/slug';
+import { JunkDrawerMetadata } from './types';
+import { Request, Response } from 'express';
+import { ipFromRequest } from 'src/utils/ip';
+
+@Controller(['junk-drawer', 'attic'])
+@ApiTags('attic')
+export class JunkDrawerController {
+  constructor(private readonly junkDrawerService: JunkDrawerService) {}
+
+  @Get('')
+  @Render('junk-drawer/upload')
+  async generateUploadForm(@Req() req: Request) {
+    let items: string[] = [];
+    const ip = ipFromRequest(req);
+    if (ip) {
+      items = await this.junkDrawerService.getItemsForIp(ip);
+    }
+    return { items, ip };
+  }
+
+  @Get(':slug.json')
+  async junkDrawerJson(
+    @Param('slug') slug: string,
+  ): Promise<JunkDrawerMetadata | undefined> {
+    return this.junkDrawerService.getJunkDrawerMetadata(slug);
+  }
+
+  @Get(':slug')
+  @Render('junk-drawer/view')
+  async viewJunkDrawer(@Param('slug') slug: string): Promise<any> {
+    const metadata = await this.junkDrawerService.getJunkDrawerMetadata(slug);
+    if (!metadata) {
+      return { error: 'File not found or expired' };
+    }
+    metadata.items = metadata.items.map((item) => ({
+      ...item,
+      isImage: item.mimetype.startsWith('image/'),
+    }));
+    return { ...metadata };
+  }
+
+  @Get(':slug/:filename')
+  async downloadJunkDrawerItem(
+    @Param('slug') slug: string,
+    @Param('filename') filename: string,
+    @Res() res: Response,
+  ) {
+    const metadata = await this.junkDrawerService.getJunkDrawerMetadata(slug);
+    if (!metadata) {
+      return res.status(404).send('File not found or expired');
+    }
+    const item = metadata.items.find((item) => item.filename === filename);
+    if (!item) {
+      return res.status(404).send('File not found or expired');
+    }
+    const buffer = await this.junkDrawerService.getJunkDrawerItem(
+      slug,
+      filename,
+    );
+    return res
+      .header('Content-Type', item.mimetype)
+      .header('Content-Disposition', `filename="${item.filename}"`)
+      .send(buffer);
+  }
+
+  @Post('upload')
+  @Redirect('', 302)
+  @ApiConsumes('multipart/form-data')
+  @UseInterceptors(FilesInterceptor('files'))
+  async handleFileUpload(
+    @UploadedFiles() files: Express.Multer.File[],
+    @Req() request: Request,
+    @Body('description') description: string,
+    @Body('private-ish') privateIsh: boolean,
+    @Body('remember') remember: boolean,
+  ): Promise<any> {
+    const uniqueSlug = generateUniqueSlug({
+      random: privateIsh,
+    });
+    const ip = ipFromRequest(request);
+    const metadata: JunkDrawerMetadata = {
+      slug: uniqueSlug,
+      version: '1.0.0',
+      lastModified: new Date(),
+      description,
+      items: files.map((file) => ({
+        filename: file.originalname,
+        size: file.size,
+        lastModified: new Date(),
+        mimetype: file.mimetype,
+      })),
+    };
+    for (const file of files) {
+      await this.junkDrawerService.storeJunkDrawerItem(
+        uniqueSlug,
+        file.originalname,
+        file.buffer,
+      );
+    }
+    await this.junkDrawerService.storeJunkDrawerMetadata(metadata);
+    if (remember && ip && !privateIsh) {
+      await this.junkDrawerService.recordItemForIp(ip, uniqueSlug);
+    }
+    return { url: `/attic/${uniqueSlug}` };
+  }
+}

src/junk-drawer/junk-drawer.module.ts

@@ -0,0 +1,10 @@
+import { Module } from '@nestjs/common';
+import { JunkDrawerService } from './junk-drawer.service';
+import { JunkDrawerController } from './junk-drawer.controller';
+import { MinioService } from 'src/minio/minio.service';
+
+@Module({
+  providers: [JunkDrawerService, MinioService],
+  controllers: [JunkDrawerController],
+})
+export class JunkDrawerModule {}

src/junk-drawer/junk-drawer.service.ts

@@ -0,0 +1,135 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import { MinioService } from 'src/minio/minio.service';
+import { JunkDrawerMetadata } from './types';
+import { generateUniqueSlug } from 'src/utils/slug';
+
+@Injectable()
+export class JunkDrawerService {
+  private readonly logger: Logger = new Logger(JunkDrawerService.name);
+  constructor(
+    private readonly minioService: MinioService,
+    private readonly configService: ConfigService,
+  ) {}
+
+  private pathForFile(filename: string): string {
+    return [this.configService.get<string>('junkDrawer.rootPath'), filename]
+      .join('/')
+      .replace(/\.\.\//g, '');
+  }
+
+  private junkDrawerBucketName(): string {
+    return this.configService.get<string>(
+      'junkDrawer.bucketName',
+      'junk-drawer',
+    );
+  }
+
+  public generateSlug(fileName: string): string {
+    return (
+      generateUniqueSlug() +
+      fileName
+        .replace(/[^a-z0-9]/gi, '-')
+        .replace(/-+/g, '-')
+        .replace(/^-|-$/g, '')
+    );
+  }
+
+  public async getJunkDrawerMetadata(
+    slug: string,
+  ): Promise<JunkDrawerMetadata | undefined> {
+    const metadata = await this.minioService.getBuffer(
+      this.junkDrawerBucketName(),
+      this.pathForFile(`${slug}/metadata.json`),
+    );
+    if (metadata) {
+      return JSON.parse(metadata.toString());
+    }
+  }
+
+  public async storeJunkDrawerMetadata(
+    metadata: JunkDrawerMetadata,
+  ): Promise<JunkDrawerMetadata | undefined> {
+    const uploadResult = await this.minioService.uploadBuffer(
+      this.junkDrawerBucketName(),
+      this.pathForFile(`${metadata.slug}/metadata.json`),
+      Buffer.from(JSON.stringify(metadata)),
+    );
+    if (uploadResult) {
+      return metadata;
+    }
+  }
+
+  public async storeJunkDrawerItem(
+    slug: string,
+    filename: string,
+    buffer: Buffer,
+  ): Promise<void> {
+    const uploadResult = this.minioService.uploadBuffer(
+      this.junkDrawerBucketName(),
+      this.pathForFile(`${slug}/${filename}`),
+      buffer,
+    );
+  }
+
+  public async storeJunkDrawerCollection(
+    metadata: JunkDrawerMetadata,
+    files: { filename: string; buffer: Buffer }[],
+  ): Promise<void> {
+    await this.storeJunkDrawerMetadata(metadata);
+    await Promise.all(
+      files.map((file) =>
+        this.storeJunkDrawerItem(metadata.slug, file.filename, file.buffer),
+      ),
+    );
+  }
+
+  public async getJunkDrawerItem(
+    slug: string,
+    filename: string,
+  ): Promise<Buffer | undefined> {
+    return this.minioService.getBuffer(
+      this.junkDrawerBucketName(),
+      this.pathForFile(`${slug}/${filename}`),
+    );
+  }
+
+  public async recordItemForIp(ip: string, itemId: string) {
+    let ipList = Buffer.from(JSON.stringify({}));
+    try {
+      ipList = await this.minioService.getBuffer(
+        this.junkDrawerBucketName(),
+        this.pathForFile(`ip-list.json`),
+      );
+    } catch (error) {
+      this.logger.warn('IP list is empty');
+    }
+    let ipListObject = ipList ? JSON.parse(ipList.toString()) : {};
+    if (!ipListObject[ip]) {
+      ipListObject[ip] = {
+        lastUpload: new Date(),
+        itemIds: [],
+      };
+    }
+    ipListObject[ip].itemIds.push(itemId);
+    await this.minioService.uploadBuffer(
+      this.junkDrawerBucketName(),
+      this.pathForFile(`ip-list.json`),
+      Buffer.from(JSON.stringify(ipListObject)),
+    );
+  }
+
+  public async getItemsForIp(ip: string): Promise<string[]> {
+    let ipList = Buffer.from(JSON.stringify({}));
+    try {
+      ipList = await this.minioService.getBuffer(
+        this.junkDrawerBucketName(),
+        this.pathForFile(`ip-list.json`),
+      );
+    } catch (error) {
+      this.logger.warn('IP list is empty');
+    }
+    let ipListObject = ipList ? JSON.parse(ipList.toString()) : {};
+    return ipListObject[ip]?.itemIds || [];
+  }
+}

src/junk-drawer/types.ts

@@ -0,0 +1,20 @@
+export interface JunkDrawerMetadata {
+  slug: string;
+  version: string;
+  lastModified: Date;
+  description: string;
+  items: JunkDrawerItem[];
+}
+
+export interface JunkDrawerItem {
+  filename: string;
+  size: number;
+  lastModified: Date;
+  mimetype: string;
+}
+
+export interface JunkDrawerIpList {
+  ip: string;
+  lastUpload: Date;
+  itemIds: string[];
+}

src/kv/kv.controller.ts

@@ -12,16 +12,17 @@ import {
 } from '@nestjs/common';
 import { KvService } from './kv.service';
 import { Request } from 'express';
-import { ApiTags } from '@nestjs/swagger';
-import exp from 'constants';
+import { ApiQuery, ApiTags } from '@nestjs/swagger';
 import { FileInterceptor } from '@nestjs/platform-express';
 
 @Controller('kv')
 @ApiTags('kv')
 export class KvController {
-  constructor(private readonly kvService: KvService) {}
+  constructor(private readonly kvService: KvService) { }
 
   @Get(':namespace/:key/metadata')
+  @ApiQuery({ name: 'namespace', required: true })
+  @ApiQuery({ name: 'key', required: true })
   async getMetadata(
     @Param('namespace') namespace: string,
     @Param('key') key: string,

src/kv/kv.service.ts

@@ -1,7 +1,7 @@
 import { InjectQueue } from '@nestjs/bull';
 import { Injectable, Logger } from '@nestjs/common';
 import { Queue } from 'bull';
-import { UploadedObjectInfo } from 'minio';
+import { UploadedObjectInfo } from 'minio/dist/main/internal/type';
 import { MinioService } from 'src/minio/minio.service';
 
 @Injectable()

src/minio/minio.module.ts

@@ -1,7 +1,8 @@
 import { Module } from '@nestjs/common';
 import { MinioService } from './minio.service';
+import { CacheModule } from '@nestjs/cache-manager';
 
 @Module({
-  providers: [MinioService]
+  providers: [MinioService],
 })
 export class MinioModule {}

src/minio/minio.service.ts

@@ -2,7 +2,9 @@ import { CACHE_MANAGER } from '@nestjs/cache-manager';
 import { Cache } from 'cache-manager';
 import { Inject, Injectable } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
-import { Client, ItemBucketMetadata, UploadedObjectInfo } from 'minio';
+import { Client, ItemBucketMetadata } from 'minio';
+import { UploadedObjectInfo } from 'minio/dist/main/internal/type';
+import { open, readFileSync } from 'fs';
 
 @Injectable()
 export class MinioService {
@@ -19,6 +21,7 @@ export class MinioService {
       useSSL: this.configService.get<string>('S3_USE_SSL', 'true') === 'true',
       accessKey: this.configService.get<string>('S3_ACCESS_KEY', ''),
       secretKey: this.configService.get<string>('S3_SECRET_KEY', ''),
+      region: this.configService.get<string>('S3_REGION', 'auto'),
     });
     this.defaultBucketName = this.configService.get<string>('S3_BUCKET', '');
   }
@@ -29,12 +32,8 @@ export class MinioService {
     filePath: string,
     metadata?: ItemBucketMetadata,
   ): Promise<UploadedObjectInfo> {
-    return await this.client.fPutObject(
-      bucketName,
-      objectName,
-      filePath,
-      metadata,
-    );
+    const file = readFileSync(filePath);
+    return this.uploadBuffer(bucketName, objectName, file, metadata);
   }
 
   public async uploadBuffer(
@@ -43,10 +42,11 @@ export class MinioService {
     buffer: Buffer,
     metadata?: ItemBucketMetadata,
   ): Promise<UploadedObjectInfo> {
-    return await this.client.putObject(
+    return this.client.putObject(
       bucketName,
       objectName,
       buffer,
+      buffer.length,
       metadata,
     );
   }
@@ -89,8 +89,13 @@ export class MinioService {
   public async listBucketObjects(
     bucketName: string,
     prefix?: string,
+    recursive: boolean = false,
   ): Promise<string[]> {
-    const objectStream = await this.client.listObjects(bucketName, prefix);
+    const objectStream = await this.client.listObjectsV2(
+      bucketName,
+      prefix,
+      recursive,
+    );
     const objects = await new Promise<ItemBucketMetadata[]>(
       (resolve, reject) => {
         const objects: ItemBucketMetadata[] = [];
@@ -109,6 +114,10 @@ export class MinioService {
     await this.client.removeObject(bucketName, objectName);
   }
 
+  public async makeBucket(bucketName: string): Promise<void> {
+    return this.client.makeBucket(bucketName);
+  }
+
   /**
    *
    * @param bucketName
@@ -123,4 +132,16 @@ export class MinioService {
   ): Promise<string> {
     return await this.client.presignedGetObject(bucketName, objectName, expiry);
   }
+
+  public async objectExists(
+    bucketName: string,
+    objectName: string,
+  ): Promise<boolean> {
+    try {
+      await this.client.statObject(bucketName, objectName);
+      return true;
+    } catch (e) {
+      return false;
+    }
+  }
 }

src/names/names.controller.ts

@@ -0,0 +1,26 @@
+import { Controller, Get, Param } from '@nestjs/common';
+import { NamesService } from './names.service';
+
+@Controller('names')
+export class NamesController {
+  constructor(private readonly namesService: NamesService) {}
+
+  @Get()
+  async getNameList() {
+    return await this.namesService.getNameList();
+  }
+
+  @Get(':name')
+  async getNameInformation(@Param('name') name: string) {
+    return {
+      popularity: await this.namesService.getSsaNameData(name),
+      otherSites: {
+        behindTheName: `https://www.behindthename.com/name/${name.toLocaleLowerCase()}`,
+        babynames: `https://www.babynames.com/name/${name.toLocaleLowerCase()}`,
+      },
+      behindTheName: {
+        synonyms: await this.namesService.getBtnSynonyms(name),
+      },
+    };
+  }
+}

src/names/names.module.ts

@@ -0,0 +1,10 @@
+import { Module } from '@nestjs/common';
+import { NamesService } from './names.service';
+import { NamesController } from './names.controller';
+import { MinioService } from 'src/minio/minio.service';
+
+@Module({
+  providers: [MinioService, NamesService],
+  controllers: [NamesController],
+})
+export class NamesModule {}

src/names/names.service.ts

@@ -0,0 +1,41 @@
+import { Injectable } from '@nestjs/common';
+import { MinioService } from 'src/minio/minio.service';
+
+@Injectable()
+export class NamesService {
+  constructor(private readonly minioService: MinioService) {}
+
+  async getNameList(): Promise<string[]> {
+    return JSON.parse(
+      (
+        await this.minioService.getCachedBuffer(
+          'cdn-source',
+          'baby-name-data/list.json',
+        )
+      ).toString(),
+    );
+  }
+
+  async getSsaNameData(name: string): Promise<any> {
+    return JSON.parse(
+      (
+        await this.minioService.getCachedBuffer(
+          'cdn-source',
+          `baby-name-data/individual/${name}.json`,
+        )
+      ).toString(),
+    );
+  }
+
+  async getBtnSynonyms(name: string): Promise<string[]> {
+    const synonymData = JSON.parse(
+      (
+        await this.minioService.getCachedBuffer(
+          'cdn-source',
+          `baby-name-data/btn_synonyms.json`,
+        )
+      ).toString(),
+    );
+    return synonymData[name]?.synonyms || [];
+  }
+}

src/ogscraper/ogscraper.controller.ts

@@ -1,34 +0,0 @@
-import { Body, Controller, Post } from '@nestjs/common';
-import { ApiProperty, ApiTags } from '@nestjs/swagger';
-const ogs = require('open-graph-scraper');
-import { SuccessResult } from 'open-graph-scraper';
-import { OgScraperService } from './ogscraper.service';
-import { InjectMetric } from '@willsoto/nestjs-prometheus';
-import { Histogram } from 'prom-client';
-
-class ScrapeOgDto {
-  @ApiProperty({
-    description: 'URL of the page to fetch Open Graph metadata of',
-    example:
-      'https://qz.com/1903322/why-pivot-tables-are-the-spreadsheets-most-powerful-tool',
-  })
-  url: string;
-}
-
-@Controller('ogscraper')
-@ApiTags('open-graph-scraper')
-export class OgScraperController {
-  constructor(
-    private readonly ogScraperService: OgScraperService,
-    @InjectMetric('generation_time')
-    public generationTime: Histogram<string>,
-  ) {}
-
-  @Post('')
-  async scrapeOg(@Body() body: ScrapeOgDto): Promise<SuccessResult> {
-    const end = this.generationTime.startTimer();
-    const response = await this.ogScraperService.getOg(body.url);
-    end();
-    return response;
-  }
-}

src/ogscraper/ogscraper.module.ts

@@ -1,28 +0,0 @@
-import { Module } from '@nestjs/common';
-import { OgScraperController } from './ogscraper.controller';
-import { OgScraperService } from './ogscraper.service';
-import {
-  PrometheusModule,
-  makeHistogramProvider,
-} from '@willsoto/nestjs-prometheus';
-
-@Module({
-  imports: [
-    PrometheusModule.register({
-      customMetricPrefix: 'ogscraper',
-      defaultMetrics: {
-        enabled: false,
-      },
-    }),
-  ],
-  controllers: [OgScraperController],
-  providers: [
-    OgScraperService,
-    makeHistogramProvider({
-      name: 'generation_time',
-      help: 'Open Graph Scraping response times',
-    }),
-  ],
-  exports: [OgScraperService],
-})
-export class OgScraperModule {}

src/ogscraper/ogscraper.service.ts

@@ -1,20 +0,0 @@
-import { Injectable } from '@nestjs/common';
-import { ConfigService } from '@nestjs/config';
-const ogs = require('open-graph-scraper');
-import { SuccessResult } from 'open-graph-scraper';
-
-@Injectable()
-export class OgScraperService {
-  constructor(public readonly configService: ConfigService) {}
-
-  async getOg(url: string): Promise<SuccessResult> {
-    return ogs({
-      url,
-      fetchOptions: {
-        headers: {
-          'user-agent': this.configService.get<string>('userAgent') || '',
-        },
-      },
-    });
-  }
-}

src/parkio/parkio.module.ts

@@ -2,9 +2,10 @@ import { Module } from '@nestjs/common';
 import { ParkioService } from './parkio.service';
 import { ParkioController } from './parkio.controller';
 import { IswordService } from 'src/isword/isword.service';
+import { CacheModule } from '@nestjs/cache-manager';
 
 @Module({
   providers: [ParkioService, IswordService],
-  controllers: [ParkioController]
+  controllers: [ParkioController],
 })
 export class ParkioModule {}

src/pow/pow.controller.ts

@@ -0,0 +1,66 @@
+import {
+  BadRequestException,
+  Body,
+  Controller,
+  Get,
+  Post,
+  Query,
+  Render,
+} from '@nestjs/common';
+import { PowService } from './pow.service';
+import { ApiBody, ApiProperty, ApiQuery, ApiTags } from '@nestjs/swagger';
+import { ConfigService } from '@nestjs/config';
+
+class SolveDto {
+  @ApiProperty({
+    description: 'The challenge to solve',
+  })
+  challenge: string;
+}
+
+@ApiTags('pow')
+@Controller('pow')
+export class PowController {
+  constructor(
+    private readonly powService: PowService,
+    private readonly configService: ConfigService,
+  ) {}
+
+  @Get('')
+  @Render('pow/index')
+  async index() {
+    return {
+      difficulty: this.powService.getDifficulty(),
+    };
+  }
+
+  @Get('challenge')
+  async generateChallenge() {
+    return this.powService.generateChallenge();
+  }
+
+  @Post('challenge')
+  @ApiBody({
+    schema: {
+      properties: { challenge: { type: 'string' }, proof: { type: 'string' } },
+    },
+  })
+  async verifyChallenge(@Body() body: { challenge: string; proof: string }) {
+    return this.powService.verifyChallenge(body.challenge, body.proof);
+  }
+
+  @Post('challenge/complete')
+  @ApiBody({ schema: { properties: { challenge: { type: 'string' } } } })
+  async markChallengeAsComplete(@Body() body: { challenge: string }) {
+    return this.powService.markChallengeAsComplete(body.challenge);
+  }
+
+  @Post('challenge/solve')
+  async solveChallenge(@Body() body: SolveDto) {
+    if (this.configService.get<boolean>('isProduction')) {
+      throw new BadRequestException('This endpoint is disabled in production');
+    }
+
+    return this.powService.performChallenge(body.challenge);
+  }
+}

src/pow/pow.module.ts

@@ -0,0 +1,34 @@
+import { Module } from '@nestjs/common';
+import { PowService } from './pow.service';
+import { PowController } from './pow.controller';
+import { makeGaugeProvider } from '@willsoto/nestjs-prometheus';
+
+@Module({
+  imports: [],
+  providers: [
+    PowService,
+    makeGaugeProvider({
+      name: 'pow_challenges_generated',
+      help: 'The total number of POW challenges generated',
+    }),
+    makeGaugeProvider({
+      name: 'pow_challenges_completed',
+      help: 'The total number of POW challenges completed',
+    }),
+    makeGaugeProvider({
+      name: 'pow_successful_verifies',
+      help: 'The total number of successful POW challenge verifications',
+    }),
+    makeGaugeProvider({
+      name: 'pow_failed_verifies',
+      help: 'The total number of failed POW challenge verifications',
+    }),
+    makeGaugeProvider({
+      name: 'pow_difficulty',
+      help: 'The current POW difficulty',
+    }),
+  ],
+  controllers: [PowController],
+  exports: [PowService],
+})
+export class PowModule {}

src/pow/pow.service.ts

@@ -0,0 +1,102 @@
+import { CACHE_MANAGER } from '@nestjs/cache-manager';
+import { Inject, Injectable, Logger } from '@nestjs/common';
+import { InjectMetric } from '@willsoto/nestjs-prometheus';
+import { Cache } from 'cache-manager';
+import { randomBytes, createHash } from 'crypto';
+
+@Injectable()
+export class PowService {
+  private readonly logger = new Logger(PowService.name);
+  private difficulty = 5;
+
+  constructor(
+    @Inject(CACHE_MANAGER) private cacheManager: Cache,
+    @InjectMetric('pow_challenges_generated') private challengesGenerated: any,
+    @InjectMetric('pow_challenges_completed') private challengesCompleted: any,
+    @InjectMetric('pow_successful_verifies') private successfulVerifies: any,
+    @InjectMetric('pow_failed_verifies') private failedVerifies: any,
+    @InjectMetric('pow_difficulty') private powDifficulty: any,
+  ) {
+    this.powDifficulty.set(this.difficulty);
+  }
+
+  /**
+   * Generate a proof of work challenge, stored to redis for verification within
+   * the next 60 seconds.
+   */
+  async generateChallenge() {
+    const challenge = this.generateRandom256BitString();
+    this.logger.verbose(`Generated challenge: ${challenge}`);
+    this.challengesGenerated.inc();
+    await this.cacheManager.set(challenge, true, 60 * 1000);
+    return challenge;
+  }
+
+  generateRandom256BitString() {
+    const randomString = randomBytes(32).toString('hex');
+    return randomString;
+  }
+
+  hashAndCheck(string: string) {
+    return this.hashPassesDifficulty(this.hashString(string), this.difficulty);
+  }
+
+  hashPassesDifficulty(hash: string, difficulty: number) {
+    return hash.startsWith('0'.repeat(difficulty));
+  }
+
+  /**
+   * Verify that the proof of work submitted has a leading number of
+   * zeroes equal to the challenge length and the challenge exists.
+   */
+  async verifyChallenge(challenge: string, proof: string): Promise<boolean> {
+    const expected = await this.cacheManager.get<boolean>(challenge);
+    const success = expected ? this.hashAndCheck(proof + challenge) : false;
+    if (success) {
+      this.successfulVerifies.inc();
+    } else {
+      this.failedVerifies.inc();
+    }
+    return success;
+  }
+
+  async markChallengeAsComplete(challenge: string) {
+    this.challengesCompleted.inc();
+    await this.cacheManager.del(challenge);
+  }
+
+  /**
+   * Perform a proof of work challenge to find a proof that hashes to a value
+   */
+  async performChallenge(challenge: string) {
+    let proof = this.generateRandom256BitString();
+    let hash = this.hashString(proof + challenge);
+    while (!this.hashPassesDifficulty(hash, this.difficulty)) {
+      proof = this.generateRandom256BitString();
+      hash = this.hashString(proof + challenge);
+    }
+    return { proof, hash };
+  }
+
+  /**
+   * sha512 the provided string and return the result.
+   */
+  hashString(input: string) {
+    return createHash('sha512').update(input).digest('hex');
+  }
+
+  /**
+   * Set the difficulty of the proof of work challenge.
+   */
+  setDifficulty(difficulty: number) {
+    this.difficulty = difficulty;
+    this.powDifficulty.set(this.difficulty);
+  }
+
+  /**
+   * Get the current difficulty of the proof of work challenge.
+   */
+  getDifficulty() {
+    return this.difficulty;
+  }
+}

src/redirects/redirects.module.ts

@@ -1,7 +1,8 @@
 import { Module } from '@nestjs/common';
 import { RedirectsController } from './redirects.controller';
+import { RequiredReadingController } from './required-reading.controller';
 
 @Module({
-  controllers: [RedirectsController]
+  controllers: [RedirectsController, RequiredReadingController]
 })
-export class RedirectsModule {}
+export class RedirectsModule { }

src/users/users.json

@@ -0,0 +1 @@
+[]

src/users/users.service.ts

@@ -2,7 +2,6 @@ import { Injectable, Logger } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
 import { Cron } from '@nestjs/schedule';
 import { MinioService } from 'src/minio/minio.service';
-require('dotenv').config();
 
 // This should be a real class/interface representing a user entity
 export type User = {
@@ -11,6 +10,15 @@ export type User = {
   password: string;
 };
 
+const developmentUsers: User[] = [
+  {
+    userId: 1,
+    username: 'admin',
+    // "password"
+    password: '$2b$10$c3d3JacaYw3KZ9qy4HniMeum5MXSj1VOOz8EWL5K23ZTL5aPnMNhS',
+  }
+]
+
 @Injectable()
 export class UsersService {
   private users: User[] = [];
@@ -28,6 +36,11 @@ export class UsersService {
   @Cron('* * * * *')
   async refreshUsers(): Promise<void> {
     this.logger.verbose('Refreshing users');
+    if (this.configService.get<boolean>('isProduction', false) === false) {
+      this.logger.verbose('Development environment, using development users');
+      this.users = developmentUsers;
+      return;
+    }
     const buffer = await this.minioService.getBuffer(this.bucket, 'users.json');
     this.users = JSON.parse(buffer.toString());
     if (this.users === undefined) {

src/utils/ip.ts

@@ -0,0 +1,9 @@
+import { Request } from 'express';
+
+export const ipFromRequest = (req: Request): string | undefined => {
+  const flyIp = req.headers['fly-client-ip'] as string | undefined;
+  const forwardedFor = (req.headers['x-forwarded-for'] as string)?.split(
+    ',',
+  )[0];
+  return flyIp || forwardedFor || req.ip;
+};

src/utils/slug.ts

@@ -0,0 +1,45 @@
+import { v4 } from 'uuid';
+
+interface SlugOptions {
+  year: boolean;
+  month: boolean;
+  day: boolean;
+  hour: boolean;
+  minute: boolean;
+  second: boolean;
+  random: boolean;
+  separator: string;
+}
+
+const defaultSlugOptions: SlugOptions = {
+  year: true,
+  month: true,
+  day: true,
+  hour: true,
+  minute: true,
+  second: false,
+  random: false,
+  separator: '',
+};
+
+export const generateUniqueSlug = (
+  options: Partial<SlugOptions> = {},
+): string => {
+  const { year, month, day, hour, minute, second, random, separator } = {
+    ...defaultSlugOptions,
+    ...options,
+  };
+
+  const date = new Date();
+  const parts = [
+    year ? date.getFullYear() : '',
+    month ? String(date.getMonth() + 1).padStart(2, '0') : '',
+    day ? String(date.getDate()).padStart(2, '0') : '',
+    hour ? String(date.getHours()).padStart(2, '0') : '',
+    minute ? String(date.getMinutes()).padStart(2, '0') : '',
+    second ? String(date.getSeconds()).padStart(2, '0') : '',
+    random ? '-' + v4() : '',
+  ].filter(Boolean);
+
+  return parts.join(separator);
+};

views/file/upload-result.hbs

@@ -0,0 +1,8 @@
+<div>
+  <h1>Upload Result</h1>
+  <p>File uploaded successfully.</p>
+  <p>File name: {{originalFilename}}</p>
+  <p>Right-click copy: <a href='/file/{{key}}'>{{key}}</a></p>
+  <p>Expires at {{expireTime}}</p>
+  <p><a href='/file'>Upload another file</a></p>
+</div>

views/file/upload.hbs

@@ -0,0 +1,9 @@
+<form method='post' action='/file/upload' enctype='multipart/form-data'>
+  <div>
+    <label for='file'>Choose file to upload</label>
+    <input type='file' id='file' name='file' multiple />
+  </div>
+  <div>
+    <button>Submit</button>
+  </div>
+</form>

views/jobs/add.hbs

@@ -0,0 +1,14 @@
+<form
+  method='post'
+  action='/jobs/{{jobName}}/addFile'
+  enctype='multipart/form-data'
+>
+  <p>File will be split on newline characters</p>
+  <div>
+    <label for='file'>Choose file to upload</label>
+    <input type='file' id='file' name='file' />
+  </div>
+  <div>
+    <button>Submit</button>
+  </div>
+</form>

views/jobs/leaderboard.hbs

@@ -0,0 +1,23 @@
+<div>
+  <div>Job: {{jobName}}</div>
+  <div style='display: flex; flex-direction: row;'>
+    <div>
+      <h2>Complete</h2>
+      <ol>
+        {{#each completes as |item|}}
+          <li>{{name}}: {{count}}</li>
+        {{/each}}
+      </ol>
+    </div>
+    <div>
+      <h2>Claims</h2>
+      <ol>
+        {{#each claims as |item|}}
+          <li>{{name}}: {{count}}</li>
+        {{/each}}
+      </ol>
+    </div>
+  </div>
+</div>
+<i>This page will auto-refresh every 10 seconds.</i>
+<meta http-equiv='refresh' content='10' />

views/jobs/stats.hbs

@@ -0,0 +1,15 @@
+<div>
+  <div>Job: {{jobName}}</div>
+  <div style='display: flex; flex-direction: row;'>
+    <div>ToDo: </div>
+    <div>{{todoCount}}</div>
+  </div>
+  <div style='display: flex; flex-direction: row;'>
+    <div>Claimed: </div>
+    <div>{{claimedCount}}</div>
+  </div>
+  <div style='display: flex; flex-direction: row;'>
+    <div>Done: </div>
+    <div>{{doneCount}}</div>
+  </div>
+</div>

views/junk-drawer/upload.hbs

@@ -0,0 +1,59 @@
+<html>
+
+  <head>
+    <meta charset='utf-8' />
+    <title>Chip's Attic</title>
+    <script src='https://cdn.tailwindcss.com'></script>
+    <meta name='viewport' content='width=device-width, initial-scale=1' />
+
+  </head>
+  <body>
+
+    <form
+      method='post'
+      action='/attic/upload'
+      enctype='multipart/form-data'
+    >
+      <div>
+        <label for='files'>Choose files to upload</label>
+        <input type='file' id='files' name='files' multiple />
+      </div>
+      <div>
+        <label
+          class='cursor-help'
+          title='Append a random-ish string to the ID to make it harder to find via enumeration'
+          for='private-ish'
+        >Private-ish</label>
+        <input type='checkbox' id='private-ish' name='private-ish' />
+      </div>
+      <div>
+        <label for='remember'>Remember for my IP</label>
+        <input type='checkbox' id='remember' name='remember' />
+      </div>
+      <div class='flex flex-col max-w-96'>
+        <label for='description'>Description:</label>
+        <textarea
+          type='text'
+          id='description'
+          name='description'
+          rows='4'
+          autofocus
+          class='border-2 rounded-md border-slate-800 border-solid p-1'
+        ></textarea>
+      </div>
+      <div>
+        <button class='border-2 rounded-md border-slate-800 p-1'>Submit</button>
+      </div>
+    </form>
+    <div>
+      <h2>Your Items ({{ip}})</h2>
+      <div>
+        {{#each items}}
+          <div>
+            <a class="text-blue-500 underline" href='/junk-drawer/{{.}}'>{{.}}</a>
+          </div>
+        {{/each}}
+      </div>
+    </div>
+  </body>
+</html>

views/junk-drawer/view.hbs

@@ -0,0 +1,39 @@
+<html>
+
+</html>
+
+<head>
+  <meta charset='utf-8' />
+  <title>Chip's Attic</title>
+  <script src='https://cdn.tailwindcss.com'></script>
+</head>
+
+<body>
+  <div>
+    <h1 class="text-xl">Attic Item</h1>
+    <p>The attic item was uploaded {{lastModified}}</p>
+    <h2>Description</h2>
+    <p class="bg-slate-100 px-2 py-4">{{description}}</p>
+    <p>Right-click copy: <a class="text-blue-500 underline" href='/attic/{{slug}}'>{{slug}}</a></p>
+    {{#if items}}
+    <p class="text-lg">Files:</p>
+    <div class="list-disc list-inside ml-1">
+      {{#each items}}
+        <div>
+          <a target="_blank" class="text-emerald-600" href='/attic/{{../slug}}/{{filename}}'>
+            <p>
+              {{filename}}
+            </p>
+            {{#if isImage}}
+              <img height="150" width="150" src='/attic/{{../slug}}/{{filename}}' alt='{{filename}}' />
+            {{/if}}
+          </a>
+        </div>
+      {{/each}}
+    </div>
+    {{/if}}
+    <p><a class="text-blue-500 underline" href='/attic'>Upload another file</a></p>
+  </div>
+</body>
+
+</html>

views/pow/index.hbs

@@ -0,0 +1,83 @@
+<html>
+  <head>
+    <title>Proof Of Work API</title>
+  </head>
+  <body>
+    <h1>PoW API</h1>
+    <p>
+      This is a simple API that allows you to generate a proof of work
+      challenges and verify the proof of work.
+    </p>
+    <p>
+      The API has two endpoints:
+      <ul>
+        <li>GET
+          <a href='/pow/challenge'>/pow/challenge</a>
+          - This endpoint generates a new proof of work challenge.</li>
+        <li>POST
+          <a href='/pow/challenge'>/pow/challenge</a>
+          - This endpoint verifies a proof of work challenge.</li>
+      </ul>
+    </p>
+    <p>
+      The proof of work challenge is to find a proof which, when concatenated
+      and hashed (via SHA 512) with a challenge string that starts with a
+      certain number of zeros. The number of zeros is determined by the
+      difficulty level. The difficulty level is a number between 1 and 10. The
+      higher the difficulty level, the more zeros the string must start with.
+    </p>
+    <p>
+      The current difficulty is
+      {{difficulty}}.
+    </p>
+    <p>
+        The example code for performing a proof of work challenge is:
+        <pre>
+          const crypto = require('crypto');
+          const axios = require('axios');
+
+          async function getChallenge() {
+            const response = await axios.get('https://api.us.dev/pow/challenge');
+            return response.data;
+          }
+
+          async function submitSolution(challenge, proof) {
+            const response = await axios.post('https://api.us.dev/pow/challenge', {
+              challenge,
+              proof
+            });
+            return response.data;
+          }
+
+          function hashProof(challenge, proof) {
+            return crypto.createHash('sha512').update(challenge + proof).digest('hex');
+          }
+
+          function verifyChallenge(hash: string, difficulty: number) {
+            return hash.startsWith('0'.repeat(difficulty));
+          } 
+
+          async function proofOfWork() {
+            const challenge = await getChallenge();
+            let proof = 0;
+            let hash = '';
+            do {
+              proof++;
+              hash = hashProof(challenge, proof);
+            } while (!verifyChallenge(hash, 5));
+            return await submitSolution(challenge, proof);
+          }
+
+          proofOfWork().then(console.log).catch(console.error);
+          </pre>
+    </p>
+    <p>
+        If you're a developer you should use the GET /pow/challenge endpoint to
+        get the challenge, then pass the challenge to your client, then have the
+        client find a solution to the challenge and then submit their proof as 
+        part of their later action. Your service should then verify the solution 
+        using the POST /pow/challenge endpoint (or your own local implementation)
+        to verify the solution.
+    </p>
+  </body>
+</html>