Improve parkio, add domainrproxy

src/app.module.ts

@@ -6,18 +6,22 @@ import { IswordModule } from './isword/isword.module';
 import { AuthModule } from './auth/auth.module';
 import { UsersModule } from './users/users.module';
 import { ConfigModule } from '@nestjs/config';
+import { DomainrproxyModule } from './domainrproxy/domainrproxy.module';
 import configuration from './config/configuration';
+import { CacheModule } from '@nestjs/cache-manager';
 
 @Module({
   imports: [
     ConfigModule.forRoot({
       isGlobal: true,
-      load: [configuration]
+      load: [configuration],
     }),
+    CacheModule.register({ isGlobal: true }),
     ParkioModule,
     IswordModule,
     AuthModule,
     UsersModule,
+    DomainrproxyModule,
   ],
   controllers: [AppController],
   providers: [AppService],

src/app.service.ts

@@ -3,6 +3,6 @@ import { Injectable } from '@nestjs/common';
 @Injectable()
 export class AppService {
   getHello(): string {
-    return 'Hello World!';
+    return "Hello World! This is Chip's generalized API for fetching information and things. You can contact me on mastodon @chip@talking.dev.";
   }
 }

src/auth/auth.controller.ts

@@ -10,6 +10,7 @@ import {
 } from '@nestjs/common';
 import { AuthService } from './auth.service';
 import { AuthGuard } from './auth.guard';
+import { ApiBearerAuth } from '@nestjs/swagger';
 
 @Controller('auth')
 export class AuthController {
@@ -28,6 +29,7 @@ export class AuthController {
   }
 
   @UseGuards(AuthGuard)
+  @ApiBearerAuth()
   @Get('profile')
   getProfile(@Request() req: any) {
     return req.user;

src/config/configuration.ts

@@ -1,3 +1,6 @@
 export default () => ({
   port: parseInt(process.env.PORT ?? '', 10) || 3000,
+  // UserAgent should be added to calls made to third party apis
+  userAgent: 'api.us.dev/@chip@talking.dev',
+  rapidApiKey: process.env.RAPID_API_KEY || '',
 });

src/domainrproxy/domainrproxy.controller.ts

@@ -0,0 +1,20 @@
+import { Body, Controller, Get, Param, Post, UseGuards } from '@nestjs/common';
+import { DomainrproxyService } from './domainrproxy.service';
+import { AuthGuard } from 'src/auth/auth.guard';
+
+@Controller('domainrproxy')
+export class DomainrproxyController {
+  constructor(private readonly proxyService: DomainrproxyService) {}
+
+  @UseGuards(AuthGuard)
+  @Get(':domain')
+  queryDomain(@Param('domain') domain: string) {
+    return this.proxyService.queryForDomain(domain);
+  }
+
+  @UseGuards(AuthGuard)
+  @Post('search')
+  search(@Body() body: { query: string }) {
+    return this.proxyService.search(body.query);
+  }
+}

src/domainrproxy/domainrproxy.module.ts

@@ -0,0 +1,9 @@
+import { Module } from '@nestjs/common';
+import { DomainrproxyService } from './domainrproxy.service';
+import { DomainrproxyController } from './domainrproxy.controller';
+
+@Module({
+  providers: [DomainrproxyService],
+  controllers: [DomainrproxyController]
+})
+export class DomainrproxyModule {}

src/domainrproxy/domainrproxy.service.ts

@@ -0,0 +1,48 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import axios from 'axios';
+
+@Injectable()
+export class DomainrproxyService {
+  private readonly logger = new Logger(DomainrproxyService.name);
+  constructor(private readonly configService: ConfigService) {}
+
+  queryForDomain = async (domain: string) => {
+    this.logger.verbose(`Handling domainr query for domain ${domain}`);
+    const options = {
+      method: 'GET',
+      url: 'https://domainr.p.rapidapi.com/v2/status',
+      params: {
+        domain,
+        'mashape-key': this.configService.get<string>('rapidApiKey'),
+      },
+      headers: {
+        'X-RapidAPI-Key': this.configService.get<string>('rapidApiKey'),
+        'X-RapidAPI-Host': 'domainr.p.rapidapi.com',
+      },
+    };
+
+    const result = await axios.request(options);
+    return result.data;
+  };
+
+  search = async (query: string) => {
+    this.logger.verbose(`Handling domainr search for term ${query}`);
+    const options = {
+      method: 'GET',
+      url: 'https://domainr.p.rapidapi.com/v2/search',
+      params: {
+        query,
+        defaults: 'com,net,org,io,sh,wtf',
+        'mashape-key': this.configService.get<string>('rapidApiKey'),
+      },
+      headers: {
+        'X-RapidAPI-Key': this.configService.get<string>('rapidApiKey'),
+        'X-RapidAPI-Host': 'domainr.p.rapidapi.com',
+      },
+    };
+
+    const result = await axios.request(options);
+    return result.data;
+  };
+}

src/main.ts

@@ -1,13 +1,19 @@
 import { NestFactory } from '@nestjs/core';
 import { AppModule } from './app.module';
 import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
+import { NestExpressApplication } from '@nestjs/platform-express';
+import { join } from 'path';
 
 async function bootstrap() {
-  const app = await NestFactory.create(AppModule);
+  const app = await NestFactory.create<NestExpressApplication>(AppModule);
+  app.useStaticAssets(join(__dirname, '..', 'public'));
+  app.setBaseViewsDir(join(__dirname, '..', 'views'));
+  app.setViewEngine('hbs');
   const config = new DocumentBuilder()
     .setTitle('us.dev API')
     .setDescription("Chip's generalized API")
     .setVersion('1.0')
+    .addBearerAuth()
     .build();
   const document = SwaggerModule.createDocument(app, config);
   SwaggerModule.setup('api', app, document);

src/parkio/parkio.controller.ts

@@ -1,35 +1,47 @@
-import { Controller, Get, Param } from '@nestjs/common';
+import { Controller, Get, Param, Render, Res } from '@nestjs/common';
 import { ParkioService } from './parkio.service';
 import { ApiTags } from '@nestjs/swagger';
+import { ParsedDomain } from './types';
 
 @ApiTags('parkio')
 @Controller('parkio')
 export class ParkioController {
-    constructor(
-        private readonly parkioService: ParkioService
-    ) {}
+  constructor(private readonly parkioService: ParkioService) {}
 
-    @Get('auctions')
-    async getAuctions () {
-        return this.parkioService.fetchAuctions();
-    }
+  @Get('auctions')
+  async getAuctions() {
+    return this.parkioService.fetchAuctions();
+  }
 
-    @Get('domains')
-    async getDomains () {
-        return this.parkioService.fetchAllDomains();
-    }
+  @Get('domains')
+  async getDomains(): Promise<ParsedDomain[]> {
+    return this.parkioService.fetchAllDomains();
+  }
 
-    @Get('short/:maxLength')
-    async getShort (@Param('maxLength') maxLength: number) {
-        return {
-            maxLength,
-            domains: (await this.parkioService.fetchAllDomains()).filter(
-                parsedDomain => parsedDomain.domain_length <= maxLength
-            ),
-            auctions: (await this.parkioService.fetchAuctions()).filter(
-                parsedAuction => parsedAuction.domain_length <= maxLength
-            ),
-        }
+  @Get('short/:maxLength')
+  async getShort(@Param('maxLength') maxLength: number) {
+    return {
+      maxLength,
+      domains: (await this.parkioService.fetchAllDomains()).filter(
+        (parsedDomain) => parsedDomain.domain_length <= maxLength,
+      ),
+      auctions: (await this.parkioService.fetchAuctions()).filter(
+        (parsedAuction) => parsedAuction.domain_length <= maxLength,
+      ),
+    };
+  }
 
-    }
+  @Get('report')
+  @Render('parkio/dailyreport')
+  async getReport() {
+    const source = {
+      auctions: await this.parkioService.fetchAuctions(),
+      domains: await this.parkioService.fetchAllDomains(),
+    };
+    return {
+      ...source,
+      day: new Date().toLocaleDateString(),
+      wordDomains: source.domains.filter((domain) => domain.is_word),
+    };
+  }
 }

src/parkio/parkio.service.ts

@@ -1,51 +1,124 @@
-import { Injectable } from '@nestjs/common';
+import { Inject, Injectable, Logger } from '@nestjs/common';
 import {
   Auction,
   AuctionsResponse,
   Domain,
-  DomainSet,
   DomainsResponse,
   ParkioTld,
   ParsedAuction,
   ParsedDomain,
+  TLDs,
 } from './types';
 import axios from 'axios';
 import { filter, map, pipe } from 'ramda';
 import { IswordService } from 'src/isword/isword.service';
+import { ConfigService } from '@nestjs/config';
+import { CACHE_MANAGER } from '@nestjs/cache-manager';
+import { Cache } from 'cache-manager';
 
 const parkIoEndpoints = {
   domains: 'https://park.io/domains.json',
   auctions: 'https://park.io/auctions.json',
-  tld: (tld: ParkioTld) => `https://park.io/domains/index/${tld}.json`,
+  tld: (tld: ParkioTld, limit: number = 80_000, page: number = 1) =>
+    `https://park.io/domains/index/${tld}/page:${page}.json?limit=${limit}`,
   all: (limit: number = 10_000) =>
     `https://park.io/domains/index/all.json?limit=${limit}`,
 };
 
 @Injectable()
 export class ParkioService {
+  private readonly logger = new Logger(ParkioService.name);
   constructor(
-    private readonly isWordService: IswordService
+    private configService: ConfigService,
+    private readonly isWordService: IswordService,
+    @Inject(CACHE_MANAGER) private cacheManager: Cache,
   ) {}
 
-  fetchDomainsForTld = async (tld: ParkioTld) => {};
+  tldRawCacheKey = (tld: ParkioTld) => `${tld}_raw`;
 
-  fetchAllDomains = async (limit: number = 10_000): Promise<ParsedDomain[]> => {
-    let response = await axios.get<DomainsResponse>(parkIoEndpoints.all(limit));
+  setTldDomainsCache = async (
+    tld: ParkioTld,
+    domains: ParsedDomain[],
+  ): Promise<void> =>
+    this.cacheManager.set(this.tldRawCacheKey(tld), domains, 300_000);
 
-    // There may have been more than our set limit
-    if (response.data.current === limit) {
-      return await this.fetchAllDomains(limit + 200);
+  getTldDomainsCache = (tld: ParkioTld): Promise<ParsedDomain[] | undefined> =>
+    this.cacheManager.get<ParsedDomain[]>(this.tldRawCacheKey(tld));
+
+  fetchDomainsForTld = async (tld: ParkioTld): Promise<ParsedDomain[]> => {
+    this.logger.verbose(`Fetching domains for ${tld}`);
+    const cacheValue = await this.getTldDomainsCache(tld);
+    if (cacheValue !== undefined) {
+      this.logger.verbose(`${tld} ... cache hit`);
+      return cacheValue;
     }
-
-    return map(this.parseDomain, response.data.domains);
+    this.logger.verbose(`${tld} ... cache miss`);
+    // Recursively fetch domain
+    const results = await this.fetchDomainPageForTld(tld);
+    // Fire the combined results into the cache
+    this.setTldDomainsCache(tld, results);
+    return results;
   };
 
+  fetchDomainPageForTld = async (
+    tld: ParkioTld,
+    pageLimit: number = 1_000,
+    page: number = 1,
+  ): Promise<ParsedDomain[]> => {
+    this.logger.verbose(`Fetching ${pageLimit} .${tld} domains, page ${page}`);
+
+    const url = parkIoEndpoints.tld(tld, pageLimit, page);
+    const response = await axios.get<DomainsResponse>(url, {
+      headers: {
+        Accept: 'application/json',
+        'User-Agent': this.configService.get<string>('userAgent'),
+      },
+    });
+
+    // There may have been more than our set limit
+    if (response.data.nextPage) {
+      return [
+        ...response.data.domains.map(this.parseDomain),
+        ...(await this.fetchDomainPageForTld(tld, pageLimit, page + 1)),
+      ];
+    }
+
+    return response.data.domains.map(this.parseDomain);
+  };
+
+  fetchAllDomains = async (): Promise<ParsedDomain[]> => {
+    const domainResults = await Promise.all(TLDs.map(this.fetchDomainsForTld));
+    return domainResults.reduce((acc, domainResponse) => [
+      ...acc,
+      ...domainResponse,
+    ]);
+  };
+
+  auctionsRawCacheKey = `auctions_raw`;
+
+  setAuctionsCache = async (auctions: ParsedAuction[]): Promise<void> =>
+    this.cacheManager.set(this.auctionsRawCacheKey, auctions);
+
+  getAuctionsCache = async (): Promise<ParsedAuction[] | undefined> =>
+    this.cacheManager.get<ParsedAuction[]>(this.auctionsRawCacheKey);
+
   fetchAuctions = async (): Promise<ParsedAuction[]> => {
+    this.logger.verbose('Fetching auction data');
+    const cachedValue = await this.getAuctionsCache();
+    if (cachedValue !== undefined) {
+      this.logger.verbose('... cache hit');
+      return cachedValue;
+    }
+    this.logger.verbose('... cache miss');
+
     const response = await axios.get<AuctionsResponse>(
       parkIoEndpoints.auctions,
     );
 
-    return map(this.parseAuction, response.data.auctions);
+    this.logger.verbose('Processing auction response');
+    const results = map(this.parseAuction, response.data.auctions);
+    this.setAuctionsCache(results);
+    return results;
   };
 
   lengthFilter = (length: number) => (parsedDomain: ParsedDomain) =>
@@ -80,10 +153,12 @@ export class ParkioService {
       date_registered: domain.date_registered
         ? this.parseDate(domain.date_registered)
         : undefined,
-        is_word: this.isWordService.isWord(domainName)
+      is_word: this.isWordService.isWord(domainName),
     };
   };
 
+  parseDomains = map(this.parseDomain);
+
   parseDate = (date: string) =>
     new Date(
       Date.parse(