Improve parkio, add domainrproxy

src/parkio/parkio.service.ts

@@ -1,51 +1,124 @@
-import { Injectable } from '@nestjs/common';
+import { Inject, Injectable, Logger } from '@nestjs/common';
 import {
   Auction,
   AuctionsResponse,
   Domain,
-  DomainSet,
   DomainsResponse,
   ParkioTld,
   ParsedAuction,
   ParsedDomain,
+  TLDs,
 } from './types';
 import axios from 'axios';
 import { filter, map, pipe } from 'ramda';
 import { IswordService } from 'src/isword/isword.service';
+import { ConfigService } from '@nestjs/config';
+import { CACHE_MANAGER } from '@nestjs/cache-manager';
+import { Cache } from 'cache-manager';
 
 const parkIoEndpoints = {
   domains: 'https://park.io/domains.json',
   auctions: 'https://park.io/auctions.json',
-  tld: (tld: ParkioTld) => `https://park.io/domains/index/${tld}.json`,
+  tld: (tld: ParkioTld, limit: number = 80_000, page: number = 1) =>
+    `https://park.io/domains/index/${tld}/page:${page}.json?limit=${limit}`,
   all: (limit: number = 10_000) =>
     `https://park.io/domains/index/all.json?limit=${limit}`,
 };
 
 @Injectable()
 export class ParkioService {
+  private readonly logger = new Logger(ParkioService.name);
   constructor(
-    private readonly isWordService: IswordService
+    private configService: ConfigService,
+    private readonly isWordService: IswordService,
+    @Inject(CACHE_MANAGER) private cacheManager: Cache,
   ) {}
 
-  fetchDomainsForTld = async (tld: ParkioTld) => {};
+  tldRawCacheKey = (tld: ParkioTld) => `${tld}_raw`;
 
-  fetchAllDomains = async (limit: number = 10_000): Promise<ParsedDomain[]> => {
-    let response = await axios.get<DomainsResponse>(parkIoEndpoints.all(limit));
+  setTldDomainsCache = async (
+    tld: ParkioTld,
+    domains: ParsedDomain[],
+  ): Promise<void> =>
+    this.cacheManager.set(this.tldRawCacheKey(tld), domains, 300_000);
 
-    // There may have been more than our set limit
-    if (response.data.current === limit) {
-      return await this.fetchAllDomains(limit + 200);
+  getTldDomainsCache = (tld: ParkioTld): Promise<ParsedDomain[] | undefined> =>
+    this.cacheManager.get<ParsedDomain[]>(this.tldRawCacheKey(tld));
+
+  fetchDomainsForTld = async (tld: ParkioTld): Promise<ParsedDomain[]> => {
+    this.logger.verbose(`Fetching domains for ${tld}`);
+    const cacheValue = await this.getTldDomainsCache(tld);
+    if (cacheValue !== undefined) {
+      this.logger.verbose(`${tld} ... cache hit`);
+      return cacheValue;
     }
-
-    return map(this.parseDomain, response.data.domains);
+    this.logger.verbose(`${tld} ... cache miss`);
+    // Recursively fetch domain
+    const results = await this.fetchDomainPageForTld(tld);
+    // Fire the combined results into the cache
+    this.setTldDomainsCache(tld, results);
+    return results;
   };
 
+  fetchDomainPageForTld = async (
+    tld: ParkioTld,
+    pageLimit: number = 1_000,
+    page: number = 1,
+  ): Promise<ParsedDomain[]> => {
+    this.logger.verbose(`Fetching ${pageLimit} .${tld} domains, page ${page}`);
+
+    const url = parkIoEndpoints.tld(tld, pageLimit, page);
+    const response = await axios.get<DomainsResponse>(url, {
+      headers: {
+        Accept: 'application/json',
+        'User-Agent': this.configService.get<string>('userAgent'),
+      },
+    });
+
+    // There may have been more than our set limit
+    if (response.data.nextPage) {
+      return [
+        ...response.data.domains.map(this.parseDomain),
+        ...(await this.fetchDomainPageForTld(tld, pageLimit, page + 1)),
+      ];
+    }
+
+    return response.data.domains.map(this.parseDomain);
+  };
+
+  fetchAllDomains = async (): Promise<ParsedDomain[]> => {
+    const domainResults = await Promise.all(TLDs.map(this.fetchDomainsForTld));
+    return domainResults.reduce((acc, domainResponse) => [
+      ...acc,
+      ...domainResponse,
+    ]);
+  };
+
+  auctionsRawCacheKey = `auctions_raw`;
+
+  setAuctionsCache = async (auctions: ParsedAuction[]): Promise<void> =>
+    this.cacheManager.set(this.auctionsRawCacheKey, auctions);
+
+  getAuctionsCache = async (): Promise<ParsedAuction[] | undefined> =>
+    this.cacheManager.get<ParsedAuction[]>(this.auctionsRawCacheKey);
+
   fetchAuctions = async (): Promise<ParsedAuction[]> => {
+    this.logger.verbose('Fetching auction data');
+    const cachedValue = await this.getAuctionsCache();
+    if (cachedValue !== undefined) {
+      this.logger.verbose('... cache hit');
+      return cachedValue;
+    }
+    this.logger.verbose('... cache miss');
+
     const response = await axios.get<AuctionsResponse>(
       parkIoEndpoints.auctions,
     );
 
-    return map(this.parseAuction, response.data.auctions);
+    this.logger.verbose('Processing auction response');
+    const results = map(this.parseAuction, response.data.auctions);
+    this.setAuctionsCache(results);
+    return results;
   };
 
   lengthFilter = (length: number) => (parsedDomain: ParsedDomain) =>
@@ -80,10 +153,12 @@ export class ParkioService {
       date_registered: domain.date_registered
         ? this.parseDate(domain.date_registered)
         : undefined,
-        is_word: this.isWordService.isWord(domainName)
+      is_word: this.isWordService.isWord(domainName),
     };
   };
 
+  parseDomains = map(this.parseDomain);
+
   parseDate = (date: string) =>
     new Date(
       Date.parse(