import { Inject, Injectable, Logger } from '@nestjs/common'; import { Auction, AuctionsResponse, Domain, DomainsResponse, ParkioTld, ParsedAuction, ParsedDomain, TLDs, } from './types'; import axios from 'axios'; import { filter, map, pipe } from 'ramda'; import { IswordService } from 'src/isword/isword.service'; import { ConfigService } from '@nestjs/config'; import { CACHE_MANAGER } from '@nestjs/cache-manager'; import { Cache } from 'cache-manager'; const parkIoEndpoints = { domains: 'https://park.io/domains.json', auctions: 'https://park.io/auctions.json', tld: (tld: ParkioTld, limit: number = 80_000, page: number = 1) => `https://park.io/domains/index/${tld}/page:${page}.json?limit=${limit}`, all: (limit: number = 10_000) => `https://park.io/domains/index/all.json?limit=${limit}`, }; @Injectable() export class ParkioService { private readonly logger = new Logger(ParkioService.name); constructor( private configService: ConfigService, private readonly isWordService: IswordService, @Inject(CACHE_MANAGER) private cacheManager: Cache, ) {} tldRawCacheKey = (tld: ParkioTld) => `${tld}_raw`; setTldDomainsCache = async ( tld: ParkioTld, domains: ParsedDomain[], ): Promise => this.cacheManager.set(this.tldRawCacheKey(tld), domains, 300_000); getTldDomainsCache = (tld: ParkioTld): Promise => this.cacheManager.get(this.tldRawCacheKey(tld)); fetchDomainsForTld = async (tld: ParkioTld): Promise => { this.logger.verbose(`Fetching domains for ${tld}`); const cacheValue = await this.getTldDomainsCache(tld); if (cacheValue !== undefined) { this.logger.verbose(`${tld} ... cache hit`); return cacheValue; } this.logger.verbose(`${tld} ... cache miss`); // Recursively fetch domain const results = await this.fetchDomainPageForTld(tld); // Fire the combined results into the cache this.setTldDomainsCache(tld, results); return results; }; fetchDomainPageForTld = async ( tld: ParkioTld, pageLimit: number = 1_000, page: number = 1, ): Promise => { this.logger.verbose(`Fetching ${pageLimit} .${tld} domains, page ${page}`); const url = parkIoEndpoints.tld(tld, pageLimit, page); const response = await axios.get(url, { headers: { Accept: 'application/json', 'User-Agent': this.configService.get('userAgent'), }, }); // There may have been more than our set limit if (response.data.nextPage) { return [ ...response.data.domains.map(this.parseDomain), ...(await this.fetchDomainPageForTld(tld, pageLimit, page + 1)), ]; } return response.data.domains.map(this.parseDomain); }; fetchAllDomains = async (): Promise => { const domainResults = await Promise.all(TLDs.map(this.fetchDomainsForTld)); return domainResults.reduce((acc, domainResponse) => [ ...acc, ...domainResponse, ]); }; auctionsRawCacheKey = `auctions_raw`; setAuctionsCache = async (auctions: ParsedAuction[]): Promise => this.cacheManager.set(this.auctionsRawCacheKey, auctions); getAuctionsCache = async (): Promise => this.cacheManager.get(this.auctionsRawCacheKey); fetchAuctions = async (): Promise => { this.logger.verbose('Fetching auction data'); const cachedValue = await this.getAuctionsCache(); if (cachedValue !== undefined) { this.logger.verbose('... cache hit'); return cachedValue; } this.logger.verbose('... cache miss'); const response = await axios.get( parkIoEndpoints.auctions, ); this.logger.verbose('Processing auction response'); const results = map(this.parseAuction, response.data.auctions); this.setAuctionsCache(results); return results; }; lengthFilter = (length: number) => (parsedDomain: ParsedDomain) => parsedDomain.domain.length === length; buildExtensions = (domains: ParsedDomain[], extensions: ParkioTld[]) => pipe( map((extension: ParkioTld) => pipe( filter(this.tldFilter(extension)), filter(this.maxLengthFilter(3)), )(domains), ), )(extensions); extractDomain = (domain: string): string => domain.split('.').slice(0, -1).join('.'); extractTld = (domain: string): string => domain.split('.').slice(-1).join('.'); parseDomain = (domain: Domain): ParsedDomain => { const domainName = this.extractDomain(domain.name); return { ...domain, id: Number(domain.id), domain: domainName, domain_length: domainName.length, date_available: domain.date_available ? this.parseDate(domain.date_available) : undefined, date_registered: domain.date_registered ? this.parseDate(domain.date_registered) : undefined, is_word: this.isWordService.isWord(domainName), }; }; parseDomains = map(this.parseDomain); parseDate = (date: string) => new Date( Date.parse( date.replace( /(\d+)-(\d+)-(\d+)UTC(\d+:\d+:\d{0,2})\d*/, '$1-$3-$2T$4Z', ), ), ); parseAuction = (auction: Auction): ParsedAuction => { const domain = this.extractDomain(auction.name); const tld = this.extractTld(auction.name); return { ...auction, id: Number(auction.id), num_bids: Number(auction.num_bids), price: Number(auction.price), close_date: this.parseDate(auction.close_date), created: this.parseDate(auction.created), domain, domain_length: domain.length, is_word: this.isWordService.isWord(domain), tld, }; }; domainToString = (domain: ParsedDomain) => `${domain.name}\t${domain.date_available}`; auctionToString = (auction: ParsedAuction) => `${auction.name}\t$${auction.price}\t${auction.close_date}`; tldFilter = (tld: ParkioTld) => (domain: ParsedDomain) => domain.tld === tld; maxLengthFilter = (length: number) => (parsedDomain: ParsedDomain) => parsedDomain.domain.length <= length; }