From b07ecef00e173ba5cd215d04ded95779b5a4c9cf Mon Sep 17 00:00:00 2001 From: Omar Aziz Date: Tue, 1 Aug 2023 17:24:00 -0700 Subject: [PATCH 1/3] nft token trade history --- src/handlers/getNFTs.ts | 62 ++++-- src/lib/fetcher.ts | 15 ++ src/lib/nft/class.ts | 11 + src/lib/nft/index.ts | 11 + src/lib/nft/quick-node.ts | 408 ++++++++++++++++++++++++++++++++++++++ src/lib/nft/reservoir.ts | 6 +- src/lib/nft/sequence.ts | 41 ++++ src/lib/type.ts | 2 + 8 files changed, 542 insertions(+), 14 deletions(-) create mode 100644 src/lib/nft/class.ts create mode 100644 src/lib/nft/quick-node.ts diff --git a/src/handlers/getNFTs.ts b/src/handlers/getNFTs.ts index a2f178ed3..91910ba20 100644 --- a/src/handlers/getNFTs.ts +++ b/src/handlers/getNFTs.ts @@ -4,7 +4,13 @@ import { ADDRESS_ZERO } from '@lib/contract' import { paginatedFetch } from '@lib/fetcher' import { parseStringJSON } from '@lib/fmt' import type { DefillamaNFTCollection } from '@lib/nft' -import { defillamaCollections, fetchNFTMetadataFrom, fetchUserNFTCollectionsFrom, fetchUserNFTsFrom } from '@lib/nft' +import { + defillamaCollections, + fetchNFTMetadataFrom, + fetchNFTTradingHistoryFrom, + fetchUserNFTCollectionsFrom, + fetchUserNFTsFrom, +} from '@lib/nft' import type { NftScanMetadata as NFTMetadata, UserNFTCollection } from '@lib/nft/nft-scan' import { fetchTokenPrices } from '@lib/price' import { isFulfilled } from '@lib/promise' @@ -40,7 +46,7 @@ export const handler: APIGatewayProxyHandler = async (event, context) => { } const response = await nftsHandler({ address }) - return success(response, { maxAge: 30 * 60 }) + return success(response, { maxAge: 1800 }) } catch (error) { console.error('Failed to fetch user NFTs', { error }) return serverError('Failed to fetch user NFTs') @@ -63,11 +69,17 @@ async function tokensPrice(ids: Array<`${Chain}:${Address}`>) { // TODO: add rate limit checker const rateLimitReached = true -export async function getUserNFTTokens(address: Address) { +export async function getUserNFTs(address: Address) { if (rateLimitReached) { const userNFTsResponse = await paginatedFetch({ fn: fetchUserNFTsFrom.alchemy, - initialParams: { address, spamConfidenceLevel: 'LOW', withMetadata: false, pageSize: 100 }, + initialParams: { + address, + pageSize: 100, + chain: 'ethereum', + withMetadata: false, + spamConfidenceLevel: 'LOW', + }, iterations: 10, pageKeyProp: 'pageKey', }) @@ -93,13 +105,17 @@ export async function getUserNFTTokens(address: Address) { .sort((a, b) => a.id.localeCompare(b.id)) } -export async function nftsHandler({ address }: { address: Address }): Promise { - const { price: ethPrice } = await tokensPrice([`ethereum:${ADDRESS_ZERO}`]) - - const userNFTs = await getUserNFTTokens(address) - - const chunks = sliceIntoChunks(userNFTs, 50) - +export async function getNFTsMetadata({ + nfts, + maxBatchSize = 50, +}: { + nfts: Array<{ + address: string + tokenID: string + }> + maxBatchSize?: number +}) { + const chunks = sliceIntoChunks(nfts, maxBatchSize) const metadataPromiseResult = await Promise.allSettled( chunks.map((chunk) => fetchNFTMetadataFrom.nftScan({ @@ -122,9 +138,29 @@ export async function nftsHandler({ address }: { address: Address }): Promise a.id.localeCompare(b.id)) + return flattenedMetadata +} + +export async function nftsHandler({ address }: { address: Address }): Promise { + const { price: ethPrice } = await tokensPrice([`ethereum:${ADDRESS_ZERO}`]) + + const userNFTs = await getUserNFTs(address) + + const nftsMetadata = await getNFTsMetadata({ + nfts: userNFTs.map((nft) => ({ address: nft.address, tokenID: nft.tokenID })), + }) + + const nftsTradeHistory = await fetchNFTTradingHistoryFrom.quickNode( + nftsMetadata.map((nft) => ({ contractAddress: nft.contract_address, tokenId: nft.token_id, chain: 'ethereum' })), + ) + const mergedNFTs = userNFTs.map((nft, index) => { - const metadata = flattenedMetadata[index] - return { ...nft, ...metadata } + const metadata = nftsMetadata[index] + const tradeHistory = nftsTradeHistory[ + `_${metadata?.contract_address?.toLowerCase()}_${metadata?.token_id?.toLowerCase()}` + ]?.nft?.tokenEvents?.edges?.map(({ node }) => node) + + return { ...nft, ...metadata, history: tradeHistory ?? [] } }) const { erc1155, erc721 } = mergedNFTs.reduce( diff --git a/src/lib/fetcher.ts b/src/lib/fetcher.ts index 4e87f771f..8413edb42 100644 --- a/src/lib/fetcher.ts +++ b/src/lib/fetcher.ts @@ -1,4 +1,19 @@ import { raise } from '@lib/error' +import type { Json } from '@lib/type' + +/** this function fetches the schema of any public GraphQL endpoint */ +export async function fetchGraphQLSchema(url: string): Promise { + const response = await fetcher(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + query: + 'query IntrospectionQuery { __schema { queryType { name } mutationType { name } subscriptionType { name } types { ...FullType } directives { name description locations args { ...InputValue } } } } fragment FullType on __Type { kind name description fields(includeDeprecated: true) { name description args { ...InputValue } type { ...TypeRef } isDeprecated deprecationReason } inputFields { ...InputValue } interfaces { ...TypeRef } enumValues(includeDeprecated: true) { name description isDeprecated deprecationReason } possibleTypes { ...TypeRef } } fragment InputValue on __InputValue { name description type { ...TypeRef } defaultValue } fragment TypeRef on __Type { kind name ofType { kind name ofType { kind name ofType { kind name ofType { kind name ofType { kind name ofType { kind name ofType { kind name } } } } } } } }', + variables: {}, + }), + }) + return response +} export async function fetcher(url: string, options?: RequestInit) { const response = await fetch(url, { diff --git a/src/lib/nft/class.ts b/src/lib/nft/class.ts new file mode 100644 index 000000000..81879def8 --- /dev/null +++ b/src/lib/nft/class.ts @@ -0,0 +1,11 @@ +import type { Chain } from '@lib/chains' +import type { Address } from 'viem' + +export abstract class UserNFTs { + constructor( + public readonly walletAddress: Address, + public readonly chain: Chain, + ) {} + + abstract fetch(): Promise +} diff --git a/src/lib/nft/index.ts b/src/lib/nft/index.ts index d47c58221..c463210ff 100644 --- a/src/lib/nft/index.ts +++ b/src/lib/nft/index.ts @@ -17,6 +17,11 @@ import { fetchUserNFTCollections as fetchUserNFTCollectionsFromNftScan, fetchUserNFTs as fetchUserNFTsFromNftScan, } from './nft-scan' +import { + batchFetchMetadataFromQuickNode, + batchFetchNFTTradingHistoryFromQuickNode, + fetchUserNFTsFromQuickNode, +} from './quick-node' import { fetchUserNFTCollections as fetchUserNFTCollectionsFromReservoir, fetchUsersNFTActivity as fetchUsersNFTActivityFromReservoir, @@ -31,12 +36,14 @@ export const fetchUserNFTsFrom = { nftScan: fetchUserNFTsFromNftScan, sequence: fetchUserNFTsFromSequence, center: fetchUserNFTsFromCenter, + quickNode: fetchUserNFTsFromQuickNode, } export const fetchNFTMetadataFrom = { center: batchFetchMetadataFromCenter, alchemy: batchFetchMetadataFromAlchemy, nftScan: batchFetchMetadataFromNftScan, + quickNode: batchFetchMetadataFromQuickNode, } export const fetchUserNFTCollectionsFrom = { @@ -47,3 +54,7 @@ export const fetchUserNFTCollectionsFrom = { export const fetchUserNFTActivityFrom = { reservoir: fetchUsersNFTActivityFromReservoir, } + +export const fetchNFTTradingHistoryFrom = { + quickNode: batchFetchNFTTradingHistoryFromQuickNode, +} diff --git a/src/lib/nft/quick-node.ts b/src/lib/nft/quick-node.ts new file mode 100644 index 000000000..844049f7f --- /dev/null +++ b/src/lib/nft/quick-node.ts @@ -0,0 +1,408 @@ +import type { Chain } from '@lib/chains' +import { raise } from '@lib/error' +import { fetcher } from '@lib/fetcher' +import { getAddress, isHex } from 'viem' + +const QUICKNODE_BASE_URL = 'https://api.quicknode.com/graphql' + +export type QuickNodeChain = Extract + +/** + * GraphQL Playgorund + * https://studio.apollographql.com/public/QuickNode-Federation-API/variant/production/explorer + */ + +export const TransactionFragment = /* graphql */ ` + fragment TransactionFragment on Transaction { + blockNumber + blockTimestamp + contractAddress + fromAddress + toAddress + type + value + gas + } +` + +export const TokenEventFragment = /* graphql */ ` + ${TransactionFragment} + fragment TokenEventFragment on TokenEvent { + type + timestamp + blockNumber + fromAddress + toAddress + transferIndex + transactionHash + transaction { ...TransactionFragment } + } +` + +export const TokenEventsFragment = /* graphql */ ` + ${TokenEventFragment} + fragment TokenEventsFragment on NFTTokenEventsConnection { + totalCount + edges { + node { ...TokenEventFragment } + } + pageInfo { hasNextPage } + } +` + +export const CollectionFragment = /* graphql */ ` + ${TokenEventsFragment} + fragment CollectionFragment on Collection { + name + address + slug + symbol + attributes { + totalCount + edges { + node { + name + value + } + } + } + } +` + +export const NFTFragment = /* graphql */ ` + ${CollectionFragment} + fragment NFTFragment on NFT { + name + contractAddress + tokenId + description + externalUrl + collection { ...CollectionFragment } + tokenEvents { ...TokenEventsFragment } + } +` +export const WalletNFTsFragment = /* graphql */ ` + ${NFTFragment} + fragment WalletNFTsFragment on WalletNFTsConnection { + totalCount + edges { + node { + nft { + ... on ERC1155NFT { + ...NFTFragment + } + ... on ERC721NFT { + ...NFTFragment + } + } + } + } + pageInfo { hasNextPage } + } +` + +export async function fetchUserNFTsFromQuickNode({ + address, + chains = ['ethereum'], +}: { + address: string + chains?: Array +}) { + const walletAddress = getAddress(address) ?? raise(`Invalid address ${address}`) + + const response = await fetcher>(QUICKNODE_BASE_URL, { + method: 'POST', + body: JSON.stringify({ + query: /* graphql */ ` + ${WalletNFTsFragment} + query WalletNFTsQuery($address: String!, $orderBy: WalletNFTsOrderBy = DATE_ACQUIRED) { + ${chains.map( + (chain) => /* graphql */ ` + ${chain} { + walletByAddress(address: $address) { + address + ensName + walletNFTs(orderBy: $orderBy) { + ...WalletNFTsFragment + } + } + } + `, + )} + }`, + variables: { + operationName: 'WalletNFTsQuery', + address: walletAddress, + }, + }), + }) + return response.data +} + +export async function batchFetchMetadataFromQuickNode< + T extends { + contractAddress: string + tokenId: string + chain?: QuickNodeChain + }, +>(parameters: Array) { + parameters.map((item) => isHex(item.contractAddress) ?? raise(`Invalid address ${item.contractAddress}`)) + + const query = /* graphql */ ` + query NFTMetadata { + ${parameters.map( + (item) => /* graphql */ ` + _${item.contractAddress.toLowerCase()}_${item.tokenId.toLowerCase()}: ${item.chain} { + nft(tokenId: "${item.tokenId}", contractAddress: "${item.contractAddress}") { + name + metadata + description + externalUrl + animationUrl + contractAddress + collectionSlug + ... on ERC1155NFT { + name + metadata + externalUrl + description + animationUrl + collectionSlug + contractAddress + } + ... on ERC721NFT { + name + metadata + externalUrl + description + contractAddress + collectionSlug + attributes { + value + name + } + } + } + }`, + )} + }` + + const response = await fetcher>(QUICKNODE_BASE_URL, { + method: 'POST', + body: JSON.stringify({ + query, + }), + }) + return response.data +} + +// batchFetchNFTTradingHistoryFromQuickNode([ +// { +// chain: 'ethereum', +// contractAddress: '0x57f1887a8bf19b14fc0df6fd9b2acc9af147ea85', +// tokenId: '7584185289723993195313796573155465997226240911118312831760400397049224254842', +// }, +// ] as const).then((_) => console.log(JSON.stringify(_, undefined, 2))) + +export async function batchFetchNFTTradingHistoryFromQuickNode< + T extends { + contractAddress: string + tokenId: string + chain?: QuickNodeChain + }, +>(parameters: ReadonlyArray) { + parameters.map((item) => isHex(item.contractAddress) ?? raise(`Invalid address ${item.contractAddress}`)) + + const query = /* graphql */ ` + query TokenEvents { + ${parameters.map( + (item) => /* graphql */ ` + _${item.contractAddress.toLowerCase()}_${item.tokenId.toLowerCase()}: ${item.chain ?? 'ethereum'} { + nft(contractAddress: "${item.contractAddress}", tokenId: "${item.tokenId}") { + tokenEvents { + totalCount + edges { + node { + type + transactionHash + fromAddress + toAddress + timestamp + blockNumber + transferIndex + ... on TokenBurnEvent { + type + transferIndex + transactionHash + tokenQuantity + tokenId + toAddress + timestamp + fromAddress + contractERCStandard + contractAddress + blockNumber + } + ... on TokenMintEvent { + type + transferIndex + transactionHash + tokenQuantity + tokenId + toAddress + timestamp + fromAddress + contractERCStandard + contractAddress + blockNumber + } + ... on TokenSaleEvent { + type + transferIndex + transactionHash + toAddress + timestamp + sentTokenQuantity + sentTokenId + receivedTokenQuantity + receivedTokenId + receivedTokenContractAddress + marketplace + fromAddress + contractERCStandard + contractAddress + blockNumber + } + } + } + } + } + }`, + )} + }` + + const response = await fetcher<{ + data: Record<`_${T['contractAddress']}_${T['tokenId']}`, { nft: { tokenEvents: TokenEvent } }> + }>(QUICKNODE_BASE_URL, { + method: 'POST', + body: JSON.stringify({ query }), + }) + return response.data +} + +interface QuickNodeResponse { + data: { + [chain in QuickNodeChain]: T | null + } +} + +interface QuickNodeUserNFTs { + walletByAddress: { + address: string + ensName: string + walletNFTs: { + totalCount: number + edges: Array<{ + node: { + nft: { + name?: string + contractAddress: string + tokenId: any + description?: string + externalUrl?: string + collection: { + name: string + address: string + slug: any + symbol?: string + attributes: { + totalCount: any + edges: Array<{ + node: { + name: string + value: string + } + }> + } + } + tokenEvents: { + totalCount: number + edges: Array<{ + node: { + type: string + timestamp: string + blockNumber: number + fromAddress: string + toAddress: string + transferIndex: number + transactionHash: string + transaction: { + blockNumber: number + blockTimestamp: string + contractAddress: any + fromAddress: string + toAddress: string + type: string + value: any + gas: number + } + } + }> + pageInfo: { + hasNextPage: boolean + } + } + } + } + }> + pageInfo: { + hasNextPage: boolean + } + } + } +} + +interface TokenEvent { + totalCount: number + edges: Array<{ + node: { + type: string + transactionHash: string + fromAddress: string + toAddress: string + timestamp: string + blockNumber: number + transferIndex: number + sentTokenQuantity?: number + sentTokenId?: number + receivedTokenQuantity?: string + receivedTokenId: any + receivedTokenContractAddress?: string + marketplace?: string + contractERCStandard?: string + contractAddress?: string + tokenQuantity?: number + tokenId?: number + } + }> +} + +interface QuickNodeNFT { + name: string | null + metadata: { + image: string + attributes: Array<{ + value: string + trait_type: string + }> + } + description: any + externalUrl: any + animationUrl: any + contractAddress: string + collectionSlug: any + attributes: Array<{ + value: string + name: string + }> +} diff --git a/src/lib/nft/reservoir.ts b/src/lib/nft/reservoir.ts index 3242c81ba..cc087dbdf 100644 --- a/src/lib/nft/reservoir.ts +++ b/src/lib/nft/reservoir.ts @@ -8,7 +8,11 @@ const RESERVOIR_BASE_URL = `https://api.reservoir.tools` const RESERVOIR_API_KEY = environment.RESERVOIR_API_KEY ?? raise('Missing RESERVOIR_API_KEY') const AUTH_HEADER = { 'X-API-KEY': RESERVOIR_API_KEY } -// https://docs.reservoir.tools/reference/getusersactivityv6 +/** + * if `includeMetadata` is true, the response will return 20 items max, if false, 1000 items max + * better set to false and use a separate API to fetch metadata + * https://docs.reservoir.tools/reference/getusersactivityv6 + */ export async function fetchUsersNFTActivity({ users, collection, diff --git a/src/lib/nft/sequence.ts b/src/lib/nft/sequence.ts index 7b425aa2a..105311590 100644 --- a/src/lib/nft/sequence.ts +++ b/src/lib/nft/sequence.ts @@ -34,6 +34,47 @@ export async function fetchUserNFTs({ address, chain = 'ethereum' }: { address: return response } +export async function fetchWalletTransactionHistory({ + accountAddresses, + contractAddresses, + transactionHashes, + fromBlock, + toBlock, + includeMetadata = false, +}: { + accountAddresses: Array
+ contractAddresses?: Array
+ transactionHashes?: string + fromBlock?: number + toBlock?: number + includeMetadata?: boolean +}) { + const walletAddresses = + accountAddresses.map(getAddress) ?? + raise(`One or more invalud addresses: ${JSON.stringify(accountAddresses, undefined, 2)}`) + const contractAddressesArray = contractAddresses + ? contractAddresses.map(getAddress) ?? + raise(`One or more invalud addresses: ${JSON.stringify(contractAddresses, undefined, 2)}`) + : undefined + const url = `${SEQUENCE_BASE_URL('ethereum')}/GetTransactionHistory` + + const response = await fetcher(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + filter: { + accountAddresses: walletAddresses, + contractAddresses: contractAddressesArray, + includeMetadata, + }, + }), + }) + if ('error' in response) { + raise(`[sequence] error for url ${url}: ${response.error}`) + } + return response +} + interface SequenceError { status: number code: string diff --git a/src/lib/type.ts b/src/lib/type.ts index cc11dbbdf..7b051a7ad 100644 --- a/src/lib/type.ts +++ b/src/lib/type.ts @@ -1,3 +1,5 @@ +export type Json = string | number | boolean | null | { [key: string]: Json | undefined } | Json[] + export function isNotNullish(param: T | undefined | null): param is T { return param != null } From 5da91779f6bd8d7e4201b1a3723aa02305758f02 Mon Sep 17 00:00:00 2001 From: Omar Aziz Date: Tue, 1 Aug 2023 19:54:12 -0700 Subject: [PATCH 2/3] split into 2 endpoints --- .vscode/settings.json | 3 ++- serverless.yml | 10 ++++++++ src/handlers/getNFTs.ts | 23 ++++-------------- src/handlers/getNFTsHistory.ts | 44 ++++++++++++++++++++++++++++++++++ src/lib/nft/quick-node.ts | 37 ++++++++++++++++++---------- src/lib/type.ts | 2 ++ tsconfig.json | 2 +- 7 files changed, 88 insertions(+), 33 deletions(-) create mode 100644 src/handlers/getNFTsHistory.ts diff --git a/.vscode/settings.json b/.vscode/settings.json index db62a7560..d93e5d50f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,7 +3,8 @@ "**/node_modules": true, "pnpm-lock.yaml": true, ".esbuild": true, - ".eslintignore": true + ".eslintignore": true, + "_": true }, "window.title": "🦙 ${rootName} ${separator} ${activeEditorMedium} 🦙", // git diff --git a/serverless.yml b/serverless.yml index 19fff097a..3498d1d9f 100644 --- a/serverless.yml +++ b/serverless.yml @@ -84,6 +84,16 @@ functions: timeout: 29 memorySize: 1024 + getNFTsHistory: + handler: src/handlers/getNFTsHistory.handler + description: Get nfts activity history + events: + - httpApi: + method: post + path: /nfts-history/{address} + timeout: 29 + memorySize: 1024 + getAdapters: handler: src/handlers/getAdapters.handler description: Get adapters diff --git a/src/handlers/getNFTs.ts b/src/handlers/getNFTs.ts index 91910ba20..89fcbadd8 100644 --- a/src/handlers/getNFTs.ts +++ b/src/handlers/getNFTs.ts @@ -4,16 +4,11 @@ import { ADDRESS_ZERO } from '@lib/contract' import { paginatedFetch } from '@lib/fetcher' import { parseStringJSON } from '@lib/fmt' import type { DefillamaNFTCollection } from '@lib/nft' -import { - defillamaCollections, - fetchNFTMetadataFrom, - fetchNFTTradingHistoryFrom, - fetchUserNFTCollectionsFrom, - fetchUserNFTsFrom, -} from '@lib/nft' +import { defillamaCollections, fetchNFTMetadataFrom, fetchUserNFTCollectionsFrom, fetchUserNFTsFrom } from '@lib/nft' import type { NftScanMetadata as NFTMetadata, UserNFTCollection } from '@lib/nft/nft-scan' import { fetchTokenPrices } from '@lib/price' import { isFulfilled } from '@lib/promise' +import type { AwaitedReturnType } from '@lib/type' import type { APIGatewayProxyHandler } from 'aws-lambda' import type { Address } from 'viem' import { isAddress } from 'viem' @@ -35,7 +30,6 @@ interface UserNFTsResponse { } export const handler: APIGatewayProxyHandler = async (event, context) => { - context.callbackWaitsForEmptyEventLoop = false try { const address = event.pathParameters?.address if (!address) { @@ -46,7 +40,7 @@ export const handler: APIGatewayProxyHandler = async (event, context) => { } const response = await nftsHandler({ address }) - return success(response, { maxAge: 1800 }) + return success(response, { maxAge: 30 * 60 }) } catch (error) { console.error('Failed to fetch user NFTs', { error }) return serverError('Failed to fetch user NFTs') @@ -126,7 +120,7 @@ export async function getNFTsMetadata({ ) const metadataFulfilledResults = ( metadataPromiseResult.filter((result) => isFulfilled(result)) as PromiseFulfilledResult< - Awaited> + AwaitedReturnType >[] ).flatMap((item) => item.value.data) @@ -150,17 +144,10 @@ export async function nftsHandler({ address }: { address: Address }): Promise ({ address: nft.address, tokenID: nft.tokenID })), }) - const nftsTradeHistory = await fetchNFTTradingHistoryFrom.quickNode( - nftsMetadata.map((nft) => ({ contractAddress: nft.contract_address, tokenId: nft.token_id, chain: 'ethereum' })), - ) - const mergedNFTs = userNFTs.map((nft, index) => { const metadata = nftsMetadata[index] - const tradeHistory = nftsTradeHistory[ - `_${metadata?.contract_address?.toLowerCase()}_${metadata?.token_id?.toLowerCase()}` - ]?.nft?.tokenEvents?.edges?.map(({ node }) => node) - return { ...nft, ...metadata, history: tradeHistory ?? [] } + return { ...nft, ...metadata } }) const { erc1155, erc721 } = mergedNFTs.reduce( diff --git a/src/handlers/getNFTsHistory.ts b/src/handlers/getNFTsHistory.ts new file mode 100644 index 000000000..f6e711f2f --- /dev/null +++ b/src/handlers/getNFTsHistory.ts @@ -0,0 +1,44 @@ +import { badRequest, serverError, success } from '@handlers/response' +import { sliceIntoChunks } from '@lib/array' +import { fetchNFTTradingHistoryFrom } from '@lib/nft' +import type { QuickNodeChain } from '@lib/nft/quick-node' +import type { APIGatewayProxyHandler } from 'aws-lambda' +import { isAddress } from 'viem' + +/** + * Takes array of nft tokens (contractAddress, tokenId, chain) and returns trading history for each token + */ +export const handler: APIGatewayProxyHandler = async (event, context) => { + context.callbackWaitsForEmptyEventLoop = false + try { + const address = event.pathParameters?.address + if (!address) { + return badRequest('Missing address parameter') + } + if (!isAddress(address)) { + return badRequest('Invalid address parameter, expected hex') + } + + const body = JSON.parse(event.body ?? '[{}]') as Array<{ + contractAddress: string + tokenId: string + chain: QuickNodeChain + }> + + if (!Array.isArray(body)) { + return badRequest('Invalid body parameter, expected array') + } + + const chunks = sliceIntoChunks(body, 100) + + const promisesResponse = await Promise.all(chunks.map((chunk) => fetchNFTTradingHistoryFrom.quickNode(chunk))) + const response = promisesResponse.flat() + + return success(response, { maxAge: 30 * 60 }) + } catch (error) { + console.error('Failed to fetch NFTs trading history', { error }) + return serverError('Failed to fetch NFTs trading history') + } finally { + console.log('NFTs trading history request took', context.getRemainingTimeInMillis() / 1000, 'seconds') + } +} diff --git a/src/lib/nft/quick-node.ts b/src/lib/nft/quick-node.ts index 844049f7f..c1ebec454 100644 --- a/src/lib/nft/quick-node.ts +++ b/src/lib/nft/quick-node.ts @@ -1,12 +1,24 @@ +import environment from '@environment' import type { Chain } from '@lib/chains' import { raise } from '@lib/error' import { fetcher } from '@lib/fetcher' import { getAddress, isHex } from 'viem' const QUICKNODE_BASE_URL = 'https://api.quicknode.com/graphql' +const QUICKNODE_API_KEY = environment.QUICKNODE_API_KEY ?? raise('QUICKNODE_API_KEY is not set') +const AUTH_HEADER = { 'X-API-KEY': QUICKNODE_API_KEY } export type QuickNodeChain = Extract +interface QuickNodeResponse { + data: { + [chain in QuickNodeChain]: T | null + } + code?: number + message?: string + name?: string +} + /** * GraphQL Playgorund * https://studio.apollographql.com/public/QuickNode-Federation-API/variant/production/explorer @@ -112,6 +124,7 @@ export async function fetchUserNFTsFromQuickNode({ const response = await fetcher>(QUICKNODE_BASE_URL, { method: 'POST', + headers: AUTH_HEADER, body: JSON.stringify({ query: /* graphql */ ` ${WalletNFTsFragment} @@ -136,6 +149,9 @@ export async function fetchUserNFTsFromQuickNode({ }, }), }) + if (Object.hasOwn(response, 'error')) { + raise(response) + } return response.data } @@ -189,21 +205,17 @@ export async function batchFetchMetadataFromQuickNode< const response = await fetcher>(QUICKNODE_BASE_URL, { method: 'POST', + headers: AUTH_HEADER, body: JSON.stringify({ query, }), }) + if (Object.hasOwn(response, 'error')) { + raise(response) + } return response.data } -// batchFetchNFTTradingHistoryFromQuickNode([ -// { -// chain: 'ethereum', -// contractAddress: '0x57f1887a8bf19b14fc0df6fd9b2acc9af147ea85', -// tokenId: '7584185289723993195313796573155465997226240911118312831760400397049224254842', -// }, -// ] as const).then((_) => console.log(JSON.stringify(_, undefined, 2))) - export async function batchFetchNFTTradingHistoryFromQuickNode< T extends { contractAddress: string @@ -285,15 +297,14 @@ export async function batchFetchNFTTradingHistoryFromQuickNode< data: Record<`_${T['contractAddress']}_${T['tokenId']}`, { nft: { tokenEvents: TokenEvent } }> }>(QUICKNODE_BASE_URL, { method: 'POST', + headers: AUTH_HEADER, body: JSON.stringify({ query }), }) - return response.data -} -interface QuickNodeResponse { - data: { - [chain in QuickNodeChain]: T | null + if (Object.hasOwn(response, 'error')) { + raise(response) } + return response.data } interface QuickNodeUserNFTs { diff --git a/src/lib/type.ts b/src/lib/type.ts index 7b051a7ad..dcc8d4b9b 100644 --- a/src/lib/type.ts +++ b/src/lib/type.ts @@ -1,3 +1,5 @@ +export type AwaitedReturnType any> = Awaited> + export type Json = string | number | boolean | null | { [key: string]: Json | undefined } | Json[] export function isNotNullish(param: T | undefined | null): param is T { diff --git a/tsconfig.json b/tsconfig.json index 5fe1ea26d..4c0447472 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -26,5 +26,5 @@ } }, "include": ["src", "scripts", "test", "*.js", "*.cjs", "*.mjs", "*.ts"], - "exclude": [".esbuild", "node_modules", "coverage", "dist"] + "exclude": [".esbuild", "node_modules", "coverage", "dist", "_"] } From 5780fd9f55998c0cdee9db2e02e3ac67d2c2dfe6 Mon Sep 17 00:00:00 2001 From: Omar Aziz Date: Wed, 2 Aug 2023 06:02:29 -0700 Subject: [PATCH 3/3] unify history format & add fallback option --- serverless.yml | 5 +- src/handlers/getNFTs.ts | 2 +- src/handlers/getNFTsHistory.ts | 115 ++++++++++++++++++++++++++++----- src/lib/nft/quick-node.ts | 46 ++++++------- src/lib/nft/reservoir.ts | 27 +++++++- 5 files changed, 154 insertions(+), 41 deletions(-) diff --git a/serverless.yml b/serverless.yml index 3498d1d9f..c5a9679da 100644 --- a/serverless.yml +++ b/serverless.yml @@ -90,7 +90,10 @@ functions: events: - httpApi: method: post - path: /nfts-history/{address} + path: /nfts/history/{address?} + - httpApi: + method: get + path: /nfts/history/{address} timeout: 29 memorySize: 1024 diff --git a/src/handlers/getNFTs.ts b/src/handlers/getNFTs.ts index 89fcbadd8..e9d0b00c1 100644 --- a/src/handlers/getNFTs.ts +++ b/src/handlers/getNFTs.ts @@ -172,7 +172,7 @@ export async function nftsHandler({ address }: { address: Address }): Promise isFulfilled(result)) as PromiseFulfilledResult< - Awaited> + AwaitedReturnType >[] ).flatMap((item) => item.value.data) diff --git a/src/handlers/getNFTsHistory.ts b/src/handlers/getNFTsHistory.ts index f6e711f2f..0487e7759 100644 --- a/src/handlers/getNFTsHistory.ts +++ b/src/handlers/getNFTsHistory.ts @@ -1,10 +1,51 @@ import { badRequest, serverError, success } from '@handlers/response' import { sliceIntoChunks } from '@lib/array' -import { fetchNFTTradingHistoryFrom } from '@lib/nft' -import type { QuickNodeChain } from '@lib/nft/quick-node' +import { fetchNFTTradingHistoryFrom, fetchUserNFTActivityFrom } from '@lib/nft' +import type { QuickNodeChain, QuickNodeTokenEvent } from '@lib/nft/quick-node' +import type { NFTActivity } from '@lib/nft/reservoir' import type { APIGatewayProxyHandler } from 'aws-lambda' import { isAddress } from 'viem' +interface NFTTokenActivity { + type: string + marketplace?: string + fromAddress: string + toAddress?: string + amount?: number + blockNumber?: number + timestamp: string + contractAddress?: string + tokenId?: string | number + transactionHash?: string +} + +async function history(address: string, body: string | null) { + try { + // try reservoir first, if it fails, try quicknode + const userNFTActivity = await fetchUserNFTActivityFrom.reservoir({ users: [address], includeMetadata: false }) + return formatNFTReservoirTokenEvents(userNFTActivity.activities) + } catch (error) { + const payload = JSON.parse(body ?? '[{}]') as Array<{ + contractAddress: string + tokenId: string + chain: QuickNodeChain + }> + if (!Array.isArray(payload)) return badRequest('Invalid body parameter, expected array') + const chunks = sliceIntoChunks(payload, 100) + const promisesResponse = await Promise.all(chunks.map((chunk) => fetchNFTTradingHistoryFrom.quickNode(chunk))) + const response = promisesResponse.flat() + + return formatNFTQuickNodeTokenEvents( + response + .map((events) => { + const [[, { nft }]] = Object.entries(events) + return nft.QuickNodeTokenEvents.edges.map((edge) => edge.node) + }) + .flat(), + ) + } +} + /** * Takes array of nft tokens (contractAddress, tokenId, chain) and returns trading history for each token */ @@ -19,20 +60,7 @@ export const handler: APIGatewayProxyHandler = async (event, context) => { return badRequest('Invalid address parameter, expected hex') } - const body = JSON.parse(event.body ?? '[{}]') as Array<{ - contractAddress: string - tokenId: string - chain: QuickNodeChain - }> - - if (!Array.isArray(body)) { - return badRequest('Invalid body parameter, expected array') - } - - const chunks = sliceIntoChunks(body, 100) - - const promisesResponse = await Promise.all(chunks.map((chunk) => fetchNFTTradingHistoryFrom.quickNode(chunk))) - const response = promisesResponse.flat() + const response = await history(address, event.body) return success(response, { maxAge: 30 * 60 }) } catch (error) { @@ -42,3 +70,58 @@ export const handler: APIGatewayProxyHandler = async (event, context) => { console.log('NFTs trading history request took', context.getRemainingTimeInMillis() / 1000, 'seconds') } } + +export function formatNFTQuickNodeTokenEvents(tokenEvents: Array): Array { + return tokenEvents.map((event) => { + const { + type, + fromAddress, + toAddress, + timestamp, + transactionHash, + blockNumber, + marketplace, + contractAddress, + tokenId, + receivedTokenId, + sentTokenId, + tokenQuantity, + sentTokenQuantity, + } = event + return { + type, + marketplace, + fromAddress, + toAddress, + amount: tokenQuantity ?? sentTokenQuantity, + blockNumber, + timestamp, + contractAddress, + tokenId: tokenId ?? sentTokenId ?? receivedTokenId, + transactionHash, + } + }) +} + +export function formatNFTReservoirTokenEvents(activities: Array): Array { + return activities.map((activity) => { + const { + type, + fromAddress, + toAddress, + timestamp, + txHash, + contract, + token: { tokenId }, + } = activity + return { + type, + fromAddress, + toAddress, + timestamp: new Date(timestamp).toISOString(), + contractAddress: contract, + tokenId, + transactionHash: txHash, + } + }) +} diff --git a/src/lib/nft/quick-node.ts b/src/lib/nft/quick-node.ts index c1ebec454..a2d0ad0b2 100644 --- a/src/lib/nft/quick-node.ts +++ b/src/lib/nft/quick-node.ts @@ -231,7 +231,7 @@ export async function batchFetchNFTTradingHistoryFromQuickNode< (item) => /* graphql */ ` _${item.contractAddress.toLowerCase()}_${item.tokenId.toLowerCase()}: ${item.chain ?? 'ethereum'} { nft(contractAddress: "${item.contractAddress}", tokenId: "${item.tokenId}") { - tokenEvents { + QuickNodeTokenEvents { totalCount edges { node { @@ -294,7 +294,7 @@ export async function batchFetchNFTTradingHistoryFromQuickNode< }` const response = await fetcher<{ - data: Record<`_${T['contractAddress']}_${T['tokenId']}`, { nft: { tokenEvents: TokenEvent } }> + data: Record<`_${T['contractAddress']}_${T['tokenId']}`, { nft: { QuickNodeTokenEvents: QuickNodeTokenEvents } }> }>(QUICKNODE_BASE_URL, { method: 'POST', headers: AUTH_HEADER, @@ -373,31 +373,33 @@ interface QuickNodeUserNFTs { } } -interface TokenEvent { +interface QuickNodeTokenEvents { totalCount: number edges: Array<{ - node: { - type: string - transactionHash: string - fromAddress: string - toAddress: string - timestamp: string - blockNumber: number - transferIndex: number - sentTokenQuantity?: number - sentTokenId?: number - receivedTokenQuantity?: string - receivedTokenId: any - receivedTokenContractAddress?: string - marketplace?: string - contractERCStandard?: string - contractAddress?: string - tokenQuantity?: number - tokenId?: number - } + node: QuickNodeTokenEvent }> } +export interface QuickNodeTokenEvent { + type: string + transactionHash: string + fromAddress: string + toAddress: string + timestamp: string + blockNumber: number + transferIndex: number + sentTokenQuantity?: number + sentTokenId?: number + receivedTokenQuantity?: string + receivedTokenId: any + receivedTokenContractAddress?: string + marketplace?: string + contractERCStandard?: string + contractAddress?: string + tokenQuantity?: number + tokenId?: number +} + interface QuickNodeNFT { name: string | null metadata: { diff --git a/src/lib/nft/reservoir.ts b/src/lib/nft/reservoir.ts index cc087dbdf..b211b7a22 100644 --- a/src/lib/nft/reservoir.ts +++ b/src/lib/nft/reservoir.ts @@ -55,8 +55,25 @@ export async function fetchUsersNFTActivity({ return response } -// https://docs.reservoir.tools/reference/getusersusercollectionsv3 +/* https://docs.reservoir.tools/reference/getapikeyskeyratelimits */ +export async function getCurrentApiRateLimit() { + const response = await fetcher<{ rateLimits: Array } | ReservoirErrorResponse>( + `${RESERVOIR_BASE_URL}/api-keys/${RESERVOIR_API_KEY}/rate-limits`, + { headers: AUTH_HEADER }, + ) + if ('error' in response) { + raise( + `[Reservoir] error for url ${RESERVOIR_BASE_URL}/api-keys/${RESERVOIR_API_KEY}/rate-limits:\n${JSON.stringify( + response, + undefined, + 2, + )}`, + ) + } + return response +} +// https://docs.reservoir.tools/reference/getusersusercollectionsv3 export async function fetchUserNFTCollections({ user, includeTopBid = true, @@ -103,6 +120,14 @@ export interface UserNFTCollection { } } +export interface ReservoirRateLimit { + route: string + method: string + allowedRequests?: number + perSeconds?: number + payload: Array +} + export interface NFTCollection { id: string slug: string