Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: remove pLimiter code repetition #265

Closed
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/events.ts
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ export const onBlock = async (

// fetch txs that include client's address with their utxo data
const txs = await getTransactionsWithDetails(
Object.entries(txsCbor).map(([txid, cbor]) => ({ txid, cbor })),
Object.entries(txsCbor).map(([txId, cbor]) => ({ txId, cbor })),
);

const notifications: TxNotification[] = [];
Expand Down
2 changes: 1 addition & 1 deletion src/methods/get-account-info.ts
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ export const getAccountInfo = async (
} else if (details === 'txs') {
// fetch full transaction objects and set account.history.transactions
const txs = await txIdsToTransactions(
requestedPageTxIds.map(item => ({ address: item.address, data: [item.tx_hash] })),
requestedPageTxIds.map(item => ({ address: item.address, txIds: [item.tx_hash] })),
);

accountInfo.history.transactions = txs;
Expand Down
2 changes: 1 addition & 1 deletion src/methods/get-balance-history.ts
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ export const getAccountBalanceHistory = async (
await txIdsToTransactions(
txIds.map(tx => ({
address: tx.address,
data: [tx.tx_hash],
txIds: [tx.tx_hash],
})),
)
)
Expand Down
112 changes: 51 additions & 61 deletions src/utils/address.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import {
} from '@blockfrost/blockfrost-js';
import memoizee from 'memoizee';
import { getAssetData, transformAsset } from './asset.js';
import { assetMetadataLimiter, pLimiter } from './limiter.js';
import { assetMetadataLimiter, limiter } from './limiter.js';

export const deriveAddress = (
publicKey: string,
Expand Down Expand Up @@ -69,9 +69,7 @@ export const discoverAddresses = async (
);

addressCount++;
const promise = pLimiter.add(() => blockfrostAPI.addresses(address), {
throwOnTimeout: true,
});
const promise = limiter(() => blockfrostAPI.addresses(address));

promisesBundle.push({ address, promise, path });
}
Expand Down Expand Up @@ -113,17 +111,15 @@ export const addressesToUtxos = async (
const promises = addresses.map(item =>
item.data === 'empty'
? []
: pLimiter.add(
() =>
// change batchSize to fetch only 1 page at a time (each page has 100 utxos)
blockfrostAPI.addressesUtxosAll(item.address, { batchSize: 1 }).catch(error => {
if (error instanceof BlockfrostServerError && error.status_code === 404) {
return [];
} else {
throw error;
}
}),
{ throwOnTimeout: true },
: limiter(() =>
// change batchSize to fetch only 1 page at a time (each page has 100 utxos)
blockfrostAPI.addressesUtxosAll(item.address, { batchSize: 1 }).catch(error => {
if (error instanceof BlockfrostServerError && error.status_code === 404) {
return [];
} else {
throw error;
}
}),
),
);

Expand Down Expand Up @@ -167,15 +163,13 @@ export const utxosWithBlocks = async (
if (utxo.data === 'empty') continue;

for (const utxoData of utxo.data) {
const promise = pLimiter.add(
() =>
blockfrostAPI.blocks(utxoData.block).then(blockData => ({
address: utxo.address,
path: utxo.path,
utxoData: utxoData,
blockInfo: blockData,
})),
{ throwOnTimeout: true },
const promise = limiter(() =>
blockfrostAPI.blocks(utxoData.block).then(blockData => ({
address: utxo.address,
path: utxo.path,
utxoData: utxoData,
blockInfo: blockData,
})),
);

promisesBundle.push(promise);
Expand All @@ -198,27 +192,25 @@ export const addressesToTxIds = async (
for (const item of addresses) {
if (item.data === 'empty') continue;

const promise = pLimiter.add(
() =>
// 1 page (100 txs) per address at a time should be more efficient default value
// compared to fetching 10 pages (1000 txs) per address
blockfrostAPI
.addressesTransactionsAll(item.address, { batchSize: 1 })
.then(data => ({
address: item.address,
data,
}))
.catch(error => {
if (error instanceof BlockfrostServerError && error.status_code === 404) {
return {
address: item.address,
data: [],
};
} else {
throw error;
}
}),
{ throwOnTimeout: true },
const promise = limiter(() =>
// 1 page (100 txs) per address at a time should be more efficient default value
// compared to fetching 10 pages (1000 txs) per address
blockfrostAPI
.addressesTransactionsAll(item.address, { batchSize: 1 })
.then(data => ({
address: item.address,
data,
}))
.catch(error => {
if (error instanceof BlockfrostServerError && error.status_code === 404) {
return {
address: item.address,
data: [],
};
} else {
throw error;
}
}),
);

promisesBundle.push(promise);
Expand All @@ -244,22 +236,20 @@ export const getAddressesData = async (
}

const promises = addresses.map(addr =>
pLimiter.add(
() =>
blockfrostAPI.addressesTotal(addr.address).catch(error => {
if (error.status_code === 404) {
return {
address: addr.address,
path: addr.path,
tx_count: 0,
received_sum: [{ unit: 'lovelace', quantity: '0' }],
sent_sum: [{ unit: 'lovelace', quantity: '0' }],
};
} else {
throw new Error(error);
}
}),
{ throwOnTimeout: true },
limiter(() =>
blockfrostAPI.addressesTotal(addr.address).catch(error => {
if (error.status_code === 404) {
return {
address: addr.address,
path: addr.path,
tx_count: 0,
received_sum: [{ unit: 'lovelace', quantity: '0' }],
sent_sum: [{ unit: 'lovelace', quantity: '0' }],
};
} else {
throw new Error(error);
}
}),
),
);
const responses = await Promise.all(promises);
Expand Down
3 changes: 3 additions & 0 deletions src/utils/limiter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,3 +28,6 @@ pLimiter.on('error', error => {
ratesLimiter.on('error', error => {
logger.warn(`ratesLimiter error`, error);
});

export const limiter = <T>(task: () => PromiseLike<T>) =>
pLimiter.add<T>(task, { throwOnTimeout: true });
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

please make sure that the passed options are still configurable

38 changes: 12 additions & 26 deletions src/utils/transaction.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import * as Types from '../types/transactions.js';
import { TxIdsToTransactionsResponse } from '../types/transactions.js';
import { blockfrostAPI } from '../utils/blockfrost-api.js';
import { getAssetData, transformAsset } from './asset.js';
import { assetMetadataLimiter, pLimiter } from './limiter.js';
import { assetMetadataLimiter, limiter } from './limiter.js';
import { logger } from './logger.js';

export const sortTransactionsCmp = <
Expand All @@ -23,12 +23,7 @@ const fetchTxWithUtxo = async (txHash: string, address: string) => {
const txData = await transformTransactionData(tx);
const txUtxos = await transformTransactionUtxo(txUtxo);

return {
txData,
txUtxos,
address: address,
txHash: txHash,
};
return { txData, txUtxos, address, txHash };
} catch (error) {
// WARNING: this will omit txs that returned 404, caller should be well aware of this fact
if (error instanceof BlockfrostServerError && error.status_code === 404) {
Expand All @@ -43,18 +38,16 @@ const fetchTxWithUtxo = async (txHash: string, address: string) => {
export const txIdsToTransactions = async (
txidsPerAddress: {
address: string;
data: string[];
txIds: string[];
}[],
): Promise<Types.TxIdsToTransactionsResponse[]> => {
if (txidsPerAddress.length === 0) return [];

const promises: Promise<Types.TxIdsToTransactionsResponse | undefined>[] = [];

for (const item of txidsPerAddress) {
for (const tx of item.data) {
promises.push(
pLimiter.add(() => fetchTxWithUtxo(tx, item.address), { throwOnTimeout: true }),
);
for (const txId of item.txIds) {
promises.push(limiter(() => fetchTxWithUtxo(txId, item.address)));
}
}

Expand All @@ -69,34 +62,27 @@ export const txIdsToTransactions = async (
};

export interface GetTransactionsDetails {
txid: string;
txId: string;
cbor?: boolean;
}

export const getTransactionsWithDetails = async (
txs: GetTransactionsDetails[],
): Promise<Pick<TxIdsToTransactionsResponse, 'txData' | 'txUtxos' | 'txCbor'>[]> => {
const txsData = await Promise.all(
txs.map(({ txid }) =>
pLimiter.add(() => blockfrostAPI.txs(txid).then(data => transformTransactionData(data)), {
throwOnTimeout: true,
}),
txs.map(({ txId }) =>
limiter(() => blockfrostAPI.txs(txId).then(data => transformTransactionData(data))),
),
);
const txsUtxo = await Promise.all(
txs.map(({ txid }) =>
pLimiter.add(
() => blockfrostAPI.txsUtxos(txid).then(data => transformTransactionUtxo(data)),
{ throwOnTimeout: true },
),
txs.map(({ txId }) =>
limiter(() => blockfrostAPI.txsUtxos(txId).then(data => transformTransactionUtxo(data))),
),
);
const txsCbors = await Promise.all(
txs.map(({ txid, cbor }) =>
txs.map(({ txId, cbor }) =>
cbor
? pLimiter.add(() => blockfrostAPI.txsCbor(txid).then(data => data.cbor), {
throwOnTimeout: true,
})
? limiter(() => blockfrostAPI.txsCbor(txId).then(data => data.cbor))
: // eslint-disable-next-line unicorn/no-useless-undefined
Promise.resolve<undefined>(undefined),
),
Expand Down
2 changes: 1 addition & 1 deletion test/unit/tests/utils/events.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ describe('events', () => {
return new Promise(resolve => {
// sanity check that the test really wanted to fetch transactions that we expected
for (const mockedTx of fixture.mocks.txsWithUtxo) {
if (!txs.find(({ txid }) => mockedTx.txData.hash === txid)) {
if (!txs.find(({ txId }) => mockedTx.txData.hash === txId)) {
throw new Error('Unexpected list of affected addresses');
}
}
Expand Down
Loading