From 88cf72b993e9a07c340208b3c4852a9dc5fc0e42 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Tue, 14 Nov 2023 15:21:24 +0300 Subject: [PATCH 1/4] fix: spade oracle sync with log2size to height conversion --- package-lock.json | 8 +++++--- packages/core/package.json | 2 +- .../core/src/deal-tracker/spade-oracle-sync-tick.js | 10 +++++++++- stacks/data-stack.js | 4 ++-- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2e6ddd0..451bb7f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15172,7 +15172,7 @@ "@ucanto/server": "^9.0.1", "@ucanto/transport": "^9.0.0", "@web3-storage/capabilities": "^11.1.0", - "@web3-storage/data-segment": "^4.0.0", + "@web3-storage/data-segment": "^5.0.0", "@web3-storage/filecoin-api": "^4.0.4", "@web3-storage/filecoin-client": "^3.0.1", "fzstd": "^0.1.0", @@ -15249,8 +15249,9 @@ } }, "packages/core/node_modules/@web3-storage/data-segment": { - "version": "4.0.0", - "license": "(Apache-2.0 AND MIT)", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@web3-storage/data-segment/-/data-segment-5.0.0.tgz", + "integrity": "sha512-5CbElsxec2DsKhEHEh3XRGISAyna+bCjKjjvFrLcYyXLCaiSt/nF3ypcllxwjpE4newMUArymGKGzzZnRWL2kg==", "dependencies": { "@ipld/dag-cbor": "^9.0.5", "multiformats": "^11.0.2", @@ -15413,6 +15414,7 @@ } }, "packages/tools": { + "name": "@w3filecoin/tools", "version": "0.0.0", "devDependencies": { "@ipld/dag-ucan": "3.4.0", diff --git a/packages/core/package.json b/packages/core/package.json index 4a1cf24..53cc3c5 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -22,7 +22,7 @@ "@ucanto/server": "^9.0.1", "@ucanto/transport": "^9.0.0", "@web3-storage/capabilities": "^11.1.0", - "@web3-storage/data-segment": "^4.0.0", + "@web3-storage/data-segment": "^5.0.0", "@web3-storage/filecoin-api": "^4.0.4", "@web3-storage/filecoin-client": "^3.0.1", "fzstd": "^0.1.0", diff --git a/packages/core/src/deal-tracker/spade-oracle-sync-tick.js b/packages/core/src/deal-tracker/spade-oracle-sync-tick.js index 1ac5f5d..39a213c 100644 --- a/packages/core/src/deal-tracker/spade-oracle-sync-tick.js +++ b/packages/core/src/deal-tracker/spade-oracle-sync-tick.js @@ -210,7 +210,7 @@ async function fetchLatestDealArchive (spadeOracleUrl) { // Convert PieceCidV1 to PieceCidV2 const pieceCid = convertPieceCidV1toPieceCidV2( parseLink(replica.piece_cid), - replica.piece_log2_size + log2PieceSizeToHeight(replica.piece_log2_size) ) dealMap.set(pieceCid.toString(), replica.contracts.map(c => ({ provider: c.provider_id, @@ -239,3 +239,11 @@ export function convertPieceCidV1toPieceCidV2 (link, height) { return piece.link } + +/** + * + * @param {number} log2Size + */ +function log2PieceSizeToHeight (log2Size) { + return Piece.Size.Expanded.toHeight(BigInt(log2Size)) +} \ No newline at end of file diff --git a/stacks/data-stack.js b/stacks/data-stack.js index 0399c39..e2c692a 100644 --- a/stacks/data-stack.js +++ b/stacks/data-stack.js @@ -112,7 +112,7 @@ export function DataStack({ stack, app }) { /** * Deal archive store used to store active replicas reported by Spade Oracle. */ - const dealTrackerDealArchiveBucket = getBucketConfig('deal-tracker-deal-archive-store', stack.stage) + const dealTrackerDealArchiveBucket = getBucketConfig('deal-tracker-deal-archive-store', stack.stage, 1) const dealTrackerDealArchiveStoreBucket = new Bucket(stack, dealTrackerDealArchiveBucket.bucketName, { cors: true, cdk: { @@ -123,7 +123,7 @@ export function DataStack({ stack, app }) { /** * Deal store used to store deal information */ - const dealTrackerDealStoreTableName = 'deal-tracker-deal-store' + const dealTrackerDealStoreTableName = 'deal-tracker-deal-store-v1' const dealTrackerDealStoreTable = new Table(stack, dealTrackerDealStoreTableName, { ...dealStoreTableProps, // information that will be written to the stream From a5a9e6498c95fef42547bdd0cdad8c6ff6b9016f Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Tue, 14 Nov 2023 15:31:13 +0300 Subject: [PATCH 2/4] fix: update integration test tracker table name --- test/helpers/deployment.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/helpers/deployment.js b/test/helpers/deployment.js index fdad0f4..78a9772 100644 --- a/test/helpers/deployment.js +++ b/test/helpers/deployment.js @@ -115,7 +115,7 @@ export const getStoreClients = () => { }, tracker: { dealStore: createDealTrackerDealStoreClient(dynamoClient, - { tableName: getTableName(`deal-tracker-deal-store`) } + { tableName: getTableName(`deal-tracker-deal-store-v1`) } ) } } From 1b7cb9a7b6c7d31c01fe54f2205848436ed77d24 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Tue, 14 Nov 2023 16:23:07 +0300 Subject: [PATCH 3/4] fix: integration tests --- .../functions/src/deal-tracker-api/ucan-invocation-router.js | 2 +- packages/functions/src/deal-tracker/spade-oracle-sync-tick.js | 2 +- packages/functions/src/types.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/functions/src/deal-tracker-api/ucan-invocation-router.js b/packages/functions/src/deal-tracker-api/ucan-invocation-router.js index 5a2ae13..2c98e82 100644 --- a/packages/functions/src/deal-tracker-api/ucan-invocation-router.js +++ b/packages/functions/src/deal-tracker-api/ucan-invocation-router.js @@ -102,7 +102,7 @@ function getLambdaEnv () { return { did: mustGetEnv('DID'), ucanLogUrl: mustGetEnv('UCAN_LOG_URL'), - dealStoreTableName: Table['deal-tracker-deal-store'], + dealStoreTableName: Table['deal-tracker-deal-store-v1'], dealStoreTableRegion: mustGetEnv('AWS_REGION'), } } diff --git a/packages/functions/src/deal-tracker/spade-oracle-sync-tick.js b/packages/functions/src/deal-tracker/spade-oracle-sync-tick.js index 4869dd4..468058a 100644 --- a/packages/functions/src/deal-tracker/spade-oracle-sync-tick.js +++ b/packages/functions/src/deal-tracker/spade-oracle-sync-tick.js @@ -59,7 +59,7 @@ function getLambdaEnv () { return { dealArchiveStoreBucketName: mustGetEnv('DEAL_ARCHIVE_STORE_BUCKET_NAME'), dealArchiveStoreBucketRegion: mustGetEnv('DEAL_ARCHIVE_STORE_REGION'), - dealStoreTableName: Table['deal-tracker-deal-store'], + dealStoreTableName: Table['deal-tracker-deal-store-v1'], dealStoreTableRegion: mustGetEnv('AWS_REGION'), spadeOracleUrl: mustGetEnv('SPADE_ORACLE_URL'), } diff --git a/packages/functions/src/types.ts b/packages/functions/src/types.ts index 05f501f..0e97774 100644 --- a/packages/functions/src/types.ts +++ b/packages/functions/src/types.ts @@ -24,7 +24,7 @@ declare module 'sst/node/table' { 'aggregator-inclusion-store': { tableName: string; }; - 'deal-tracker-deal-store': { + 'deal-tracker-deal-store-v1': { tableName: string; }; 'dealer-aggregate-store': { From 8129bf99141e1285b9f9e2e15c28dafe1301fb37 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Tue, 14 Nov 2023 16:44:07 +0300 Subject: [PATCH 4/4] fix: address review --- packages/core/src/deal-tracker/spade-oracle-sync-tick.js | 4 ++-- packages/core/test/deal-tracker/spade-oracle-sync.test.js | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/core/src/deal-tracker/spade-oracle-sync-tick.js b/packages/core/src/deal-tracker/spade-oracle-sync-tick.js index 39a213c..9b11f92 100644 --- a/packages/core/src/deal-tracker/spade-oracle-sync-tick.js +++ b/packages/core/src/deal-tracker/spade-oracle-sync-tick.js @@ -244,6 +244,6 @@ export function convertPieceCidV1toPieceCidV2 (link, height) { * * @param {number} log2Size */ -function log2PieceSizeToHeight (log2Size) { - return Piece.Size.Expanded.toHeight(BigInt(log2Size)) +export function log2PieceSizeToHeight (log2Size) { + return Piece.Size.Expanded.toHeight(2n ** BigInt(log2Size)) } \ No newline at end of file diff --git a/packages/core/test/deal-tracker/spade-oracle-sync.test.js b/packages/core/test/deal-tracker/spade-oracle-sync.test.js index d979b51..6cfd2ea 100644 --- a/packages/core/test/deal-tracker/spade-oracle-sync.test.js +++ b/packages/core/test/deal-tracker/spade-oracle-sync.test.js @@ -206,6 +206,10 @@ test('converts PieceCidV1 to PieceCidV2', t => { t.falsy(pieceCidV1.equals(pieceCidV2)) }) +test('converts log2pieceSize to height', t => { + const height = spadeOracleSyncTick.log2PieceSizeToHeight(35) + t.is(height, 30) +}) /** * @param {string} source */