From ba2ae4fb37863b6a6d95e2bbcb8cff3590682214 Mon Sep 17 00:00:00 2001 From: wujunze Date: Wed, 9 Aug 2023 17:14:26 +0800 Subject: [PATCH 1/5] feat: ignore ide cache and vendor --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index d8c8dcc..c9193d7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ main.go go.sum test-keyfile.json -.DS_Store \ No newline at end of file +.DS_Store +.idea +vendor \ No newline at end of file From 160ca567c5068f8ecf780ffa204480e042d32fc2 Mon Sep 17 00:00:00 2001 From: wujunze Date: Wed, 9 Aug 2023 17:14:58 +0800 Subject: [PATCH 2/5] feat: add func ContainsInSlice --- utils/slice.go | 11 +++++++++++ utils/slice_test.go | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 utils/slice.go create mode 100644 utils/slice_test.go diff --git a/utils/slice.go b/utils/slice.go new file mode 100644 index 0000000..0e3d7f9 --- /dev/null +++ b/utils/slice.go @@ -0,0 +1,11 @@ +package utils + +// ContainsInSlice checks if a string is in a slice of strings +func ContainsInSlice(items []string, item string) bool { + for _, eachItem := range items { + if eachItem == item { + return true + } + } + return false +} diff --git a/utils/slice_test.go b/utils/slice_test.go new file mode 100644 index 0000000..54af0fb --- /dev/null +++ b/utils/slice_test.go @@ -0,0 +1,40 @@ +package utils + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestContainsInSlice(t *testing.T) { + type args struct { + items []string + item string + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "string in slice", + args: args{ + items: []string{"a", "b", "c"}, + item: "a", + }, + want: true, + }, + { + name: "string not in slice", + args: args{ + items: []string{"a", "b", "c"}, + item: "x", + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, tt.want, ContainsInSlice(tt.args.items, tt.args.item), "ContainsInSlice(%v, %v)", tt.args.items, tt.args.item) + }) + } +} From fcafadb2b7e9d6bf085b3786d25a66b9ad80891b Mon Sep 17 00:00:00 2001 From: wujunze Date: Wed, 9 Aug 2023 17:19:20 +0800 Subject: [PATCH 3/5] feat: add GetBundleItems --- client.go | 89 ++++++++++++++++++++++++++++++++++++++++++++++++++ client_test.go | 44 +++++++++++++++++++++++++ 2 files changed, 133 insertions(+) diff --git a/client.go b/client.go index 608bd7a..cd3112d 100644 --- a/client.go +++ b/client.go @@ -10,6 +10,7 @@ import ( "github.com/tidwall/gjson" "gopkg.in/h2non/gentleman.v2" "io/ioutil" + "math" "math/big" "net/http" "net/url" @@ -1052,3 +1053,91 @@ func (c *Client) SubmitToWarp(tx *types.Transaction) ([]byte, error) { defer resp.Body.Close() return ioutil.ReadAll(resp.Body) } + +/** + ++------------------+-----------------------------+--------------------------------------------+ +| Number of Items | Headers (64 bytes) | Items' Binary Data | +| 32 bytes | 32 bytes (length) + 32 bytes | | +| | (ID) for each item | | ++------------------+-----------------------------+--------------------------------------------+ +*/ + +func (c *Client) GetBundleItems(bundleInId string, itemsIds []string) (items []*types.BundleItem, err error) { + offset, err := c.getTransactionOffset(bundleInId) + if err != nil { + return nil, err + } + + size, err := strconv.ParseInt(offset.Size, 10, 64) + if err != nil { + return nil, err + } + endOffset, err := strconv.ParseInt(offset.Offset, 10, 64) + startOffset := endOffset - size + 1 + + firstChunk, err := c.getChunkData(startOffset) + if err != nil { + return nil, err + } + + itemsNum := utils.ByteArrayToLong(firstChunk[:32]) + + // get Headers endOffset + bundleItemStart := 32 + itemsNum*64 + var containHeadersChunks []byte + if len(firstChunk) < bundleItemStart { + // To calculate headers, you need to pull several chunks and fetch an integer upwards + chunkNum := int(math.Ceil(float64(bundleItemStart) / float64(types.MAX_CHUNK_SIZE))) + + for i := 0; i < chunkNum; i++ { + chunk, err := c.getChunkData(startOffset + int64(i*types.MAX_CHUNK_SIZE)) + if err != nil { + return nil, err + } + containHeadersChunks = append(containHeadersChunks, chunk...) + } + } else { + containHeadersChunks = firstChunk + } + + for i := 0; i < itemsNum; i++ { + headerBegin := 32 + i*64 + end := headerBegin + 64 + + headerByte := containHeadersChunks[headerBegin:end] + itemBinaryLength := utils.ByteArrayToLong(headerByte[:32]) + id := utils.Base64Encode(headerByte[32:64]) + + // if item is in itemsIds + if utils.ContainsInSlice(itemsIds, id) { + + startChunkNum := bundleItemStart / types.MAX_CHUNK_SIZE + startChunkOffset := bundleItemStart % types.MAX_CHUNK_SIZE + data := make([]byte, 0, itemBinaryLength) + + for offset := startOffset + int64(startChunkNum*types.MAX_CHUNK_SIZE); offset <= startOffset+int64(bundleItemStart+itemBinaryLength); { + chunk, err := c.getChunkData(offset) + + if err != nil { + return nil, err + } + data = append(data, chunk...) + offset += int64(len(chunk)) + } + + itemData := data[startChunkOffset : startChunkOffset+itemBinaryLength] + item, err := utils.DecodeBundleItem(itemData) + if err != nil { + return nil, err + } + + items = append(items, item) + } + // next itemBy start offset + bundleItemStart += itemBinaryLength + } + + return items, nil + +} diff --git a/client_test.go b/client_test.go index fa9cb86..db6879e 100644 --- a/client_test.go +++ b/client_test.go @@ -348,3 +348,47 @@ func TestNewTempConn2(t *testing.T) { err = utils.VerifyBundleItem(*item) assert.NoError(t, err) } + +// https://arweave.net/tx/x-q8ibbTfXIcdDXqQ3xaPD3PuShj832G_xzNT5QrVjY/offset +// {"size":"753","offset":"146739359163367"} +func Test_getChunkData(t *testing.T) { + c := NewClient("https://arweave.net") + data, err := c.getChunkData(146739359163367) + assert.NoError(t, err) + + t.Log(string(data)) + +} + +func TestClient_GetBundleItems(t *testing.T) { + c := NewClient("https://arweave.net") + itemsIds := []string{"UCTEOaljmuutGJId-ktPY_q_Gbal8tyJuLfyR6BeaGw"} + items, err := c.GetBundleItems("47KozLIAfVMKdxq1q3D1xFZmRpkahOOBQ8boOjSydnQ", itemsIds) + assert.NoError(t, err) + assert.Equal(t, 1, len(items)) + assert.Equal(t, "UCTEOaljmuutGJId-ktPY_q_Gbal8tyJuLfyR6BeaGw", items[0].Id) +} + +func TestClient_GetBundleItems2(t *testing.T) { + c := NewClient("https://arweave.net") + itemsIds := []string{"UCTEOaljmuutGJId-ktPY_q_Gbal8tyJuLfyR6BeaGw", "FCUfgEEPmZB3YQMTfbwYl6VA-JT54zLr5PrcJw2EFeM", "zlU0o99c81n0CP64F31ANpyJeOtlz5DKvsKohmbMxqU"} + items, err := c.GetBundleItems("47KozLIAfVMKdxq1q3D1xFZmRpkahOOBQ8boOjSydnQ", itemsIds) + + assert.NoError(t, err) + assert.Equal(t, 3, len(items)) + assert.Equal(t, "UCTEOaljmuutGJId-ktPY_q_Gbal8tyJuLfyR6BeaGw", items[2].Id) + assert.Equal(t, "FCUfgEEPmZB3YQMTfbwYl6VA-JT54zLr5PrcJw2EFeM", items[1].Id) + assert.Equal(t, "zlU0o99c81n0CP64F31ANpyJeOtlz5DKvsKohmbMxqU", items[0].Id) +} + +func TestClient_GetBundleItems3(t *testing.T) { + c := NewClient("https://arweave.net") + // UCTEOaljmuutGJId-ktPY_q_Gbal8tyJuLfyR6BeaGx not in bundle + itemsIds := []string{"UCTEOaljmuutGJId-ktPY_q_Gbal8tyJuLfyR6BeaGx", "FCUfgEEPmZB3YQMTfbwYl6VA-JT54zLr5PrcJw2EFeM", "zlU0o99c81n0CP64F31ANpyJeOtlz5DKvsKohmbMxqU"} + items, err := c.GetBundleItems("47KozLIAfVMKdxq1q3D1xFZmRpkahOOBQ8boOjSydnQ", itemsIds) + + assert.NoError(t, err) + assert.Equal(t, 2, len(items)) + assert.Equal(t, "FCUfgEEPmZB3YQMTfbwYl6VA-JT54zLr5PrcJw2EFeM", items[1].Id) + assert.Equal(t, "zlU0o99c81n0CP64F31ANpyJeOtlz5DKvsKohmbMxqU", items[0].Id) +} From ba203125c974f1f3b149d143ca4c6de4dd860377 Mon Sep 17 00:00:00 2001 From: wujunze Date: Wed, 9 Aug 2023 17:51:40 +0800 Subject: [PATCH 4/5] feat: test 5000 items bundle --- client_test.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/client_test.go b/client_test.go index db6879e..78672fc 100644 --- a/client_test.go +++ b/client_test.go @@ -381,14 +381,16 @@ func TestClient_GetBundleItems2(t *testing.T) { assert.Equal(t, "zlU0o99c81n0CP64F31ANpyJeOtlz5DKvsKohmbMxqU", items[0].Id) } +// https://viewblock.io/zh-CN/arweave/tx/PRBVxEX00aVMN59EY8gznt83FTlGXZvESwv1WTP7ReQ 5000 items func TestClient_GetBundleItems3(t *testing.T) { c := NewClient("https://arweave.net") - // UCTEOaljmuutGJId-ktPY_q_Gbal8tyJuLfyR6BeaGx not in bundle - itemsIds := []string{"UCTEOaljmuutGJId-ktPY_q_Gbal8tyJuLfyR6BeaGx", "FCUfgEEPmZB3YQMTfbwYl6VA-JT54zLr5PrcJw2EFeM", "zlU0o99c81n0CP64F31ANpyJeOtlz5DKvsKohmbMxqU"} - items, err := c.GetBundleItems("47KozLIAfVMKdxq1q3D1xFZmRpkahOOBQ8boOjSydnQ", itemsIds) + itemsIds := []string{"QD0ryQTy4CBr7kluWRLT1strRcXWJOgUUoIYat4lk1s", "BzsIVzo6rPfGQg0PP-5Y_HErPey51_it0d6aGIUfQnY", "fy3aOYoRf7OzCEd9_WrD-RfqbzNZ1LsJ4PKIIGUALik"} + items, err := c.GetBundleItems("PRBVxEX00aVMN59EY8gznt83FTlGXZvESwv1WTP7ReQ", itemsIds) assert.NoError(t, err) - assert.Equal(t, 2, len(items)) - assert.Equal(t, "FCUfgEEPmZB3YQMTfbwYl6VA-JT54zLr5PrcJw2EFeM", items[1].Id) - assert.Equal(t, "zlU0o99c81n0CP64F31ANpyJeOtlz5DKvsKohmbMxqU", items[0].Id) + assert.Equal(t, 3, len(items)) + assert.Equal(t, "QD0ryQTy4CBr7kluWRLT1strRcXWJOgUUoIYat4lk1s", items[0].Id) + assert.Equal(t, "fy3aOYoRf7OzCEd9_WrD-RfqbzNZ1LsJ4PKIIGUALik", items[1].Id) + assert.Equal(t, "BzsIVzo6rPfGQg0PP-5Y_HErPey51_it0d6aGIUfQnY", items[2].Id) + } From 20594510ab129318ee14c11e8dd59235d2fa3726 Mon Sep 17 00:00:00 2001 From: wujunze Date: Wed, 9 Aug 2023 18:02:41 +0800 Subject: [PATCH 5/5] feat: add doc --- client.go | 1 + 1 file changed, 1 insertion(+) diff --git a/client.go b/client.go index cd3112d..6d6e79a 100644 --- a/client.go +++ b/client.go @@ -1056,6 +1056,7 @@ func (c *Client) SubmitToWarp(tx *types.Transaction) ([]byte, error) { /** + bundleBinary Data Format +------------------+-----------------------------+--------------------------------------------+ | Number of Items | Headers (64 bytes) | Items' Binary Data | | 32 bytes | 32 bytes (length) + 32 bytes | |