From 831310777a3f1c9d03c74fd8cb91b87cbee7f560 Mon Sep 17 00:00:00 2001 From: Evgenii Baidakov Date: Mon, 12 Feb 2024 12:02:16 +0400 Subject: [PATCH 1/3] multipart: Sort parts by number and server creation time Closes #901. Signed-off-by: Evgenii Baidakov --- api/data/tree.go | 5 ++++- api/layer/multipart_upload.go | 18 ++++++++++++++++++ go.mod | 2 +- internal/neofs/tree.go | 18 +++++++++++++----- 4 files changed, 36 insertions(+), 7 deletions(-) diff --git a/api/data/tree.go b/api/data/tree.go index 801040b4b..7809c98e4 100644 --- a/api/data/tree.go +++ b/api/data/tree.go @@ -85,7 +85,10 @@ type PartInfo struct { OID oid.ID Size int64 ETag string - Created time.Time + // Creation time from the client. + Created time.Time + // Server creation time. + ServerCreated time.Time } // ToHeaderString form short part representation to use in S3-Completed-Parts header. diff --git a/api/layer/multipart_upload.go b/api/layer/multipart_upload.go index 601306086..7a51aa925 100644 --- a/api/layer/multipart_upload.go +++ b/api/layer/multipart_upload.go @@ -19,6 +19,7 @@ import ( oid "github.com/nspcc-dev/neofs-sdk-go/object/id" "github.com/nspcc-dev/neofs-sdk-go/user" "go.uber.org/zap" + "golang.org/x/exp/slices" ) const ( @@ -616,6 +617,23 @@ func (n *layer) getUploadParts(ctx context.Context, p *UploadInfoParams) (*data. return nil, nil, err } + // Sort parts by part number, then by server creation time to make actual last uploaded parts with the same number. + slices.SortFunc(parts, func(a, b *data.PartInfo) int { + if a.Number < b.Number { + return -1 + } + + if a.ServerCreated.Before(b.ServerCreated) { + return -1 + } + + if a.ServerCreated.Equal(b.ServerCreated) { + return 0 + } + + return 1 + }) + res := make(map[int]*data.PartInfo, len(parts)) partsNumbers := make([]int, len(parts)) oids := make([]string, len(parts)) diff --git a/go.mod b/go.mod index 49645d20a..3ea16e8d7 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/urfave/cli/v2 v2.3.0 go.uber.org/zap v1.26.0 golang.org/x/crypto v0.17.0 + golang.org/x/exp v0.0.0-20231006140011-7918f672742d google.golang.org/grpc v1.59.0 google.golang.org/protobuf v1.31.0 ) @@ -34,7 +35,6 @@ require ( github.com/nspcc-dev/go-ordered-json v0.0.0-20231123160306-3374ff1e7a3c // indirect github.com/nspcc-dev/hrw/v2 v2.0.0-20231115095647-bf62f4ad0a43 // indirect github.com/twmb/murmur3 v1.1.8 // indirect - golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect golang.org/x/sync v0.3.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect ) diff --git a/internal/neofs/tree.go b/internal/neofs/tree.go index 914c6ebff..5534daff9 100644 --- a/internal/neofs/tree.go +++ b/internal/neofs/tree.go @@ -70,6 +70,7 @@ const ( isDeleteMarkerKV = "IsDeleteMarker" ownerKV = "Owner" createdKV = "Created" + serverCreatedKV = "SrvCreated" settingsFileName = "bucket-settings" notifConfFileName = "bucket-notifications" @@ -259,6 +260,12 @@ func newPartInfo(node NodeResponse) (*data.PartInfo, error) { return nil, fmt.Errorf("invalid created timestamp: %w", err) } partInfo.Created = time.UnixMilli(utcMilli) + case serverCreatedKV: + var utcMilli int64 + if utcMilli, err = strconv.ParseInt(value, 10, 64); err != nil { + return nil, fmt.Errorf("invalid server created timestamp: %w", err) + } + partInfo.ServerCreated = time.UnixMilli(utcMilli) } } @@ -903,11 +910,12 @@ func (c *TreeClient) AddPart(ctx context.Context, bktInfo *data.BucketInfo, mult } meta := map[string]string{ - partNumberKV: strconv.Itoa(info.Number), - oidKV: info.OID.EncodeToString(), - sizeKV: strconv.FormatInt(info.Size, 10), - createdKV: strconv.FormatInt(info.Created.UTC().UnixMilli(), 10), - etagKV: info.ETag, + partNumberKV: strconv.Itoa(info.Number), + oidKV: info.OID.EncodeToString(), + sizeKV: strconv.FormatInt(info.Size, 10), + createdKV: strconv.FormatInt(info.Created.UTC().UnixMilli(), 10), + serverCreatedKV: strconv.FormatInt(time.Now().UTC().UnixMilli(), 10), + etagKV: info.ETag, } var foundPartID uint64 From f926a6fe441ae1abcf6b5807d255de472684db21 Mon Sep 17 00:00:00 2001 From: Evgenii Baidakov Date: Mon, 12 Feb 2024 13:47:04 +0400 Subject: [PATCH 2/3] linter: Fix revive - seems to be unused Signed-off-by: Evgenii Baidakov --- cmd/s3-authmate/main.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/s3-authmate/main.go b/cmd/s3-authmate/main.go index 10538de15..478b095e4 100644 --- a/cmd/s3-authmate/main.go +++ b/cmd/s3-authmate/main.go @@ -301,7 +301,7 @@ It will be ceil rounded to the nearest amount of epoch.`, Destination: &slicerEnabledFlag, }, }, - Action: func(c *cli.Context) error { + Action: func(_ *cli.Context) error { ctx, log := prepare() password := wallet.GetPassword(viper.GetViper(), envWalletPassphrase) @@ -465,7 +465,7 @@ It will be ceil rounded to the nearest amount of epoch.`, Destination: &secretAccessKeyFlag, }, }, - Action: func(c *cli.Context) error { + Action: func(_ *cli.Context) error { var cfg aws.Config if regionFlag != "" { cfg.Region = ®ionFlag @@ -646,7 +646,7 @@ func obtainSecret() *cli.Command { Value: poolStreamTimeout, }, }, - Action: func(c *cli.Context) error { + Action: func(_ *cli.Context) error { ctx, log := prepare() password := wallet.GetPassword(viper.GetViper(), envWalletPassphrase) From 72d5830a1d63568160089c972ed2c108225f1e13 Mon Sep 17 00:00:00 2001 From: Evgenii Baidakov Date: Mon, 12 Feb 2024 13:47:29 +0400 Subject: [PATCH 3/3] workflow: Update linter flow Signed-off-by: Evgenii Baidakov --- .github/workflows/tests.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 721c5582f..d556c663e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,11 +17,17 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Set up Go + uses: actions/setup-go@v4 + with: + cache: true + go-version: '1.21' + - name: Get tree-service client run: make sync-tree - name: golangci-lint - uses: golangci/golangci-lint-action@v2 + uses: golangci/golangci-lint-action@v3 with: version: latest