From 3098b335f78fbad4785a0eaea473b15f0dcc76b2 Mon Sep 17 00:00:00 2001 From: Evgenii Baidakov Date: Wed, 22 Jan 2025 11:03:00 +0400 Subject: [PATCH] neofs: Deduplicate object attributes Closes #1054. Signed-off-by: Evgenii Baidakov --- api/layer/multipart_upload.go | 18 ++++++------- api/layer/neofs.go | 2 +- api/layer/neofs_mock.go | 4 +-- api/layer/object.go | 17 +++++-------- api/layer/system_object.go | 10 +++----- internal/neofs/neofs.go | 48 +++++++++++++++-------------------- 6 files changed, 43 insertions(+), 56 deletions(-) diff --git a/api/layer/multipart_upload.go b/api/layer/multipart_upload.go index 8cea1e32d..8d5414fa8 100644 --- a/api/layer/multipart_upload.go +++ b/api/layer/multipart_upload.go @@ -157,7 +157,7 @@ type ( bktInfo *data.BucketInfo multipartInfo *data.MultipartInfo tzHash hash.Hash - attributes [][2]string + attributes map[string]string uploadPartParams *UploadPartParams creationTime time.Time payloadReader io.Reader @@ -257,7 +257,7 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf bktInfo = p.Info.Bkt payloadReader = p.Reader decSize = p.Size - attributes [][2]string + attributes = make(map[string]string) ) if p.Info.Encryption.Enabled() { @@ -265,7 +265,7 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf if err != nil { return nil, fmt.Errorf("failed to create ecnrypted reader: %w", err) } - attributes = append(attributes, [2]string{AttributeDecryptedSize, strconv.FormatInt(p.Size, 10)}) + attributes[AttributeDecryptedSize] = strconv.FormatInt(p.Size, 10) payloadReader = r p.Size = int64(encSize) } @@ -452,7 +452,7 @@ func (n *layer) uploadZeroPart(ctx context.Context, multipartInfo *data.Multipar var ( bktInfo = p.Bkt - attributes [][2]string + attributes = make(map[string]string) multipartHash = sha256.New() tzHash hash.Hash id oid.ID @@ -462,7 +462,7 @@ func (n *layer) uploadZeroPart(ctx context.Context, multipartInfo *data.Multipar ) if p.Encryption.Enabled() { - attributes = append(attributes, [2]string{AttributeDecryptedSize, "0"}) + attributes[AttributeDecryptedSize] = "0" } if n.neoFS.IsHomomorphicHashingEnabled() { @@ -1348,11 +1348,9 @@ func (n *layer) uploadPartAsSlot(ctx context.Context, params uploadPartAsSlotPar multipartHash = sha256.New() ) - params.attributes = append(params.attributes, - [2]string{headerS3MultipartUpload, params.multipartInfo.UploadID}, - [2]string{headerS3MultipartNumber, strconv.FormatInt(int64(params.uploadPartParams.PartNumber), 10)}, - [2]string{headerS3MultipartCreated, strconv.FormatInt(time.Now().UnixNano(), 10)}, - ) + params.attributes[headerS3MultipartUpload] = params.multipartInfo.UploadID + params.attributes[headerS3MultipartNumber] = strconv.FormatInt(int64(params.uploadPartParams.PartNumber), 10) + params.attributes[headerS3MultipartCreated] = strconv.FormatInt(time.Now().UnixNano(), 10) prm := PrmObjectCreate{ Container: params.bktInfo.CID, diff --git a/api/layer/neofs.go b/api/layer/neofs.go index 7290db761..b4a44faeb 100644 --- a/api/layer/neofs.go +++ b/api/layer/neofs.go @@ -96,7 +96,7 @@ type PrmObjectCreate struct { Creator user.ID // Key-value object attributes. - Attributes [][2]string + Attributes map[string]string // Value for Timestamp attribute (optional). CreationTime time.Time diff --git a/api/layer/neofs_mock.go b/api/layer/neofs_mock.go index dd2f369ed..4e2d0c363 100644 --- a/api/layer/neofs_mock.go +++ b/api/layer/neofs_mock.go @@ -256,8 +256,8 @@ func (t *TestNeoFS) CreateObject(_ context.Context, prm PrmObjectCreate) (oid.ID attrs = append(attrs, *a) } - for i := range prm.Attributes { - a := object.NewAttribute(prm.Attributes[i][0], prm.Attributes[i][1]) + for k, v := range prm.Attributes { + a := object.NewAttribute(k, v) attrs = append(attrs, *a) } diff --git a/api/layer/object.go b/api/layer/object.go index 2d22bffdb..0925bdfa9 100644 --- a/api/layer/object.go +++ b/api/layer/object.go @@ -233,6 +233,12 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend } } + for _, v := range p.Header { + if v == "" { + return nil, ErrMetaEmptyParameterValue + } + } + prm := PrmObjectCreate{ Container: p.BktInfo.CID, Creator: owner, @@ -241,16 +247,7 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend Payload: r, CreationTime: TimeNow(ctx), CopiesNumber: p.CopiesNumber, - } - - prm.Attributes = make([][2]string, 0, len(p.Header)) - - for k, v := range p.Header { - if v == "" { - return nil, ErrMetaEmptyParameterValue - } - - prm.Attributes = append(prm.Attributes, [2]string{k, v}) + Attributes: p.Header, } id, hash, err := n.objectPutAndHash(ctx, prm, p.BktInfo) diff --git a/api/layer/system_object.go b/api/layer/system_object.go index 25005d074..d2f3eeaed 100644 --- a/api/layer/system_object.go +++ b/api/layer/system_object.go @@ -217,11 +217,11 @@ func (n *layer) PutBucketSettings(ctx context.Context, p *PutSettingsParams) err return nil } -func (n *layer) attributesFromLock(ctx context.Context, lock *data.ObjectLock) ([][2]string, error) { +func (n *layer) attributesFromLock(ctx context.Context, lock *data.ObjectLock) (map[string]string, error) { var ( err error expEpoch uint64 - result [][2]string + result = make(map[string]string) ) if lock.Retention != nil { @@ -230,7 +230,7 @@ func (n *layer) attributesFromLock(ctx context.Context, lock *data.ObjectLock) ( } if lock.Retention.IsCompliance { - result = append(result, [2]string{AttributeComplianceMode, "true"}) + result[AttributeComplianceMode] = "true" } } @@ -242,9 +242,7 @@ func (n *layer) attributesFromLock(ctx context.Context, lock *data.ObjectLock) ( } if expEpoch != 0 { - result = append(result, [2]string{ - object.AttributeExpirationEpoch, strconv.FormatUint(expEpoch, 10), - }) + result[object.AttributeExpirationEpoch] = strconv.FormatUint(expEpoch, 10) } return result, nil diff --git a/internal/neofs/neofs.go b/internal/neofs/neofs.go index 8e8df7959..f0abf2e47 100644 --- a/internal/neofs/neofs.go +++ b/internal/neofs/neofs.go @@ -11,6 +11,7 @@ import ( "fmt" "hash" "io" + "maps" "math" "strconv" "sync" @@ -262,39 +263,33 @@ func (x *NeoFS) signMultipartObject(obj *object.Object, signer neofscrypto.Signe // CreateObject implements neofs.NeoFS interface method. func (x *NeoFS) CreateObject(ctx context.Context, prm layer.PrmObjectCreate) (oid.ID, error) { - attrNum := len(prm.Attributes) + 1 // + creation time - - if prm.Filepath != "" { - attrNum++ - } - - attrs := make([]object.Attribute, 0, attrNum) - creationTime := prm.CreationTime if creationTime.IsZero() { creationTime = time.Now() } - var a *object.Attribute - a = object.NewAttribute(object.AttributeTimestamp, strconv.FormatInt(creationTime.Unix(), 10)) - attrs = append(attrs, *a) + nonce := make([]byte, objectNonceSize) + if _, err := rand.Read(nonce); err != nil { + return oid.ID{}, fmt.Errorf("object nonce: %w", err) + } - for i := range prm.Attributes { - a = object.NewAttribute(prm.Attributes[i][0], prm.Attributes[i][1]) - attrs = append(attrs, *a) + uniqAttributes := maps.Clone(prm.Attributes) + if uniqAttributes == nil { + uniqAttributes = make(map[string]string) } + uniqAttributes[object.AttributeTimestamp] = strconv.FormatInt(creationTime.Unix(), 10) + uniqAttributes[objectNonceAttribute] = base64.StdEncoding.EncodeToString(nonce) + if prm.Filepath != "" { - a = object.NewAttribute(object.AttributeFilePath, prm.Filepath) - attrs = append(attrs, *a) + uniqAttributes[object.AttributeFilePath] = prm.Filepath } - nonce := make([]byte, objectNonceSize) - if _, err := rand.Read(nonce); err != nil { - return oid.ID{}, fmt.Errorf("object nonce: %w", err) + attrs := make([]object.Attribute, 0, len(uniqAttributes)) + for k, v := range uniqAttributes { + attr := object.NewAttribute(k, v) + attrs = append(attrs, *attr) } - objectNonceAttr := object.NewAttribute(objectNonceAttribute, base64.StdEncoding.EncodeToString(nonce)) - attrs = append(attrs, *objectNonceAttr) var obj object.Object obj.SetContainerID(prm.Container) @@ -688,12 +683,11 @@ func (x *AuthmateNeoFS) ReadObjectPayload(ctx context.Context, addr oid.Address) // CreateObject implements authmate.NeoFS interface method. func (x *AuthmateNeoFS) CreateObject(ctx context.Context, prm tokens.PrmObjectCreate) (oid.ID, error) { return x.neoFS.CreateObject(ctx, layer.PrmObjectCreate{ - Creator: prm.Creator, - Container: prm.Container, - Filepath: prm.Filepath, - Attributes: [][2]string{ - {object.AttributeExpirationEpoch, strconv.FormatUint(prm.ExpirationEpoch, 10)}}, - Payload: bytes.NewReader(prm.Payload), + Creator: prm.Creator, + Container: prm.Container, + Filepath: prm.Filepath, + Attributes: map[string]string{object.AttributeExpirationEpoch: strconv.FormatUint(prm.ExpirationEpoch, 10)}, + Payload: bytes.NewReader(prm.Payload), }) }