diff --git a/common/types.go b/common/types.go index 438a59bba..225ca8bb9 100644 --- a/common/types.go +++ b/common/types.go @@ -64,6 +64,7 @@ const ( DefaultAllowOtherPermissionBits os.FileMode = 0777 MbToBytes = 1024 * 1024 + GbToBytes = 1024 * MbToBytes BfuseStats = "blobfuse_stats" FuseAllowedFlags = "invalid FUSE options. Allowed FUSE configurations are: `-o attr_timeout=TIMEOUT`, `-o negative_timeout=TIMEOUT`, `-o entry_timeout=TIMEOUT` `-o allow_other`, `-o allow_root`, `-o umask=PERMISSIONS -o default_permissions`, `-o ro`" diff --git a/component/azstorage/block_blob.go b/component/azstorage/block_blob.go index c72dcd8a2..06842e6ea 100644 --- a/component/azstorage/block_blob.go +++ b/component/azstorage/block_blob.go @@ -1073,11 +1073,50 @@ func (bb *BlockBlob) TruncateFile(name string, size int64) error { return err } } - //TODO: the resize might be very big - need to allocate in chunks if size == 0 || attr.Size == 0 { - err := bb.WriteFromBuffer(name, nil, make([]byte, size)) - if err != nil { - log.Err("BlockBlob::TruncateFile : Failed to set the %s to 0 bytes [%s]", name, err.Error()) + // If we are resizing to a value > 1GB then we need to upload multiple blocks to resize + if size > 1*common.GbToBytes { + blkSize := int64(16 * common.MbToBytes) + blobName := filepath.Join(bb.Config.prefixPath, name) + blobURL := bb.Container.NewBlockBlobURL(blobName) + + blkList := make([]string, 0) + id := base64.StdEncoding.EncodeToString(common.NewUUIDWithLength(16)) + + for i := 0; size > 0; i++ { + if i == 0 || size < blkSize { + // Only first and last block we upload and rest all we replicate with the first block itself + if size < blkSize { + blkSize = size + id = base64.StdEncoding.EncodeToString(common.NewUUIDWithLength(16)) + } + data := make([]byte, blkSize) + + _, err = blobURL.StageBlock(context.Background(), + id, + bytes.NewReader(data), + bb.blobAccCond.LeaseAccessConditions, + nil, + bb.downloadOptions.ClientProvidedKeyOptions) + if err != nil { + log.Err("BlockBlob::TruncateFile : Failed to stage block for %s [%s]", name, err.Error()) + return err + } + } + blkList = append(blkList, id) + size -= blkSize + } + + err = bb.CommitBlocks(blobName, blkList) + if err != nil { + log.Err("BlockBlob::TruncateFile : Failed to commit blocks for %s [%s]", name, err.Error()) + return err + } + } else { + err := bb.WriteFromBuffer(name, nil, make([]byte, size)) + if err != nil { + log.Err("BlockBlob::TruncateFile : Failed to set the %s to 0 bytes [%s]", name, err.Error()) + } } return err } diff --git a/component/block_cache/block_cache.go b/component/block_cache/block_cache.go index 6fba68ec1..1402dd9e0 100644 --- a/component/block_cache/block_cache.go +++ b/component/block_cache/block_cache.go @@ -1191,6 +1191,15 @@ func (bc *BlockCache) commitBlocks(handle *handlemap.Handle) error { return err } + lst, err := bc.NextComponent().GetCommittedBlockList(handle.Path) + if err != nil { + log.Err("BlockCache::commitBlocks : Failed to get committed block list for %s [%s]", handle.Path, err.Error()) + } + + if len(blockIdList) > 0 && (lst == nil || len(*lst) != len(blockIdList)) { + log.Err("BlockCache::commitBlocks : Committed list does not match for %s", handle.Path) + } + return nil }