From ab1b2e9d689e2be29deab2ce03f0f3e96a5c35c8 Mon Sep 17 00:00:00 2001 From: Vikas Bhansali <64532198+vibhansa-msft@users.noreply.github.com> Date: Wed, 5 Feb 2025 13:05:11 +0530 Subject: [PATCH 1/5] Upgrade git version (#1623) --- go.mod | 14 +++++++------- go.sum | 12 ++++++++++++ go_installer.sh | 2 +- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 3cf3bdb78..584abc1b6 100755 --- a/go.mod +++ b/go.mod @@ -1,12 +1,12 @@ module github.com/Azure/azure-storage-fuse/v2 -go 1.22.7 +go 1.23.5 require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.1 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 - github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.4.0-beta.1 + github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.4.0 github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda github.com/fsnotify/fsnotify v1.8.0 github.com/golang/mock v1.6.0 @@ -16,7 +16,7 @@ require ( github.com/radovskyb/watcher v1.0.7 github.com/sevlyar/go-daemon v0.1.6 github.com/spf13/cobra v1.8.1 - github.com/spf13/pflag v1.0.5 + github.com/spf13/pflag v1.0.6 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.10.0 github.com/vibhansa-msft/blobfilter v0.0.0-20250115104552-d9d40722be3e @@ -29,7 +29,7 @@ require ( require ( github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.3.3 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/golang-jwt/jwt/v5 v5.2.1 // indirect @@ -51,10 +51,10 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.32.0 // indirect - golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 // indirect + golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c // indirect golang.org/x/net v0.34.0 // indirect - golang.org/x/sys v0.29.0 // indirect - golang.org/x/text v0.21.0 // indirect + golang.org/x/sys v0.30.0 // indirect + golang.org/x/text v0.22.0 // indirect ) replace github.com/spf13/cobra => github.com/gapra-msft/cobra v1.4.1-0.20220411185530-5b83e8ba06dd diff --git a/go.sum b/go.sum index 6f8084133..5215a3e23 100644 --- a/go.sum +++ b/go.sum @@ -12,10 +12,14 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 h1:UXT0o77lXQrikd1kg github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0/go.mod h1:cTvi54pg19DoT07ekoeMgE/taAwNtCShVeZqA+Iv2xI= github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.4.0-beta.1 h1:0yYjWwH2PzbU61ogcXZbYJWQs+WleiB42bj+wJMy7zg= github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.4.0-beta.1/go.mod h1:8wNWnf3Kk8c9moptloY3YxGAFNOMqu1JMtJbQ33rzVw= +github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.4.0 h1:FVP7qKI1g9rcEgnxiDRmOzvI2l4ydNIYSRR/qMMFQdQ= +github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.4.0/go.mod h1:KkcFZGL0F/6ooKPl8Ub1EPtGOCVXBayeWuJ1IQomreA= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 h1:kYRSnvJju5gYVyhkij+RTJ/VR6QIUaCfWeaFm2ycsjQ= github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= +github.com/AzureAD/microsoft-authentication-library-for-go v1.3.3 h1:H5xDQaE3XowWfhZRUpnfC+rGZMEVoSiji+b+/HFAPU4= +github.com/AzureAD/microsoft-authentication-library-for-go v1.3.3/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda h1:NOo6+gM9NNPJ3W56nxOKb4164LEw094U0C8zYQM8mQU= github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda/go.mod h1:2CaSFTh2ph9ymS6goiOKIBdfhwWUVsX4nQ5QjIYFHHs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -93,6 +97,8 @@ github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= @@ -114,6 +120,8 @@ golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 h1:yqrTHse8TCMW1M1ZCP+VAR/l0kKxwaAIqN/il7x4voA= golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= +golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c h1:KL/ZBHXgKGVmuZBZ01Lt57yE5ws8ZPSkkihmEyq7FXc= +golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -130,11 +138,15 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= diff --git a/go_installer.sh b/go_installer.sh index d2f813ad4..7bb771e55 100755 --- a/go_installer.sh +++ b/go_installer.sh @@ -1,6 +1,6 @@ #!/bin/bash work_dir=$(echo $1 | sed 's:/*$::') -version="1.22.7" +version="1.23.5" arch=`hostnamectl | grep "Arch" | rev | cut -d " " -f 1 | rev` if [ $arch != "arm64" ] From 2061ecb95d747ae2c75e0c120b221ae0cfa778b7 Mon Sep 17 00:00:00 2001 From: vibhansa Date: Thu, 6 Feb 2025 02:14:24 -0800 Subject: [PATCH 2/5] Downgrading the version as new version is giving errors in notifying the parent process --- go.mod | 4 +++- go_installer.sh | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 584abc1b6..057623087 100755 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/Azure/azure-storage-fuse/v2 -go 1.23.5 +go 1.22.7 + +toolchain go1.23.1 require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 diff --git a/go_installer.sh b/go_installer.sh index 7bb771e55..cc27d13ae 100755 --- a/go_installer.sh +++ b/go_installer.sh @@ -1,6 +1,6 @@ #!/bin/bash work_dir=$(echo $1 | sed 's:/*$::') -version="1.23.5" +version="1.23.1" arch=`hostnamectl | grep "Arch" | rev | cut -d " " -f 1 | rev` if [ $arch != "arm64" ] From f4888d66c315f1416fb1c9734de532fade8a3278 Mon Sep 17 00:00:00 2001 From: vibhansa Date: Thu, 6 Feb 2025 02:15:56 -0800 Subject: [PATCH 3/5] Downgrading the version as new version is giving errors in notifying the parent process --- go.sum | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/go.sum b/go.sum index 5215a3e23..18b038c0d 100644 --- a/go.sum +++ b/go.sum @@ -10,14 +10,10 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 h1:UXT0o77lXQrikd1kgwIPQOUect7EoR/+sbP4wQKdzxM= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0/go.mod h1:cTvi54pg19DoT07ekoeMgE/taAwNtCShVeZqA+Iv2xI= -github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.4.0-beta.1 h1:0yYjWwH2PzbU61ogcXZbYJWQs+WleiB42bj+wJMy7zg= -github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.4.0-beta.1/go.mod h1:8wNWnf3Kk8c9moptloY3YxGAFNOMqu1JMtJbQ33rzVw= github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.4.0 h1:FVP7qKI1g9rcEgnxiDRmOzvI2l4ydNIYSRR/qMMFQdQ= github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.4.0/go.mod h1:KkcFZGL0F/6ooKPl8Ub1EPtGOCVXBayeWuJ1IQomreA= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= -github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 h1:kYRSnvJju5gYVyhkij+RTJ/VR6QIUaCfWeaFm2ycsjQ= -github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/AzureAD/microsoft-authentication-library-for-go v1.3.3 h1:H5xDQaE3XowWfhZRUpnfC+rGZMEVoSiji+b+/HFAPU4= github.com/AzureAD/microsoft-authentication-library-for-go v1.3.3/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda h1:NOo6+gM9NNPJ3W56nxOKb4164LEw094U0C8zYQM8mQU= @@ -95,7 +91,6 @@ github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -118,8 +113,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= -golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 h1:yqrTHse8TCMW1M1ZCP+VAR/l0kKxwaAIqN/il7x4voA= -golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c h1:KL/ZBHXgKGVmuZBZ01Lt57yE5ws8ZPSkkihmEyq7FXc= golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -136,15 +129,11 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= -golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 09448b04becef26fa17124f181befbd0193c6ecd Mon Sep 17 00:00:00 2001 From: Liza Shakury <42377481+LizaShak@users.noreply.github.com> Date: Thu, 6 Feb 2025 18:28:01 +0200 Subject: [PATCH 4/5] Documentation on required private endpoints to access hns enabled storage (#1616) --- CHANGELOG.md | 1 + README.md | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e354758af..f9fa81ecd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ **Other Changes** - Optimized listing operation on HNS account to support symlinks. - Optimized Rename operation to do less number of REST calls. +- Add documentation on usage of Private Endpoints with HNS-Enabled Storage Accounts **Features** - Mount container or directory but restrict the view of blobs that you can see. This feature is available only in read-only mount. diff --git a/README.md b/README.md index 1a33fac54..9d4aaca08 100755 --- a/README.md +++ b/README.md @@ -265,6 +265,23 @@ Below diagrams guide you to choose right configuration for your workloads. - Multiple filters can be combined using '&&' and '||' operator as well, however precedence using '()' is not supported yet. - e.g. ```--filter=name=^testfil.* && size>130000000``` +## Using Private Endpoints with HNS-Enabled Storage Accounts + +When accessing an HNS (Hierarchical Namespace) enabled Azure Storage account behind private endpoints, it is crucial to create **two separate private endpoints** to ensure proper connectivity: + +1. **Private Endpoint for DFS** + - Target: `privatelink.dfs.core.windows.net` + - This endpoint is necessary for accessing the Data Lake Storage Gen2 (HNS) functionality. + +2. **Private Endpoint for Blob** + - Target: `privatelink.blob.core.windows.net` + - This endpoint is necessary for accessing Blob Storage operations. + +### Why Both Endpoints Are Required + +HNS-enabled storage accounts utilize separate endpoints for Blob and DFS operations: +- The DFS endpoint (`dfs.core.windows.net`) is used for namespace-related operations like directory and file management. +- The Blob endpoint (`blob.core.windows.net`) is used for operations like streaming data to and from blobs. ## Frequently Asked Questions - How do I generate a SAS with permissions for rename? From 5331dde698a4df05f17cc4c7d11efcddc1097811 Mon Sep 17 00:00:00 2001 From: jainakanksha-msft Date: Mon, 10 Feb 2025 11:08:30 +0530 Subject: [PATCH 5/5] Create highspeed_write_nonzero.py (#1628) --- .../scripts/highspeed_write_nonzero.py | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 perf_testing/scripts/highspeed_write_nonzero.py diff --git a/perf_testing/scripts/highspeed_write_nonzero.py b/perf_testing/scripts/highspeed_write_nonzero.py new file mode 100644 index 000000000..4cfe95eaa --- /dev/null +++ b/perf_testing/scripts/highspeed_write_nonzero.py @@ -0,0 +1,77 @@ +import os +import shutil +import time +import argparse +import subprocess +import mmap +import io +from multiprocessing import Pool, cpu_count + + +# Function to create files using dd command +def create_file_dd(file_index, folder, source_file, timestamp): + filename = os.path.join(folder, f'ddFile_{timestamp}_{file_index}') + block_size = 1 # in GB + count = 36 + file_size_gb = (block_size * count) + + command = f"dd if={source_file} of={filename} bs={block_size}G count={count} oflag=direct" + + start_time = time.time() + result = subprocess.run(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + end_time = time.time() + + if result.returncode != 0: + return (filename, 0, 0.0, f"Error creating file {filename}: {result.stderr.decode().strip()}") + + write_time = end_time - start_time + write_speed = (file_size_gb * 1024) / write_time # MB/s + + return (filename, write_time, write_speed, file_size_gb, None) + +# Main function to handle parallel execution +def main(folder, num_files, source_file): + if not os.path.exists(folder): + os.makedirs(folder) + + timestamp = int(time.time()) # Get current timestamp for file naming + + start_time = time.time() + results = [] + + + with Pool(processes=cpu_count()) as pool: # Pool of workers based on the CPU count + futures=[] + futures += [pool.apply_async(create_file_dd, (i, folder, source_file, timestamp)) for i in range(num_files)] + + # Collect results from async operations + for future in futures: + result = future.get() + if result[4] is None: # No error + results.append(result) + else: + print(result[4]) # Print error messages + + end_time = time.time() + + total_time = end_time - start_time + total_data_written = sum(r[3] for r in results) # in GB + speed_gbps = (total_data_written *8 ) / total_time # Convert GB to Gigabits (1 GB = 8 Gb) + + throughput = (total_data_written * 1024) / total_time + print(f"Number of files written: {num_files}") + print(f"Total amount of data written: {total_data_written:.2f} GB") + print(f"Total time taken: {total_time:.2f} seconds") + print(f"Overall Speed: {speed_gbps:.2f} Gbps") + print(f"Throughput: {throughput:.2f} MiB/s") + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Create multiple files using various methods in parallel.') + parser.add_argument('folder', type=str, help='The folder where the files will be written.') + parser.add_argument('num_files', type=int, help='The number of files to create.') + parser.add_argument('source_file', type=str, help='The source file to copy data from.') + + args = parser.parse_args() + main(args.folder, args.num_files, args.source_file) + +# python3 highspeed_write_nonzero.py ~/drs/random_data_test/ 5 /mnt/azcopy_test_180GB.log