diff --git a/README.md b/README.md index b53004b4d..48f328198 100755 --- a/README.md +++ b/README.md @@ -86,6 +86,7 @@ For more information, see the [wiki](https://github.com/Azure/azure-storage-fuse * [OPTIONAL] **--max-blocks-per-file=3** : Maximum number of blocks to be cached in memory for a read streaming. * [OPTIONAL] **--block-size-mb=16** : Size (in MB) of a block to be downloaded during streaming. If configured block-size is <= 64MB then block will be downloaded in a single thread. For higher block size parts of it will be downloaded in parallel. "--max-concurrency" parameter can be used to control parallelism factor. When higher block size is configured, memory usage of blobfuse will be high as these blocks are cached in memory only. * [OPTIONAL] **--ignore-open-flags=false** : There are certain flags in Open file-system call, which are not supported by blobfuse. If file handle is open with such flags, read/write operations fail at later stage. This CLI option allows user to supress (ignore) those flags while opening the file handle. Ignored flags are O_SYNC and O_DIRECT. + * [OPTIONAL] **--debug-libcurl=false** : Sometimes the HTTP client fails in an unexpected way. This CLI option allows users to debug libcurl calls. ### Valid authentication setups: diff --git a/blobfuse/blobfuse.cpp b/blobfuse/blobfuse.cpp index f6f3d1e2b..887ba77de 100755 --- a/blobfuse/blobfuse.cpp +++ b/blobfuse/blobfuse.cpp @@ -75,6 +75,7 @@ const struct fuse_opt option_spec[] = OPTION("--max-blocks-per-file=%s", max_blocks_per_file), OPTION("--block-size-mb=%s", block_size_mb), OPTION("--ignore-open-flags=%s", ignore_open_flags), + OPTION("--debug-libcurl=%s", debug_libcurl), OPTION("--version", version), OPTION("-v", version), @@ -1222,6 +1223,17 @@ read_and_set_arguments(int argc, char *argv[], struct fuse_args *args) } } + gEnableDebugLibcurl = false; + if(cmd_options.debug_libcurl != NULL) + { + std::string debug(cmd_options.debug_libcurl); + if(debug == "true") + { + syslog(LOG_INFO, "Debug libcurl is turned on"); + gEnableDebugLibcurl = true; + } + } + if (config_options.streaming && !config_options.readOnlyMount) { syslog(LOG_ERR, "Read-Streaming is supported only on Readonly Mounts. Use '-o ro' option in mount command"); fprintf(stderr, "Read-Streaming is supported only on Readonly Mounts. Use '-o ro' option in mount command"); diff --git a/blobfuse/include/BlobfuseGlobals.h b/blobfuse/include/BlobfuseGlobals.h index a9e2e2007..1d05f5ab5 100755 --- a/blobfuse/include/BlobfuseGlobals.h +++ b/blobfuse/include/BlobfuseGlobals.h @@ -75,7 +75,7 @@ struct configParams unsigned long long cachePollTimeout; unsigned long long maxEviction; - // Azure retry policty config + // Azure retry policy config int maxTryCount; double maxTimeoutSeconds; double retryDelay; @@ -134,6 +134,7 @@ struct cmdlineOptions const char *max_blocks_per_file; // Number of blocks to be cached per file in case of streaming const char *block_size_mb; // Size of each block to be downloaded during streaming const char *ignore_open_flags; // Ignore unsupported flags provided in open file call + const char *debug_libcurl; // Whether or not libcurl debug should be turned on }; diff --git a/blobfuse/include/DfsProperties.h b/blobfuse/include/DfsProperties.h index 98ed45645..6188c7cf4 100644 --- a/blobfuse/include/DfsProperties.h +++ b/blobfuse/include/DfsProperties.h @@ -56,7 +56,7 @@ class adls_client_ext : public adls_client adls_client_ext( std::shared_ptr account, - int max_concurrency, + int max_concurrency, bool exception_enabled = true) : adls_client(account, max_concurrency, exception_enabled), maxConcurrency(max_concurrency) diff --git a/blobfuse/include/OAuthTokenCredentialManager.h b/blobfuse/include/OAuthTokenCredentialManager.h index 9fbadaacb..307687027 100755 --- a/blobfuse/include/OAuthTokenCredentialManager.h +++ b/blobfuse/include/OAuthTokenCredentialManager.h @@ -69,7 +69,7 @@ static std::shared_ptr TokenManagerSingleton; /// If no callback is supplied and the token manager doesn't exist, this function will throw. /// No callback is necessary to get the current instance. /// -std::shared_ptr GetTokenManagerInstance(std::function)> , const std::string& caCertFile = "", const std::string& httpsProxy = ""); +std::shared_ptr GetTokenManagerInstance(std::function)>, const std::string& caCertFile = "", const std::string& httpsProxy = ""); // maybe TODO: SetUpSPNCallback, SetUpDeviceOAuthCallback. diff --git a/blobfuse/src/BlobfuseGlobals.cpp b/blobfuse/src/BlobfuseGlobals.cpp index 0ed934389..f15151658 100755 --- a/blobfuse/src/BlobfuseGlobals.cpp +++ b/blobfuse/src/BlobfuseGlobals.cpp @@ -4,6 +4,7 @@ #include bool gEnableLogsHttp; +bool gEnableDebugLibcurl; std::string to_lower(std::string original) { diff --git a/cpplite/include/blob/blob_client.h b/cpplite/include/blob/blob_client.h index 88bebb6cc..1d678bbb2 100644 --- a/cpplite/include/blob/blob_client.h +++ b/cpplite/include/blob/blob_client.h @@ -412,7 +412,7 @@ namespace azure { namespace storage_lite { /// /// The storage account name. /// The storage account key. - /// A sas token for the container. + /// A sas token for the container. /// The maximum number requests could be executed in the same time. /// Return a object. AZURE_STORAGE_API static blob_client_wrapper blob_client_wrapper_init(const std::string &account_name, const std::string &account_key, const std::string &sas_token, const unsigned int concurrency); @@ -422,7 +422,7 @@ namespace azure { namespace storage_lite { /// /// The storage account name. /// The storage account key. - /// A sas token for the container. + /// A sas token for the container. /// The maximum number requests could be executed in the same time. /// True if https should be used (instead of HTTP). Note that this may cause a sizable perf loss, due to issues in libcurl. /// Blob endpoint URI to allow non-public clouds as well as custom domains. diff --git a/cpplite/include/http/libcurl_http_client.h b/cpplite/include/http/libcurl_http_client.h index 07339aba7..ed611c528 100644 --- a/cpplite/include/http/libcurl_http_client.h +++ b/cpplite/include/http/libcurl_http_client.h @@ -21,6 +21,7 @@ #include "http_base.h" extern bool gEnableLogsHttp; +extern bool gEnableDebugLibcurl; namespace azure { namespace storage_lite { class CurlEasyClient; @@ -268,6 +269,45 @@ namespace azure { namespace storage_lite { return actual_size; } + static int debug_callback(CURL *handle, curl_infotype type, char *data, size_t size, void *userptr) + { + /* Taken from curl docs: https://curl.se/libcurl/c/CURLOPT_DEBUGFUNCTION.html */ + const char *text; + (void)handle; /* prevent compiler warning */ + (void)userptr; + (void)size; + + switch (type) { + case CURLINFO_TEXT: + text = "== Info"; + break; + default: + return 0; + + case CURLINFO_HEADER_OUT: + text = "=> Send header"; + break; + case CURLINFO_DATA_OUT: + text = "=> Send data"; + break; + case CURLINFO_SSL_DATA_OUT: + text = "=> Send SSL data"; + break; + case CURLINFO_HEADER_IN: + text = "<= Recv header"; + break; + case CURLINFO_DATA_IN: + text = "<= Recv data"; + break; + case CURLINFO_SSL_DATA_IN: + text = "<= Recv SSL data"; + break; + } + + syslog(LOG_DEBUG, "libcurl: %s: %s", text, data); + return 0; + } + static void check_code(CURLcode code, std::string = std::string()) { if (code == CURLE_OK) diff --git a/cpplite/src/http/libcurl_http_client.cpp b/cpplite/src/http/libcurl_http_client.cpp index c0ff0a6af..272a90bb6 100644 --- a/cpplite/src/http/libcurl_http_client.cpp +++ b/cpplite/src/http/libcurl_http_client.cpp @@ -70,6 +70,11 @@ namespace azure { namespace storage_lite { { check_code(curl_easy_setopt(m_curl, CURLOPT_PROXY, m_client->get_proxy().data())); } + if (gEnableDebugLibcurl) + { + check_code(curl_easy_setopt(m_curl, CURLOPT_VERBOSE, 1L)); + check_code(curl_easy_setopt(m_curl, CURLOPT_DEBUGFUNCTION, debug_callback)); + } const auto result = curl_easy_perform(m_curl); check_code(result); // has nothing to do with checks, just resets errno for succeeded ops.