Read the footers in parallel when reading multiple Parquet files

cpp/CMakeLists.txt

@@ -778,6 +778,7 @@ add_library(
   src/utilities/cuda_memcpy.cu
   src/utilities/default_stream.cpp
   src/utilities/host_memory.cpp
+  src/utilities/host_worker_pool.cpp
   src/utilities/linked_column.cpp
   src/utilities/prefetch.cpp
   src/utilities/stacktrace.cpp

cpp/include/cudf/detail/utilities/host_worker_pool.hpp

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2025, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <BS_thread_pool.hpp>
+
+namespace cudf::detail {
+
+/**
+ * @brief Retrieves a reference to the global host worker thread pool.
+ *
+ * This function returns a reference to a thread pool that can be used for executing host-only
+ * tasks. The pool size is potentially not optimal for tasks that include device operations, like
+ * copies between host and device and kernel calls.
+ *
+ * @return A reference to the host worker thread pool.
+ */
+BS::thread_pool& host_worker_pool();
+
+}  // namespace cudf::detail

cpp/src/io/parquet/reader_impl_helpers.cpp

@@ -23,6 +23,7 @@
 #include "ipc/Message_generated.h"
 #include "ipc/Schema_generated.h"
 
+#include <cudf/detail/utilities/host_worker_pool.hpp>
 #include <cudf/logger.hpp>
 
 #include <thrust/iterator/counting_iterator.h>
@@ -352,11 +353,21 @@ metadata::metadata(datasource* source)
 std::vector<metadata> aggregate_reader_metadata::metadatas_from_sources(
   host_span<std::unique_ptr<datasource> const> sources)
 {
+  // Avoid using the thread pool for a single source
+  if (sources.size() == 1) { return {metadata{sources[0].get()}}; }
+
+  std::vector<std::future<metadata>> metadata_ctor_tasks;
+  metadata_ctor_tasks.reserve(sources.size());
+  for (auto const& source : sources) {
+    metadata_ctor_tasks.emplace_back(cudf::detail::host_worker_pool().submit_task(
+      [source = source.get()] { return metadata{source}; }));
+  }
   std::vector<metadata> metadatas;
-  std::transform(
-    sources.begin(), sources.end(), std::back_inserter(metadatas), [](auto const& source) {
-      return metadata(source.get());
-    });
+  metadatas.reserve(sources.size());
+  std::transform(metadata_ctor_tasks.begin(),
+                 metadata_ctor_tasks.end(),
+                 std::back_inserter(metadatas),
+                 [](std::future<metadata>& task) { return std::move(task).get(); });
   return metadatas;
 }
 

cpp/src/utilities/host_worker_pool.cpp

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2025, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "io/utilities/getenv_or.hpp"
+
+#include <cudf/detail/utilities/host_worker_pool.hpp>
+
+namespace cudf::detail {
+
+BS::thread_pool& host_worker_pool()
+{
+  static const std::size_t default_pool_size = std::min(32u, std::thread::hardware_concurrency());
+  static const std::size_t pool_size = getenv_or("LIBCUDF_NUM_HOST_WORKERS", default_pool_size);
+  static BS::thread_pool pool(pool_size);
+  return pool;
+}
+
+}  // namespace cudf::detail