Add Initial PMTiles support (#2882)

Co-authored-by: Bart Louwers <bart.louwers@gmail.com>

Author: tdcosta100

.gitmodules

@@ -55,3 +55,6 @@
 [submodule "vendor/glslang"]
 	path = vendor/glslang
 	url = https://github.com/KhronosGroup/glslang.git
+[submodule "vendor/PMTiles"]
+	path = vendor/PMTiles
+	url = https://github.com/protomaps/PMTiles.git

BUILD.bazel

@@ -129,6 +129,7 @@ cc_library(
         "//vendor:eternal",
         "//vendor:mapbox-base",
         "//vendor:parsedate",
+        "//vendor:pmtiles",
         "//vendor:polylabel",
         "//vendor:protozero",
         "//vendor:unique_resource",

CMakeLists.txt

@@ -11,6 +11,7 @@ option(MLN_WITH_OPENGL "Build with OpenGL renderer" ON)
 option(MLN_WITH_EGL "Build with EGL renderer" OFF)
 option(MLN_WITH_VULKAN "Build with Vulkan renderer" OFF)
 option(MLN_WITH_OSMESA "Build with OSMesa (Software) renderer" OFF)
+option(MLN_WITH_PMTILES "Build with PMTiles support" ON)
 option(MLN_WITH_WERROR "Make all compilation warnings errors" ON)
 option(MLN_LEGACY_RENDERER "Include the legacy rendering pathway" ON)
 option(MLN_DRAWABLE_RENDERER "Include the drawable rendering pathway" OFF)
@@ -1450,6 +1451,7 @@ include(${PROJECT_SOURCE_DIR}/vendor/earcut.hpp.cmake)
 include(${PROJECT_SOURCE_DIR}/vendor/eternal.cmake)
 include(${PROJECT_SOURCE_DIR}/vendor/mapbox-base.cmake)
 include(${PROJECT_SOURCE_DIR}/vendor/parsedate.cmake)
+include(${PROJECT_SOURCE_DIR}/vendor/pmtiles.cmake)
 include(${PROJECT_SOURCE_DIR}/vendor/polylabel.cmake)
 include(${PROJECT_SOURCE_DIR}/vendor/protozero.cmake)
 include(${PROJECT_SOURCE_DIR}/vendor/tracy.cmake)
@@ -1473,6 +1475,7 @@ target_link_libraries(
         mbgl-vendor-earcut.hpp
         mbgl-vendor-eternal
         mbgl-vendor-parsedate
+        mbgl-vendor-pmtiles
         mbgl-vendor-polylabel
         mbgl-vendor-protozero
         mbgl-vendor-unique_resource
@@ -1512,6 +1515,7 @@ export(TARGETS
     mbgl-vendor-earcut.hpp
     mbgl-vendor-eternal
     mbgl-vendor-parsedate
+    mbgl-vendor-pmtiles
     mbgl-vendor-polylabel
     mbgl-vendor-protozero
     mbgl-vendor-unique_resource

bazel/core.bzl

@@ -377,6 +377,7 @@ MLN_CORE_SOURCE = [
     "src/mbgl/storage/local_file_source.hpp",
     "src/mbgl/storage/main_resource_loader.hpp",
     "src/mbgl/storage/network_status.cpp",
+    "src/mbgl/storage/pmtiles_file_source.hpp",
     "src/mbgl/storage/resource.cpp",
     "src/mbgl/storage/resource_options.cpp",
     "src/mbgl/storage/resource_transform.cpp",

include/mbgl/storage/file_source.hpp

@@ -25,6 +25,7 @@ enum FileSourceType : uint8_t {
     FileSystem,
     Network,
     Mbtiles,
+    Pmtiles,
     ResourceLoader ///< %Resource loader acts as a proxy and has logic
     /// for request delegation to Asset, Cache, and other
     /// file sources.

include/mbgl/storage/resource.hpp

@@ -95,6 +95,7 @@ class Resource {
     // Includes auxiliary data if this is a tile request.
     std::optional<TileData> tileData;
 
+    std::optional<std::pair<uint64_t, uint64_t>> dataRange = std::nullopt;
     std::optional<Timestamp> priorModified = std::nullopt;
     std::optional<Timestamp> priorExpires = std::nullopt;
     std::optional<std::string> priorEtag = std::nullopt;

include/mbgl/util/constants.hpp

@@ -60,6 +60,7 @@ constexpr int DEFAULT_RATE_LIMIT_TIMEOUT = 5;
 constexpr const char* ASSET_PROTOCOL = "asset://";
 constexpr const char* FILE_PROTOCOL = "file://";
 constexpr const char* MBTILES_PROTOCOL = "mbtiles://";
+constexpr const char* PMTILES_PROTOCOL = "pmtiles://";
 constexpr uint32_t DEFAULT_MAXIMUM_CONCURRENT_REQUESTS = 20;
 
 constexpr uint8_t TERRAIN_RGB_MAXZOOM = 15;

platform/android/MapLibreAndroid/src/cpp/http_file_source.cpp

@@ -91,9 +91,15 @@ void RegisterNativeHTTPRequest(jni::JNIEnv& env) {
 HTTPRequest::HTTPRequest(jni::JNIEnv& env, const Resource& resource_, FileSource::Callback callback_)
     : resource(resource_),
       callback(callback_) {
+    std::string dataRangeStr;
     std::string etagStr;
     std::string modifiedStr;
 
+    if (resource.dataRange) {
+        dataRangeStr = std::string("bytes=") + std::to_string(resource.dataRange->first) + std::string("-") +
+                       std::to_string(resource.dataRange->second);
+    }
+
     if (resource.priorEtag) {
         etagStr = *resource.priorEtag;
     } else if (resource.priorModified) {
@@ -104,13 +110,14 @@ HTTPRequest::HTTPRequest(jni::JNIEnv& env, const Resource& resource_, FileSource
 
     static auto& javaClass = jni::Class<HTTPRequest>::Singleton(env);
     static auto constructor =
-        javaClass.GetConstructor<jni::jlong, jni::String, jni::String, jni::String, jni::jboolean>(env);
+        javaClass.GetConstructor<jni::jlong, jni::String, jni::String, jni::String, jni::String, jni::jboolean>(env);
 
     javaRequest = jni::NewGlobal(env,
                                  javaClass.New(env,
                                                constructor,
                                                reinterpret_cast<jlong>(this),
                                                jni::Make<jni::String>(env, resource.url),
+                                               jni::Make<jni::String>(env, dataRangeStr),
                                                jni::Make<jni::String>(env, etagStr),
                                                jni::Make<jni::String>(env, modifiedStr),
                                                (jboolean)(resource_.usage == Resource::Usage::Offline)));

platform/android/MapLibreAndroid/src/main/java/org/maplibre/android/http/HttpRequest.java

@@ -18,12 +18,13 @@ public interface HttpRequest {
    * @param httpRequest  callback to be invoked when we receive a response
    * @param nativePtr    the pointer associated to the request
    * @param resourceUrl  the resource url to download
+   * @param dataRange    http header, used to indicate the part of a resource that the server should return
    * @param etag         http header, identifier for a specific version of a resource
    * @param modified     http header, used to determine if a resource hasn't been modified since
    * @param offlineUsage flag to indicate a resource will be used for offline, appends offline=true as a query parameter
    */
   void executeRequest(HttpResponder httpRequest, long nativePtr, String resourceUrl,
-                      String etag, String modified, boolean offlineUsage);
+                      String dataRange, String etag, String modified, boolean offlineUsage);
 
   /**
    * Cancels the request.

platform/android/MapLibreAndroid/src/main/java/org/maplibre/android/http/NativeHttpRequest.java

@@ -19,15 +19,16 @@ public class NativeHttpRequest implements HttpResponder {
   private long nativePtr;
 
   @Keep
-  private NativeHttpRequest(long nativePtr, String resourceUrl, String etag, String modified, boolean offlineUsage) {
+  private NativeHttpRequest(long nativePtr, String resourceUrl, String dataRange, String etag, String modified,
+                            boolean offlineUsage) {
     this.nativePtr = nativePtr;
 
     if (resourceUrl.startsWith("local://")) {
       // used by render test to serve files from assets
       executeLocalRequest(resourceUrl);
       return;
     }
-    httpRequest.executeRequest(this, nativePtr, resourceUrl, etag, modified, offlineUsage);
+    httpRequest.executeRequest(this, nativePtr, resourceUrl, dataRange, etag, modified, offlineUsage);
   }
 
   public void cancel() {

platform/android/MapLibreAndroid/src/main/java/org/maplibre/android/module/http/HttpRequestImpl.java

@@ -58,7 +58,8 @@ public class HttpRequestImpl implements HttpRequest {
 
   @Override
   public void executeRequest(HttpResponder httpRequest, long nativePtr, @NonNull String resourceUrl,
-                             @NonNull String etag, @NonNull String modified, boolean offlineUsage) {
+                             @NonNull String dataRange, @NonNull String etag, @NonNull String modified,
+                             boolean offlineUsage) {
     OkHttpCallback callback = new OkHttpCallback(httpRequest);
     try {
       HttpUrl httpUrl = HttpUrl.parse(resourceUrl);
@@ -74,6 +75,11 @@ public void executeRequest(HttpResponder httpRequest, long nativePtr, @NonNull S
         .url(resourceUrl)
         .tag(resourceUrl.toLowerCase(MapLibreConstants.MAPLIBRE_LOCALE))
         .addHeader("User-Agent", userAgentString);
+
+      if (dataRange.length() > 0) {
+        builder.addHeader("Range", dataRange);
+      }
+
       if (etag.length() > 0) {
         builder.addHeader("If-None-Match", etag);
       } else if (modified.length() > 0) {

platform/android/MapLibreAndroidTestApp/src/main/java/org/maplibre/android/testapp/utils/ExampleHttpRequestImpl.kt

@@ -11,11 +11,11 @@ import org.maplibre.android.module.http.HttpRequestImpl
  */
 class ExampleHttpRequestImpl : HttpRequest {
     override fun executeRequest(httpRequest: HttpResponder, nativePtr: Long, resourceUrl: String,
-                                etag: String, modified: String, offlineUsage: Boolean)
+                                dataRange: String, etag: String, modified: String, offlineUsage: Boolean)
     {
         // Load all json documents and any pbf ending with a 0.
         if (resourceUrl.endsWith(".json") || resourceUrl.endsWith("0.pbf")) {
-            impl.executeRequest(httpRequest, nativePtr, resourceUrl, etag, modified, offlineUsage)
+            impl.executeRequest(httpRequest, nativePtr, resourceUrl, dataRange, etag, modified, offlineUsage)
         } else {
             // All other requests get an instant 404!
             httpRequest.onResponse(

platform/android/android.cmake

@@ -61,6 +61,7 @@ target_sources(
         ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/offline_database.cpp
         ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/offline_download.cpp
         ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/online_file_source.cpp
+        ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/$<IF:$<BOOL:${MLN_WITH_PMTILES}>,pmtiles_file_source.cpp,pmtiles_file_source_stub.cpp>
         ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/sqlite3.cpp
         ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/text/bidi.cpp
         ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/util/compression.cpp

platform/darwin/src/http_file_source.mm

@@ -264,6 +264,13 @@ BOOL isValidMapboxEndpoint(NSURL *url) {
             [req addValue:@(util::rfc1123(*resource.priorModified).c_str())
                  forHTTPHeaderField:@"If-Modified-Since"];
         }
+        
+        if (resource.dataRange) {
+            NSString *rangeHeader = [NSString stringWithFormat:@"bytes=%lld-%lld",
+                                     static_cast<long long>(resource.dataRange->first),
+                                     static_cast<long long>(resource.dataRange->second)];
+            [req setValue:rangeHeader forHTTPHeaderField:@"Range"];
+        }
 
         [req addValue:impl->userAgent forHTTPHeaderField:@"User-Agent"];
 
@@ -360,7 +367,7 @@ BOOL isValidMapboxEndpoint(NSURL *url) {
                         response.etag = std::string([etag UTF8String]);
                     }
 
-                    if (responseCode == 200) {
+                    if (responseCode == 200 || responseCode == 206) {
                         response.data = std::make_shared<std::string>((const char *)[data bytes], [data length]);
                     } else if (responseCode == 204 || (responseCode == 404 && isTile)) {
                         response.noContent = true;

platform/default/BUILD.bazel

@@ -57,6 +57,7 @@ cc_library(
         "src/mbgl/storage/offline_database.cpp",
         "src/mbgl/storage/offline_download.cpp",
         "src/mbgl/storage/online_file_source.cpp",
+        "src/mbgl/storage/pmtiles_file_source.cpp",
         "src/mbgl/storage/sqlite3.cpp",
         "src/mbgl/text/bidi.cpp",
         "src/mbgl/util/compression.cpp",

platform/default/include/mbgl/storage/local_file_request.hpp

@@ -8,6 +8,8 @@ template <typename>
 class ActorRef;
 class FileSourceRequest;
 
-void requestLocalFile(const std::string&, const ActorRef<FileSourceRequest>&);
+void requestLocalFile(const std::string& path,
+                      const ActorRef<FileSourceRequest>& req,
+                      const std::optional<std::pair<uint64_t, uint64_t>>& dataRange = std::nullopt);
 
 } // namespace mbgl

platform/default/src/mbgl/storage/file_source_manager.cpp

@@ -5,6 +5,7 @@
 #include <mbgl/storage/main_resource_loader.hpp>
 #include <mbgl/storage/online_file_source.hpp>
 #include <mbgl/storage/mbtiles_file_source.hpp>
+#include <mbgl/storage/pmtiles_file_source.hpp>
 #include <mbgl/storage/resource_options.hpp>
 
 namespace mbgl {
@@ -37,6 +38,11 @@ class DefaultFileSourceManagerImpl final : public FileSourceManager {
                                       return std::make_unique<MBTilesFileSource>(resourceOptions, clientOptions);
                                   });
 
+        registerFileSourceFactory(FileSourceType::Pmtiles,
+                                  [](const ResourceOptions& resourceOptions, const ClientOptions& clientOptions) {
+                                      return std::make_unique<PMTilesFileSource>(resourceOptions, clientOptions);
+                                  });
+
         registerFileSourceFactory(FileSourceType::Network,
                                   [](const ResourceOptions& resourceOptions, const ClientOptions& clientOptions) {
                                       return std::make_unique<OnlineFileSource>(resourceOptions, clientOptions);

platform/default/src/mbgl/storage/http_file_source.cpp

@@ -269,6 +269,12 @@ HTTPRequest::HTTPRequest(HTTPFileSource::Impl *context_, Resource resource_, Fil
       resource(std::move(resource_)),
       callback(std::move(callback_)),
       handle(context->getHandle()) {
+    if (resource.dataRange) {
+        const std::string header = std::string("Range: bytes=") + std::to_string(resource.dataRange->first) +
+                                   std::string("-") + std::to_string(resource.dataRange->second);
+        headers = curl_slist_append(headers, header.c_str());
+    }
+
     // If there's already a response, set the correct etags/modified headers to
     // make sure we are getting a 304 response if possible. This avoids
     // redownloading unchanged data.
@@ -417,7 +423,7 @@ void HTTPRequest::handleResult(CURLcode code) {
         long responseCode = 0;
         curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &responseCode);
 
-        if (responseCode == 200) {
+        if (responseCode == 200 || responseCode == 206) {
             if (data) {
                 response->data = std::move(data);
             } else {

platform/default/src/mbgl/storage/local_file_request.cpp

@@ -11,7 +11,9 @@
 
 namespace mbgl {
 
-void requestLocalFile(const std::string& path, const ActorRef<FileSourceRequest>& req) {
+void requestLocalFile(const std::string& path,
+                      const ActorRef<FileSourceRequest>& req,
+                      const std::optional<std::pair<uint64_t, uint64_t>>& dataRange) {
     Response response;
     struct stat buf;
     int result = stat(path.c_str(), &buf);
@@ -21,7 +23,7 @@ void requestLocalFile(const std::string& path, const ActorRef<FileSourceRequest>
     } else if (result == -1 && errno == ENOENT) {
         response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound);
     } else {
-        auto data = util::readFile(path);
+        auto data = util::readFile(path, dataRange);
         if (!data) {
             response.error = std::make_unique<Response::Error>(Response::Error::Reason::Other,
                                                                std::string("Cannot read file ") + path);

platform/default/src/mbgl/storage/local_file_source.cpp

@@ -25,17 +25,18 @@ class LocalFileSource::Impl {
         : resourceOptions(resourceOptions_.clone()),
           clientOptions(clientOptions_.clone()) {}
 
-    void request(const std::string& url, const ActorRef<FileSourceRequest>& req) {
-        if (!acceptsURL(url)) {
+    void request(const Resource& resource, const ActorRef<FileSourceRequest>& req) {
+        if (!acceptsURL(resource.url)) {
             Response response;
             response.error = std::make_unique<Response::Error>(Response::Error::Reason::Other, "Invalid file URL");
             req.invoke(&FileSourceRequest::setResponse, response);
             return;
         }
 
         // Cut off the protocol and prefix with path.
-        const auto path = mbgl::util::percentDecode(url.substr(std::char_traits<char>::length(util::FILE_PROTOCOL)));
-        requestLocalFile(path, req);
+        const auto path = mbgl::util::percentDecode(
+            resource.url.substr(std::char_traits<char>::length(util::FILE_PROTOCOL)));
+        requestLocalFile(path, req, resource.dataRange);
     }
 
     void setResourceOptions(ResourceOptions options) {
@@ -77,7 +78,7 @@ LocalFileSource::~LocalFileSource() = default;
 std::unique_ptr<AsyncRequest> LocalFileSource::request(const Resource& resource, Callback callback) {
     auto req = std::make_unique<FileSourceRequest>(std::move(callback));
 
-    impl->actor().invoke(&Impl::request, resource.url, req->actor());
+    impl->actor().invoke(&Impl::request, resource, req->actor());
 
     return req;
 }