Extends shapes to store bounding boxes (#148)

* Move bounding box calculation into nigiri

* Make bounding boxes for segments optional

Clients are now required to provide a fallback when needed.

* Fix duplicated entries

* Make filtering more explicit

* Delay value calculation until needed

This avoids duplicated and unused entries for repeated calls.

* Simplify route iteration

* Update geo dependency

* Fix out of bounds shape offset

* Add test case for invalid offset

* Improve comment

* Unify boxes size

* Store stops per shapes to process

* Move code into struct

* WIP: Perform parallel calculations

* Make methods const

* Update progress bar

* Move nullptr check into struct

* Check for execution policy support

* Remove unused code

* Make nullopt more explicit

* Fix MSVC build

* Fix formatting

* Fix formatting

* Add test for routes containing sequential trips

* Rename shape result objects

* Make intended usage more obvious

* Move data on last usage

* Update variable names

* Renane variable

* Add missing const

* Apply suggested newline

Co-authored-by: Felix Gündling <felix.guendling@gmail.com>

* Improve readability

* Update parallel computations

* Improve method names

* Split shape boxes into two files

* Simplify parallel usage

* Rename variables

* WIP: Change code to tasks

Warning: Dangling references will cause use after free

* Fix use after free

* Create offset entries

* Create bounding boxes

* Remove mutex

* Delete old code

* Rename variables

* Remove intermediate struct

* Fix compile issues

* Fix invalid range

* Fix empty bound boxes

* Fix empty bound boxes for sequential trips

* Reduce stored data for simple sequential trips

* Fix formatting

* Fix size type

* Attempt to fix macos build

* wip

* Apply pending changes

* Delete old code

* Remove unused includes

* Cleanup code

* Add missing check for end element

* Update progress status text

---------

Co-authored-by: Felix Gündling <felix.guendling@gmail.com>

.pkg

@@ -9,7 +9,7 @@
 [geo]
   url=git@github.com:motis-project/geo.git
   branch=master
-  commit=0a14addf42e91b267906a156c9c2564935c03eaf
+  commit=cee17208dba98f5f3ddaf78d0bfff9c083557ba3
 [utl]
   url=git@github.com:motis-project/utl.git
   branch=master

.pkg.lock

@@ -1,24 +1,24 @@
-139770128757021168
-cista 48210e6925658163952e458a4d13629ffdb0cea3
+5786421993668347507
+cista fabfc0cbcf07343a2e0c2def3009555b22f0ae46
 PEGTL 1c1aa6e650e4d26f10fa398f148ec0cdc5f0808d
 res b759b93316afeb529b6cb5b2548b24c41e382fb0
 date ce88cc33b5551f66655614eeebb7c5b7189025fb
 googletest 7b64fca6ea0833628d6f86255a81424365f7cc0c
 fmt dc10f83be70ac2873d5f8d1ce317596f1fd318a2
-utl 77aac494c45d2b070e65fe712abc34ac74a91d0f
+utl 8bfa7fc4edc23f716173bdfbcab8294fcc31e457
 oh d21c30f40e52a83d6dc09bcffd0067598b5ec069
-zlib ee0742244d93c4237154ae16c3db42b5f284b442
-boost 60cae66449fa3c9599b2b7d3d5d44c65301ed3a3
+zlib-ng 68ab3e2d80253ec5dc3c83691d9ff70477b32cd3
+boost 73549ebca677fe6214202a1ab580362b4f80e653
 doctest 70e8f76437b76dd5e9c0a2eb9b907df190ab71a0
-geo 0a14addf42e91b267906a156c9c2564935c03eaf
+geo cee17208dba98f5f3ddaf78d0bfff9c083557ba3
 miniz 1edbdece9d71dc65c6ff405572ee37cbdcef7af4
 libressl 24acd9e710fbe842e863572da9d738715fbc74b8
-curl 3358dac21192864ef2ba47c88704d3b8d8203804
+curl 39c8a51e8ee0ab7ea712886df79c068405a2e008
 json 410c74782230daaa15054d6ee0975c0607091cb3
 opentelemetry-proto 1624689398a3226c45994d70cb544a1e781dc032
 abseil-cpp ba5240842d352b4b67a32092453a2fe5fe53a62e
-protobuf d8136b9c6a62db6ce09900ecdeb82bb793096cbd
-opentelemetry-cpp ec4aef6b17b697052edef5417825ad71947b2ed1
+protobuf df2dd518c68b882c9dce5346393f8c388108e733
+opentelemetry-cpp 60770dc9dc63e3543fc87d605b2e88fd53d7a414
 pugixml 60175e80e2f5e97e027ac78f7e14c5acc009ce50
 unordered_dense 77e91016354e6d8cba24a86c5abb807de2534c02
 wyhash 1e012b57fc2227a9e583a57e2eacb3da99816d99

include/nigiri/common/interval.h

@@ -47,8 +47,8 @@ struct interval {
       t_ -= x;
       return *this;
     }
-    iterator operator+(difference_type const x) const { return *this += x; }
-    iterator operator-(difference_type const x) const { return *this -= x; }
+    iterator operator+(difference_type const x) const { return {t_ + x}; }
+    iterator operator-(difference_type const x) const { return {t_ - x}; }
     friend difference_type operator-(iterator const& a, iterator const& b) {
       return static_cast<difference_type>(cista::to_idx(a.t_) -
                                           cista::to_idx(b.t_));

include/nigiri/loader/gtfs/shape.h

@@ -10,10 +10,20 @@ struct shapes_storage;
 
 namespace nigiri::loader::gtfs {
 
+using relative_shape_idx_t =
+    cista::strong<std::uint32_t, struct relative_shape_idx_>;
+
 struct shape_loader_state {
+  shape_idx_t get_shape_idx(relative_shape_idx_t const i) const {
+    return shape_idx_t{to_idx(i) + index_offset_};
+  }
+  relative_shape_idx_t get_relative_idx(shape_idx_t const i) const {
+    return relative_shape_idx_t{to_idx(i) - index_offset_};
+  }
+
   hash_map<std::string, shape_idx_t> id_map_{};
-  std::vector<std::vector<double>> distances_{};
-  shape_idx_t index_offset_;
+  vector_map<relative_shape_idx_t, std::vector<double>> distances_{};
+  shape_idx_t::value_t index_offset_;
 };
 
 shape_loader_state parse_shapes(std::string_view const, shapes_storage&);

include/nigiri/loader/gtfs/shape_prepare.h

@@ -1,18 +1,20 @@
 #pragma once
 
-#include "nigiri/loader/gtfs/shape.h"
 #include "nigiri/loader/gtfs/trip.h"
+#include "nigiri/types.h"
 
 namespace nigiri {
+struct shape_loader_state;
 struct shapes_storage;
 struct timetable;
 }  // namespace nigiri
 
 namespace nigiri::loader::gtfs {
 
-void calculate_shape_offsets(timetable const&,
-                             shapes_storage&,
-                             vector_map<gtfs_trip_idx_t, trip> const&,
-                             shape_loader_state const&);
+void calculate_shape_offsets_and_bboxes(
+    timetable const&,
+    shapes_storage&,
+    shape_loader_state const&,
+    vector_map<gtfs_trip_idx_t, trip> const&);
 
 }  // namespace nigiri::loader::gtfs

include/nigiri/shapes_storage.h

@@ -1,10 +1,12 @@
 #pragma once
 
 #include <filesystem>
+#include <optional>
 #include <span>
 
 #include "cista/containers/pair.h"
 
+#include "geo/box.h"
 #include "geo/latlng.h"
 
 #include "nigiri/types.h"
@@ -23,6 +25,9 @@ struct shapes_storage {
   shape_offset_idx_t add_offsets(std::vector<shape_offset_t> const&);
   void add_trip_shape_offsets(
       trip_idx_t, cista::pair<shape_idx_t, shape_offset_idx_t> const&);
+  geo::box get_bounding_box(route_idx_t) const;
+  std::optional<geo::box> get_bounding_box(route_idx_t,
+                                           std::size_t segment) const;
 
   cista::mmap::protection mode_;
   std::filesystem::path p_;
@@ -31,6 +36,8 @@ struct shapes_storage {
   mm_vecvec<shape_offset_idx_t, shape_offset_t, std::uint64_t> offsets_;
   mm_vec_map<trip_idx_t, cista::pair<shape_idx_t, shape_offset_idx_t>>
       trip_offset_indices_;
+  mm_vec_map<route_idx_t, geo::box> route_bboxes_;
+  mm_vecvec<route_idx_t, geo::box, std::uint64_t> route_segment_bboxes_;
 };
 
 }  // namespace nigiri

src/loader/gtfs/load_timetable.cc

@@ -261,9 +261,6 @@ void load_timetable(loader_config const& config,
                               {source_file_idx, trp.from_line_, trp.to_line_},
                               train_nr, stop_seq_numbers);
     }
-    if (shapes_data != nullptr) {
-      calculate_shape_offsets(tt, *shapes_data, trip_data.data_, shape_states);
-    }
 
     auto const timer = scoped_timer{"loader.gtfs.routes.build"};
     auto const attributes = std::basic_string<attribute_combination_idx_t>{};
@@ -367,6 +364,11 @@ void load_timetable(loader_config const& config,
       progress_tracker->increment();
     }
 
+    if (shapes_data != nullptr) {
+      calculate_shape_offsets_and_bboxes(tt, *shapes_data, shape_states,
+                                         trip_data.data_);
+    }
+
     // Build location_routes map
     for (auto l = tt.location_routes_.size(); l != tt.n_locations(); ++l) {
       tt.location_routes_.emplace_back(location_routes[location_idx_t{l}]);

src/loader/gtfs/shape.cc

@@ -26,12 +26,11 @@ shape_loader_state parse_shapes(std::string_view const data,
     utl::csv_col<double, UTL_NAME("shape_dist_traveled")> distance_;
   };
 
-  auto const index_offset = static_cast<shape_idx_t>(shapes.size());
   auto states = shape_loader_state{
-      .index_offset_ = index_offset,
+      .index_offset_ = static_cast<shape_idx_t::value_t>(shapes.size()),
   };
   auto lookup = cached_lookup(states.id_map_);
-  auto seq = std::vector<std::vector<std::uint32_t>>{};
+  auto seq = vector_map<relative_shape_idx_t, std::vector<std::uint32_t>>{};
 
   auto const progress_tracker = utl::get_active_progress_tracker();
   progress_tracker->status("Parse Shapes")
@@ -49,7 +48,7 @@ shape_loader_state parse_shapes(std::string_view const data,
           });
           auto polyline = shapes[shape_idx];
           polyline.push_back(geo::latlng{*entry.lat_, *entry.lon_});
-          auto const state_idx = to_idx(shape_idx - index_offset);
+          auto const state_idx = states.get_relative_idx(shape_idx);
           seq[state_idx].push_back(*entry.seq_);
           auto& distances = states.distances_[state_idx];
           if (distances.empty()) {
@@ -65,11 +64,13 @@ shape_loader_state parse_shapes(std::string_view const data,
         });
 
   auto polyline = std::vector<geo::latlng>();
-  auto shape_idx = states.index_offset_;
-  for (auto i = 0U; i != states.distances_.size(); ++i) {
+  for (auto const i :
+       interval{relative_shape_idx_t{0U},
+                relative_shape_idx_t{states.distances_.size()}}) {
     if (utl::is_sorted(seq[i], std::less<>{})) {
       continue;
     }
+    auto const shape_idx = states.get_shape_idx(i);
 
     polyline.resize(shapes[shape_idx].size());
     for (auto j = 0U; j != shapes[shape_idx].size(); ++j) {
@@ -79,7 +80,6 @@ shape_loader_state parse_shapes(std::string_view const data,
     std::tie(seq[i], states.distances_[i], polyline) =
         utl::sort_by(seq[i], states.distances_[i], polyline);
     std::copy(begin(polyline), end(polyline), begin(shapes[shape_idx]));
-    ++shape_idx;
   }
 
   return states;