Calculate stop to subshape mapping during import

exe/import.cc

@@ -115,10 +115,9 @@ int main(int ac, char** av) {
  assistance = std::make_unique<assistance_times>(read_assistance(f.view()));
  }
 
- auto shapes = std::unique_ptr<shapes_storage_t>();
+ auto shapes = std::unique_ptr<shapes_storage>{};
  if (vm.contains("shapes")) {
- shapes =
- std::make_unique<shapes_storage_t>(create_shapes_storage(out_shapes));
+ shapes = std::make_unique<shapes_storage>(out_shapes);
  }
 
  auto const start = parse_date(start_date);

include/nigiri/loader/gtfs/load_timetable.h

@@ -2,6 +2,7 @@
 
 #include "nigiri/loader/dir.h"
 #include "nigiri/loader/loader_interface.h"
+#include "nigiri/shape.h"
 #include "nigiri/types.h"
 
 namespace nigiri {
@@ -23,14 +24,14 @@ void load_timetable(loader_config const&,
  dir const&,
  timetable&,
  assistance_times* = nullptr,
- shapes_storage_t* = nullptr);
+ shapes_storage* = nullptr);
 
 void load_timetable(loader_config const&,
  source_idx_t,
  dir const&,
  timetable&,
  hash_map<bitfield, bitfield_idx_t>&,
  assistance_times* = nullptr,
- shapes_storage_t* = nullptr);
+ shapes_storage* = nullptr);
 
 } // namespace nigiri::loader::gtfs

include/nigiri/loader/gtfs/loader.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "nigiri/loader/loader_interface.h"
+#include "nigiri/shape.h"
 
 namespace nigiri::loader::gtfs {
 
@@ -12,7 +13,7 @@ struct gtfs_loader : public loader_interface {
  timetable&,
  hash_map<bitfield, bitfield_idx_t>&,
  assistance_times*,
- shapes_storage_t*) const override;
+ shapes_storage*) const override;
  cista::hash_t hash(dir const&) const override;
  std::string_view name() const override;
 };

include/nigiri/loader/gtfs/shape.h

@@ -1,18 +1,21 @@
 #pragma once
 
 #include <string_view>
+#include <vector>
 
+#include "nigiri/shape.h"
 #include "nigiri/types.h"
 
 namespace nigiri::loader::gtfs {
 
 struct shape_state {
  shape_idx_t index_{};
  std::size_t last_seq_{};
+ std::vector<double> distances_{};
 };
 
 using shape_id_map_t = hash_map<std::string, shape_state>;
 
-shape_id_map_t parse_shapes(std::string_view const, shapes_storage_t&);
+shape_id_map_t parse_shapes(std::string_view const, shapes_storage&);
 
 } // namespace nigiri::loader::gtfs

include/nigiri/loader/gtfs/shape_prepare.h

@@ -0,0 +1,15 @@
+#pragma once
+
+#include "nigiri/loader/gtfs/shape.h"
+#include "nigiri/loader/gtfs/trip.h"
+#include "nigiri/shape.h"
+#include "nigiri/timetable.h"
+
+namespace nigiri::loader::gtfs {
+
+void calculate_shape_offsets(timetable const&,
+ shapes_storage&,
+ vector_map<gtfs_trip_idx_t, trip> const&,
+ shape_id_map_t const&);
+
+} // namespace nigiri::loader::gtfs

include/nigiri/loader/gtfs/trip.h

@@ -15,6 +15,7 @@
 #include "nigiri/loader/gtfs/shape.h"
 #include "nigiri/loader/gtfs/stop.h"
 #include "nigiri/timetable.h"
+#include "nigiri/types.h"
 
 namespace nigiri::loader::gtfs {
 
@@ -98,6 +99,7 @@ struct trip {
  std::vector<std::uint16_t> seq_numbers_;
  std::vector<stop_events> event_times_;
  std::vector<trip_direction_idx_t> stop_headsigns_;
+ std::vector<double> distance_traveled_;
 
  std::optional<std::vector<frequency>> frequency_;
  bool requires_interpolation_{false};

include/nigiri/loader/hrd/loader.h

@@ -2,6 +2,7 @@
 
 #include "nigiri/loader/hrd/parser_config.h"
 #include "nigiri/loader/loader_interface.h"
+#include "nigiri/shape.h"
 
 namespace nigiri::loader::hrd {
 
@@ -14,7 +15,7 @@ struct hrd_loader : public loader_interface {
  timetable& tt,
  hash_map<bitfield, bitfield_idx_t>&,
  assistance_times*,
- shapes_storage_t*) const override;
+ shapes_storage*) const override;
  cista::hash_t hash(dir const&) const override;
  nigiri::loader::hrd::config config_;
 };

include/nigiri/loader/load.h

@@ -7,6 +7,7 @@
 
 #include "nigiri/loader/assistance.h"
 #include "nigiri/common/interval.h"
+#include "nigiri/shape.h"
 #include "nigiri/timetable.h"
 #include "nigiri/types.h"
 
@@ -18,7 +19,7 @@ timetable load(std::vector<std::filesystem::path> const&,
  loader_config const&,
  interval<date::sys_days> const&,
  assistance_times*,
- shapes_storage_t*,
+ shapes_storage*,
  bool ignore = false);
 
 } // namespace nigiri::loader

include/nigiri/loader/loader_interface.h

@@ -5,6 +5,7 @@
 
 #include "nigiri/loader/assistance.h"
 #include "nigiri/loader/dir.h"
+#include "nigiri/shape.h"
 #include "nigiri/types.h"
 
 namespace nigiri {
@@ -34,7 +35,7 @@ struct loader_interface {
  timetable&,
  hash_map<bitfield, bitfield_idx_t>&,
  assistance_times*,
- shapes_storage_t*) const = 0;
+ shapes_storage*) const = 0;
  virtual cista::hash_t hash(dir const&) const = 0;
  virtual std::string_view name() const = 0;
 };

include/nigiri/rt/frun.h

@@ -1,10 +1,17 @@
 #pragma once
 
+#include <functional>
 #include <iosfwd>
+#include <span>
 
+#include "geo/latlng.h"
+
+#include "nigiri/common/interval.h"
 #include "nigiri/location.h"
 #include "nigiri/rt/run.h"
+#include "nigiri/shape.h"
 #include "nigiri/stop.h"
+#include "nigiri/types.h"
 
 namespace nigiri {
 struct rt_timetable;
@@ -136,6 +143,11 @@ struct frun : public run {
  trip_idx_t trip_idx() const;
  clasz get_clasz() const noexcept;
 
+ void for_each_shape_point(
+ shapes_storage const*,
+ interval<stop_idx_t> const&,
+ std::function<void(geo::latlng const&)> const&) const;
+
  void print(std::ostream&, interval<stop_idx_t>);
  friend std::ostream& operator<<(std::ostream&, frun const&);
 

include/nigiri/shape.h

@@ -3,24 +3,32 @@
 #include <filesystem>
 #include <span>
 
+#include "cista/containers/pair.h"
+
 #include "geo/latlng.h"
 
 #include "nigiri/types.h"
 
 namespace nigiri {
-struct timetable;
-}
-
-namespace nigiri {
-
-shapes_storage_t create_shapes_storage(
- std::filesystem::path const&,
- cista::mmap::protection = cista::mmap::protection::WRITE);
-
-std::span<geo::latlng const> get_shape(timetable const&,
- shapes_storage_t const&,
- trip_idx_t);
 
-std::span<geo::latlng const> get_shape(shapes_storage_t const&, shape_idx_t);
+struct shapes_storage {
+ explicit shapes_storage(
+ std::filesystem::path const&,
+ cista::mmap::protection = cista::mmap::protection::WRITE);
+ std::span<geo::latlng const> get_shape(shape_idx_t) const;
+ std::span<geo::latlng const> get_shape(trip_idx_t) const;
+ std::span<geo::latlng const> get_shape(trip_idx_t,
+ interval<stop_idx_t> const&) const;
+ std::pair<std::span<geo::latlng const>, int> get_shape_with_stop_count(
+ trip_idx_t, stop_idx_t) const;
+ shape_offset_idx_t add_offsets(std::vector<shape_offset_t> const&);
+ void add_trip_shape_offsets(
+ trip_idx_t, cista::pair<shape_idx_t, shape_offset_idx_t> const&);
+
+ mm_vecvec<shape_idx_t, geo::latlng> data_;
+ mm_vecvec<shape_offset_idx_t, shape_offset_t> offsets_;
+ mm_vec_map<trip_idx_t, cista::pair<shape_idx_t, shape_offset_idx_t>>
+ trip_offset_indices_;
+};
 
 } // namespace nigiri

include/nigiri/timetable.h

@@ -386,9 +386,6 @@ struct timetable {
  // Trip index -> all transports with a stop interval
  paged_vecvec<trip_idx_t, transport_range_t> trip_transport_ranges_;
 
- // Trip index -> shape per trip
- vector_map<trip_idx_t, shape_idx_t> trip_shape_indices_;
-
  // Transport -> stop sequence numbers (relevant for GTFS-RT stop matching)
  // Compaction:
  // - empty = zero-based sequence 0,1,2,...

include/nigiri/types.h

@@ -112,6 +112,9 @@ using optional = cista::optional<T>;
 template <typename Key, typename T, std::size_t N>
 using nvec = cista::raw::nvec<Key, T, N>;
 
+template <typename K, typename V>
+using mm_vec_map = cista::basic_mmap_vec<V, K>;
+
 template <typename T>
 using mm_vec = cista::basic_mmap_vec<T, std::uint64_t>;
 
@@ -134,6 +137,9 @@ using route_idx_t = cista::strong<std::uint32_t, struct _route_idx>;
 using section_idx_t = cista::strong<std::uint32_t, struct _section_idx>;
 using section_db_idx_t = cista::strong<std::uint32_t, struct _section_db_idx>;
 using shape_idx_t = cista::strong<std::uint32_t, struct _shape_idx>;
+using shape_offset_t = cista::strong<std::uint32_t, struct _shape_offset>;
+using shape_offset_idx_t =
+ cista::strong<std::uint32_t, struct _shape_offset_idx>;
 using trip_idx_t = cista::strong<std::uint32_t, struct _trip_idx>;
 using trip_id_idx_t = cista::strong<std::uint32_t, struct _trip_id_str_idx>;
 using transport_idx_t = cista::strong<std::uint32_t, struct _transport_idx>;
@@ -171,7 +177,6 @@ using attribute_combination_idx_t =
  cista::strong<std::uint32_t, struct _attribute_combination>;
 using provider_idx_t = cista::strong<std::uint32_t, struct _provider_idx>;
 
-using shapes_storage_t = mm_vecvec<shape_idx_t, geo::latlng>;
 using transport_range_t = pair<transport_idx_t, interval<stop_idx_t>>;
 
 struct trip_debug {

src/abi.cc

@@ -20,6 +20,7 @@
 #include "nigiri/rt/create_rt_timetable.h"
 #include "nigiri/rt/gtfsrt_update.h"
 #include "nigiri/rt/rt_timetable.h"
+#include "nigiri/shape.h"
 #include "nigiri/timetable.h"
 #include "nigiri/types.h"
 

src/loader/gtfs/load_timetable.cc

@@ -24,6 +24,7 @@
 #include "nigiri/loader/gtfs/route_key.h"
 #include "nigiri/loader/gtfs/services.h"
 #include "nigiri/loader/gtfs/shape.h"
+#include "nigiri/loader/gtfs/shape_prepare.h"
 #include "nigiri/loader/gtfs/stop.h"
 #include "nigiri/loader/gtfs/stop_seq_number_encoding.h"
 #include "nigiri/loader/gtfs/stop_time.h"
@@ -84,10 +85,10 @@ void load_timetable(loader_config const& config,
  dir const& d,
  timetable& tt,
  assistance_times* assistance,
- shapes_storage_t* shapes) {
+ shapes_storage* shapes_data) {
  auto local_bitfield_indices = hash_map<bitfield, bitfield_idx_t>{};
  load_timetable(config, src, d, tt, local_bitfield_indices, assistance,
- shapes);
+ shapes_data);
 }
 
 void load_timetable(loader_config const& config,
@@ -96,7 +97,7 @@ void load_timetable(loader_config const& config,
  timetable& tt,
  hash_map<bitfield, bitfield_idx_t>& bitfield_indices,
  assistance_times* assistance,
- shapes_storage_t* shapes) {
+ shapes_storage* shapes_data) {
  nigiri::scoped_timer const global_timer{"gtfs parser"};
 
  auto const load = [&](std::string_view file_name) -> file {
@@ -116,8 +117,9 @@ void load_timetable(loader_config const& config,
  auto const service =
  merge_traffic_days(tt.internal_interval_days(), calendar, dates);
  auto const shape_indices =
- (shapes != nullptr) ? parse_shapes(load(kShapesFile).data(), *shapes)
- : shape_id_map_t{};
+ (shapes_data != nullptr)
+ ? parse_shapes(load(kShapesFile).data(), *shapes_data)
+ : shape_id_map_t{};
  auto trip_data =
  read_trips(tt, routes, service, shape_indices, load(kTripsFile).data(),
  config.bikes_allowed_default_);
@@ -129,10 +131,10 @@ void load_timetable(loader_config const& config,
  for (auto& t : trip_data.data_) {
  if (t.requires_sorting_) {
  t.stop_headsigns_.resize(t.seq_numbers_.size());
- std::tie(t.seq_numbers_, t.stop_seq_, t.event_times_,
- t.stop_headsigns_) =
+ std::tie(t.seq_numbers_, t.stop_seq_, t.event_times_, t.stop_headsigns_,
+ t.distance_traveled_) =
  sort_by(t.seq_numbers_, t.stop_seq_, t.event_times_,
- t.stop_headsigns_);
+ t.stop_headsigns_, t.distance_traveled_);
  }
  }
  }
@@ -203,7 +205,7 @@ void load_timetable(loader_config const& config,
 
  {
  progress_tracker->status("Expand Trips")
- .out_bounds(70.F, 85.F)
+ .out_bounds(68.F, 83.F)
  .in_high(trip_data.data_.size());
  auto const timer = scoped_timer{"loader.gtfs.trips.expand"};
 
@@ -218,7 +220,7 @@ void load_timetable(loader_config const& config,
 
  {
  progress_tracker->status("Stay Seated")
- .out_bounds(85.F, 87.F)
+ .out_bounds(83.F, 85.F)
  .in_high(route_services.size());
  auto const timer = scoped_timer{"loader.gtfs.trips.block_id"};
 
@@ -231,7 +233,7 @@ void load_timetable(loader_config const& config,
 
  {
  progress_tracker->status("Write Trips")
- .out_bounds(87.F, 100.F)
+ .out_bounds(85.F, 98.F)
  .in_high(route_services.size());
 
  auto const is_train_number = [](auto const& s) {
@@ -258,6 +260,9 @@ void load_timetable(loader_config const& config,
  {source_file_idx, trp.from_line_, trp.to_line_},
  train_nr, stop_seq_numbers);
  }
+ if (shapes_data != nullptr) {
+ calculate_shape_offsets(tt, *shapes_data, trip_data.data_, shape_indices);
+ }
 
  auto const timer = scoped_timer{"loader.gtfs.routes.build"};
  auto const attributes = std::basic_string<attribute_combination_idx_t>{};
@@ -370,7 +375,6 @@ void load_timetable(loader_config const& config,
  // Build transport ranges.
  for (auto const& t : trip_data.data_) {
  tt.trip_transport_ranges_.emplace_back(t.transport_ranges_);
- tt.trip_shape_indices_.push_back(t.shape_idx_);
  }
  }
 }

src/loader/gtfs/loader.cc

@@ -15,9 +15,9 @@ void gtfs_loader::load(
  timetable& tt,
  hash_map<bitfield, bitfield_idx_t>& global_bitfield_indices,
  assistance_times* assistance,
- shapes_storage_t* shapes) const {
+ shapes_storage* shapes_data) const {
  return nigiri::loader::gtfs::load_timetable(
- c, src, d, tt, global_bitfield_indices, assistance, shapes);
+ c, src, d, tt, global_bitfield_indices, assistance, shapes_data);
 }
 
 cista::hash_t gtfs_loader::hash(dir const& d) const {