From 11c99d9d8a057bd0eb46009f34de8a99c0983afc Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Thu, 7 Sep 2023 15:20:16 -0300 Subject: [PATCH 1/8] + Lines for raw data. Compatibility between Underpass and Raw Data API --- docs/osmstats.md | 4 +- docs/statistics.md | 4 +- python/dbapi/api/raw.py | 73 ++++-- python/dbapi/api/report.py | 14 +- python/restapi/main.py | 10 + setup/underpass.sql | 55 +++-- src/osm/changeset.cc | 10 +- src/osm/changeset.hh | 6 +- src/osm/osmchange.cc | 34 +-- src/osm/osmchange.hh | 4 +- src/osm/osmobjects.cc | 4 +- src/osm/osmobjects.hh | 4 +- src/raw/queryraw.cc | 60 +++-- src/raw/queryraw.hh | 3 + src/replicator/planetreplicator.cc | 2 +- src/replicator/planetreplicator.hh | 2 +- src/replicator/replication.cc | 2 +- src/replicator/threads.cc | 2 +- src/stats/querystats.cc | 10 +- src/stats/querystats.hh | 4 +- .../libunderpass.all/areafilter-test.cc | 4 +- src/testsuite/libunderpass.all/change-test.cc | 20 +- .../libunderpass.all/hashtags-test.cc | 4 +- src/testsuite/libunderpass.all/stats-test.cc | 6 +- src/testsuite/libunderpass.all/val-test.cc | 4 +- src/testsuite/testdata/stats/107235440.yaml | 2 +- src/testsuite/testdata/stats/highway.yaml | 2 +- src/testsuite/testdata/stats/test_stats.yaml | 2 +- .../testdata/stats/test_statsconfig2.yaml | 2 +- .../testdata/stats/test_statsconfig3.yaml | 2 +- src/testsuite/testdata/test_stats.yaml | 2 +- src/underpass.cc | 2 +- src/validate/hotosm.cc | 4 +- src/validate/queryvalidate.cc | 8 +- src/validate/validate.hh | 16 +- src/wrappers/python.cc | 4 +- utils/clean-osmchanges.sql | 2 +- utils/raw-underpass.lua | 216 +++++++++++++----- utils/raw-underpass.sql | 12 +- 39 files changed, 408 insertions(+), 213 deletions(-) diff --git a/docs/osmstats.md b/docs/osmstats.md index 9d9219e4..d349dd2a 100644 --- a/docs/osmstats.md +++ b/docs/osmstats.md @@ -57,7 +57,7 @@ buildings_modified | This value is updated by counting the existing buildings mo pois_added | This value is updated by counting the POIs added by the user in the change file pois_modified | This value is updated by counting the existing POIs modified by the user in the change file editor | The editor used, and comes from the changeset -user_id | The user ID, comes from the changeset +uid | The user ID, comes from the changeset created_at | The timestamp this changeset was created, comes from the changeset closed_at | The timestamp this changeset was closed, comes from the changeset verified | Whether this data has been validated @@ -128,7 +128,7 @@ level | The badge level Keyword | Description --------|------------ -user_id | The OSM user ID +uid | The OSM user ID badge_id | The badge ID updated_at | The timestamp of the user receiving this badge diff --git a/docs/statistics.md b/docs/statistics.md index e8a0a4e7..a7a2d59b 100644 --- a/docs/statistics.md +++ b/docs/statistics.md @@ -211,7 +211,7 @@ backend and the frontend, without having modify the database schema. An example query to count the total number of buildings added by the user **4321** for a Tasking Manager project **1234** would be this: > SELECT SUM(CAST(added::hstore->'building' AS DOUBLE precision)) FROM -changesets WHERE 'hotosm-project-1234' = ANY(hashtags) AND user_id=4321; +changesets WHERE 'hotosm-project-1234' = ANY(hashtags) AND uid=4321; The source is the satellite imagery used for remote mapping. @@ -221,7 +221,7 @@ Keyword | Description --------|------------ id | The ID of this changeset editor | The editor used for this changeset -user_id | The OSM User ID of the mapper +uid | The OSM User ID of the mapper created_at | The timestamp when this changes was uploaded closed_at | The timestamp when this uploaded change completed processing updated_at | The timestamp when this last had data updated diff --git a/python/dbapi/api/raw.py b/python/dbapi/api/raw.py index 421b62aa..e7af4c04 100644 --- a/python/dbapi/api/raw.py +++ b/python/dbapi/api/raw.py @@ -35,8 +35,8 @@ def getPolygons( page = None ): query = "with t_ways AS ( \ - SELECT raw_poly.osm_id as id, raw_poly.timestamp, geometry, tags, status FROM raw_poly \ - LEFT JOIN validation ON validation.osm_id = raw_poly.osm_id \ + SELECT ways_poly.osm_id as id, ways_poly.timestamp, geom as geometry, tags, status FROM ways_poly \ + LEFT JOIN validation ON validation.osm_id = ways_poly.osm_id \ WHERE \ {0} {1} {2} {3} \ ), \ @@ -45,10 +45,37 @@ def getPolygons( - 'geometry' , 'geometry', ST_AsGeoJSON(geometry)::jsonb ) AS feature FROM t_ways \ ) SELECT jsonb_build_object( 'type', 'FeatureCollection', 'features', jsonb_agg(t_features.feature) ) \ as result FROM t_features;".format( - "ST_Intersects(\"geometry\", ST_GeomFromText('POLYGON(({0}))', 4326) )".format(area) if area else "1=1 ", - "and raw_poly.tags ? '{0}'".format(key) if key and not value else "", - "and raw_poly.tags->'{0}' ~* '^{1}'".format(key, value) if key and value else "", - "ORDER BY raw_poly.timestamp DESC LIMIT " + str(RESULTS_PER_PAGE) + " OFFSET {0}".format(page * RESULTS_PER_PAGE) if page else "", + "ST_Intersects(\"geom\", ST_GeomFromText('POLYGON(({0}))', 4326) )".format(area) if area else "1=1 ", + "and ways_poly.tags ? '{0}'".format(key) if key and not value else "", + "and ways_poly.tags->'{0}' ~* '^{1}'".format(key, value) if key and value else "", + "ORDER BY ways_poly.timestamp DESC LIMIT " + str(RESULTS_PER_PAGE) + " OFFSET {0}".format(page * RESULTS_PER_PAGE) if page else "", + ) + return self.underpassDB.run(query, responseType, True) + + def getLines( + self, + area = None, + key = None, + value = None, + hashtag = None, + responseType = "json", + page = None + ): + query = "with t_ways AS ( \ + SELECT ways_line.osm_id as id, ways_line.timestamp, geom as geometry, tags, status FROM ways_line \ + LEFT JOIN validation ON validation.osm_id = ways_line.osm_id \ + WHERE \ + {0} {1} {2} {3} \ + ), \ + t_features AS ( \ + SELECT jsonb_build_object( 'type', 'Feature', 'id', id, 'properties', to_jsonb(t_ways) \ + - 'geometry' , 'geometry', ST_AsGeoJSON(geometry)::jsonb ) AS feature FROM t_ways \ + ) SELECT jsonb_build_object( 'type', 'FeatureCollection', 'features', jsonb_agg(t_features.feature) ) \ + as result FROM t_features;".format( + "ST_Intersects(\"geom\", ST_GeomFromText('POLYGON(({0}))', 4326) )".format(area) if area else "1=1 ", + "and ways_line.tags ? '{0}'".format(key) if key and not value else "", + "and ways_line.tags->'{0}' ~* '^{1}'".format(key, value) if key and value else "", + "ORDER BY ways_line.timestamp DESC LIMIT " + str(RESULTS_PER_PAGE) + " OFFSET {0}".format(page * RESULTS_PER_PAGE) if page else "", ) return self.underpassDB.run(query, responseType, True) @@ -61,10 +88,10 @@ def getNodes( responseType = "json" ): query = "with t_nodes AS ( \ - SELECT raw_node.osm_id as id, geometry, tags, status FROM raw_node \ - LEFT JOIN validation ON validation.osm_id = raw_node.osm_id \ + SELECT nodes.osm_id as id, geom as geometry, tags, status FROM nodes \ + LEFT JOIN validation ON validation.osm_id = nodes.osm_id \ WHERE \ - ST_Intersects(\"geometry\", \ + ST_Intersects(\"geom\", \ ST_GeomFromText('POLYGON(({0}))', 4326) \ ) {1} {2} \ ), \ @@ -74,8 +101,8 @@ def getNodes( ) SELECT jsonb_build_object( 'type', 'FeatureCollection', 'features', jsonb_agg(t_features.feature) ) \ as result FROM t_features;".format( area, - "and raw_node.tags ? '{0}'".format(key) if key and not value else "", - "and raw_node.tags->'{0}' ~* '^{1}'".format(key, value) if key and value else "", + "and nodes.tags ? '{0}'".format(key) if key and not value else "", + "and nodes.tags->'{0}' ~* '^{1}'".format(key, value) if key and value else "", ) return self.underpassDB.run(query, responseType, True) @@ -92,17 +119,17 @@ def getPolygonsList( page = 1 query = "with t_ways AS ( \ - SELECT raw_poly.osm_id as id, ST_X(ST_Centroid(geometry)) as lat, ST_Y(ST_Centroid(geometry)) as lon, raw_poly.timestamp, tags, status FROM raw_poly \ - LEFT JOIN validation ON validation.osm_id = raw_poly.osm_id \ + SELECT ways_poly.osm_id as id, ST_X(ST_Centroid(geom)) as lat, ST_Y(ST_Centroid(geom)) as lon, ways_poly.timestamp, tags, status FROM ways_poly \ + LEFT JOIN validation ON validation.osm_id = ways_poly.osm_id \ WHERE \ {0} {1} {2} {3} \ ), t_features AS ( \ SELECT to_jsonb(t_ways) as feature from t_ways \ ) SELECT jsonb_agg(t_features.feature) as result FROM t_features;".format( - "ST_Intersects(\"geometry\", ST_GeomFromText('POLYGON(({0}))', 4326) )".format(area) if area else "1=1 ", - "and raw_poly.tags ? '{0}'".format(key) if key and not value else "", - "and raw_poly.tags->'{0}' ~* '^{1}'".format(key, value) if key and value else "", - "ORDER BY raw_poly.timestamp DESC LIMIT " + str(RESULTS_PER_PAGE) + " OFFSET {0}".format(page * RESULTS_PER_PAGE) if page else "", + "ST_Intersects(\"geom\", ST_GeomFromText('POLYGON(({0}))', 4326) )".format(area) if area else "1=1 ", + "and ways_poly.tags ? '{0}'".format(key) if key and not value else "", + "and ways_poly.tags->'{0}' ~* '^{1}'".format(key, value) if key and value else "", + "ORDER BY ways_poly.timestamp DESC LIMIT " + str(RESULTS_PER_PAGE) + " OFFSET {0}".format(page * RESULTS_PER_PAGE) if page else "", ) return self.underpassDB.run(query, responseType, True) @@ -119,16 +146,16 @@ def getNodesList( page = 1 query = "with t_nodes AS ( \ - SELECT raw_node.osm_id as id, ST_X(ST_Centroid(geometry)) as lat, ST_Y(ST_Centroid(geometry)) as lon, tags, status FROM raw_node \ - LEFT JOIN validation ON validation.osm_id = raw_node.osm_id \ + SELECT nodes.osm_id as id, ST_X(ST_Centroid(geom)) as lat, ST_Y(ST_Centroid(geom)) as lon, tags, status FROM nodes \ + LEFT JOIN validation ON validation.osm_id = nodes.osm_id \ WHERE {0} {1} {2} {3} \ ), \ t_features AS ( \ SELECT to_jsonb(t_nodes) AS feature FROM t_nodes \ ) SELECT jsonb_agg(t_features.feature) as result FROM t_features;".format( - "ST_Intersects(\"geometry\", ST_GeomFromText('POLYGON(({0}))', 4326) )".format(area) if area else "1=1 ", - "and raw_node.tags ? '{0}'".format(key) if key and not value else "", - "and raw_node.tags->'{0}' ~* '^{1}'".format(key, value) if key and value else "", - "ORDER BY raw_node.timestamp DESC LIMIT " + str(RESULTS_PER_PAGE) + " OFFSET {0}".format(page * RESULTS_PER_PAGE) if page else "", + "ST_Intersects(\"geom\", ST_GeomFromText('POLYGON(({0}))', 4326) )".format(area) if area else "1=1 ", + "and nodes.tags ? '{0}'".format(key) if key and not value else "", + "and nodes.tags->'{0}' ~* '^{1}'".format(key, value) if key and value else "", + "ORDER BY nodes.timestamp DESC LIMIT " + str(RESULTS_PER_PAGE) + " OFFSET {0}".format(page * RESULTS_PER_PAGE) if page else "", ) return self.underpassDB.run(query, responseType, True) \ No newline at end of file diff --git a/python/dbapi/api/report.py b/python/dbapi/api/report.py index 97b39648..36654370 100644 --- a/python/dbapi/api/report.py +++ b/python/dbapi/api/report.py @@ -42,7 +42,7 @@ def getDataQualityGeo( st_y(location) as lon \ from changesets \ INNER JOIN validation \ - ON validation.change_id = changesets.id \ + ON validation.changeset = changesets.id \ where validation.status = 'badgeom' \ {0} {1} {2} {3} \ order by osm_id \ @@ -91,16 +91,16 @@ def getDataQualityTag( {0} {1} {2} {3} \ ), \ t1 AS ( \ - SELECT change_id, source, osm_id, type, \ + SELECT changeset, source, osm_id, type, \ unnest(values) as unnest_values \ from validation, t2 \ - where change_id = t2.id \ + where changeset = t2.id \ ) \ select \ 'https://osm.org/' || t1.type || '/' || t1.osm_id as link, \ t1.unnest_values as tag, t1.source \ FROM t1, t2 \ - where t1.change_id = t2.id \ + where t1.changeset = t2.id \ limit {4} offset {5}".format( "and closed_at >= '{0}'".format(fromDate) if fromDate else "", "and closed_at <= '{0}'".format(toDate) if toDate else "", @@ -127,15 +127,15 @@ def getDataQualityTagStats( {0} {1} {2} {3} \ ), \ t1 AS ( \ - SELECT change_id, source, \ + SELECT changeset, source, \ unnest(values) as unnest_values \ from validation, t2 \ - where change_id = t2.id \ + where changeset = t2.id \ ) \ SELECT \ t1.unnest_values as tag, t1.source, count(t1.unnest_values) \ FROM t1, t2 \ - where t1.change_id = t2.id \ + where t1.changeset = t2.id \ group by t1.unnest_values, t1.source \ order by count desc \ limit {4} offset {5}".format( diff --git a/python/restapi/main.py b/python/restapi/main.py index a7544013..66db55d4 100644 --- a/python/restapi/main.py +++ b/python/restapi/main.py @@ -51,6 +51,7 @@ origins = [ "http://localhost", "http://localhost:5000", + "http://localhost:3000", "http://127.0.0.1", "http://127.0.0.1:5000" ] @@ -171,6 +172,15 @@ def getNodes(request: RawRequest): ) return results +@app.post("/raw/lines") +def getLines(request: RawRequest): + results = rawer.getLines( + area = request.area, + key = request.key or "", + value = request.value or "" + ) + return results + @app.post("/raw/polygonsList") def getPolygonsList(request: RawRequest): results = rawer.getPolygonsList( diff --git a/setup/underpass.sql b/setup/underpass.sql index b3a8a6bf..7911367b 100644 --- a/setup/underpass.sql +++ b/setup/underpass.sql @@ -23,7 +23,7 @@ SET default_tablespace = ''; CREATE TABLE IF NOT EXISTS public.changesets ( id int8 NOT NULL, editor text, - user_id integer NOT NULL, + uid integer NOT NULL, created_at timestamptz, closed_at timestamptz, updated_at timestamptz, @@ -46,8 +46,8 @@ CREATE TYPE public.status AS ENUM ('notags', 'complete', 'incomplete', 'badvalue CREATE TABLE IF NOT EXISTS public.validation ( osm_id int8, - user_id int8, - change_id int8, + uid int8, + changeset int8, type public.objtype, status public.status, values text[], @@ -59,23 +59,39 @@ CREATE TABLE IF NOT EXISTS public.validation ( ALTER TABLE ONLY public.validation ADD CONSTRAINT validation_pkey PRIMARY KEY (osm_id, status, source); -CREATE TABLE IF NOT EXISTS public.raw_poly ( +CREATE TABLE IF NOT EXISTS public.ways_poly ( osm_id int8, - change_id int8, - geometry public.geometry(Polygon,4326), + changeset int8, + geom public.geometry(Polygon,4326), tags public.hstore, refs int8[], timestamp timestamp with time zone, - version int + version int, + "user" text, + uid int8 ); -CREATE TABLE IF NOT EXISTS public.raw_node ( +CREATE TABLE IF NOT EXISTS public.ways_line ( osm_id int8, - change_id int8, - geometry public.geometry(Point,4326), + changeset int8, + geom public.geometry(Line,4326), + tags public.hstore, + refs int8[], + timestamp timestamp with time zone, + version int, + "user" text, + uid int8 +); + +CREATE TABLE IF NOT EXISTS public.nodes ( + osm_id int8, + changeset int8, + geom public.geometry(Point,4326), tags public.hstore, timestamp timestamp with time zone, - version int + version int, + "user" text, + uid int8 ); CREATE TABLE IF NOT EXISTS public.way_refs ( @@ -83,7 +99,16 @@ CREATE TABLE IF NOT EXISTS public.way_refs ( node_id int8 ); -CREATE UNIQUE INDEX IF NOT EXISTS raw_osm_id_idx ON public.raw_node (osm_id); -CREATE UNIQUE INDEX IF NOT EXISTS raw_poly_osm_id_idx ON public.raw_poly (osm_id); -CREATE INDEX IF NOT EXISTS way_refs_nodes_idx ON public.way_refs (node_id); -CREATE INDEX IF NOT EXISTS way_refs_ways_idx ON public.way_refs (way_id); +CREATE UNIQUE INDEX nodes_id_idx ON public.nodes (osm_id); +CREATE UNIQUE INDEX ways_poly_id_idx ON public.ways_poly (osm_id); +CREATE UNIQUE INDEX ways_line_id_idx ON public.ways_line(osm_id); +CREATE INDEX way_refs_node_id_idx ON public.way_refs (node_id); +CREATE INDEX way_refs_way_id_idx ON public.way_refs (way_id); + +CREATE INDEX nodes_version_idx ON public.nodes (version); +CREATE INDEX ways_poly_version_idx ON public.ways_poly (version); +CREATE INDEX ways_line_version_idx ON public.ways_line (version); + +CREATE INDEX nodes_timestamp_idx ON public.nodes(timestamp DESC); +CREATE INDEX ways_poly_timestamp_idx ON public.ways_poly(timestamp DESC); +CREATE INDEX ways_line_timestamp_idx ON public.ways_line(timestamp DESC); diff --git a/src/osm/changeset.cc b/src/osm/changeset.cc index 4c0e5aa2..8a80738d 100644 --- a/src/osm/changeset.cc +++ b/src/osm/changeset.cc @@ -84,8 +84,8 @@ using namespace boost::gregorian; #include "utils/log.hh" using namespace logger; -/// \namespace changeset -namespace changeset { +/// \namespace changesets +namespace changesets { bool ChangeSetFile::readChanges(const std::vector &buffer) @@ -305,7 +305,7 @@ ChangeSetFile::readXML(std::istream &xml) for (auto value: pt.get_child("osm")) { if (value.first == "changeset") { - changeset::ChangeSet change; + changesets::ChangeSet change; // Process the tags. These don't exist for every element for (auto tag: value.second) { if (tag.first == "tag") { @@ -349,7 +349,7 @@ ChangeSetFile::on_start_element(const Glib::ustring &name, { // log_debug("Element %1%", name); if (name == "changeset") { - auto change = std::make_shared(attributes); + auto change = std::make_shared(attributes); changes.push_back(change); if (change->closed_at != not_a_date_time && (last_closed_at == not_a_date_time || change->closed_at > last_closed_at)) { last_closed_at = change->closed_at; @@ -436,4 +436,4 @@ ChangeSetFile::on_start_element(const Glib::ustring &name, } #endif // EOF LIBXML -} // namespace changeset +} // namespace changesets diff --git a/src/osm/changeset.hh b/src/osm/changeset.hh index 03eec492..d43d8829 100644 --- a/src/osm/changeset.hh +++ b/src/osm/changeset.hh @@ -61,8 +61,8 @@ namespace geoutil { class GeoUtil; }; -/// \namespace changeset -namespace changeset { +/// \namespace changesets +namespace changesets { /// \file changeset.hh @@ -179,7 +179,7 @@ class ChangeSetFile ptime last_closed_at = not_a_date_time; }; -} // namespace changeset +} // namespace changesets #endif // EOF __CHANGESET_HH__ diff --git a/src/osm/osmchange.cc b/src/osm/osmchange.cc index 73793b11..d5a9705c 100644 --- a/src/osm/osmchange.cc +++ b/src/osm/osmchange.cc @@ -360,7 +360,7 @@ OsmChangeFile::on_start_element(const Glib::ustring &name, } else if (attr_pair.name == "user") { change->obj->user = attr_pair.value; } else if (attr_pair.name == "changeset") { - change->obj->change_id = std::stol(attr_pair.value); + change->obj->changeset = std::stol(attr_pair.value); } else if (attr_pair.name == "lat") { auto lat = reinterpret_cast(change->obj.get()); lat->setLatitude(std::stod(attr_pair.value)); @@ -519,14 +519,14 @@ OsmChangeFile::collectStats(const multipolygon_t &poly) node->tags.find("created_at") != node->tags.end()) { continue; } - ostats = (*mstats)[node->change_id]; + ostats = (*mstats)[node->changeset]; if (ostats.get() == 0) { ostats = std::make_shared(); - ostats->change_id = node->change_id; - ostats->user_id = node->uid; + ostats->changeset = node->changeset; + ostats->uid = node->uid; ostats->username = node->user; ostats->closed_at = node->timestamp; - (*mstats)[node->change_id] = ostats; + (*mstats)[node->changeset] = ostats; } auto hits = scanTags(node->tags, osmchange::node); for (auto hit = std::begin(*hits); hit != std::end(*hits); ++hit) { @@ -557,14 +557,14 @@ OsmChangeFile::collectStats(const multipolygon_t &poly) if (way->tags.size() == 1 && way->tags.find("created_at") != way->tags.end()) { continue; } - ostats = (*mstats)[way->change_id]; + ostats = (*mstats)[way->changeset]; if (ostats.get() == 0) { ostats = std::make_shared(); - ostats->change_id = way->change_id; - ostats->user_id = way->uid; + ostats->changeset = way->changeset; + ostats->uid = way->uid; ostats->username = way->user; ostats->closed_at = way->timestamp; - (*mstats)[way->change_id] = ostats; + (*mstats)[way->changeset] = ostats; } auto hits = scanTags(way->tags, osmchange::way); @@ -597,7 +597,7 @@ OsmChangeFile::collectStats(const multipolygon_t &poly) } double length = boost::geometry::length(globe, boost::geometry::strategy::distance::haversine(6371.0)); - // log_debug("LENGTH: %1% %2%", std::to_string(length), way->change_id); + // log_debug("LENGTH: %1% %2%", std::to_string(length), way->changeset); ostats->added[tag] += length; } } @@ -613,14 +613,14 @@ OsmChangeFile::collectStats(const multipolygon_t &poly) if (relation->tags.size() == 0) { continue; } - ostats = (*mstats)[relation->change_id]; + ostats = (*mstats)[relation->changeset]; if (ostats.get() == 0) { ostats = std::make_shared(); - ostats->change_id = relation->change_id; - ostats->user_id = relation->uid; + ostats->changeset = relation->changeset; + ostats->uid = relation->uid; ostats->username = relation->user; ostats->closed_at = relation->timestamp; - (*mstats)[relation->change_id] = ostats; + (*mstats)[relation->changeset] = ostats; } auto hits = scanTags(relation->tags, osmchange::relation); for (auto hit = std::begin(*hits); hit != std::end(*hits); ++hit) { @@ -663,8 +663,8 @@ OsmChangeFile::scanTags(std::map tags, osmchange::osmt void ChangeStats::dump(void) { - std::cerr << "Dumping ChangeStats for: \t " << change_id << std::endl; - std::cerr << "\tUser ID: \t\t " << user_id << std::endl; + std::cerr << "Dumping ChangeStats for: \t " << changeset << std::endl; + std::cerr << "\tUser ID: \t\t " << uid << std::endl; std::cerr << "\tUser Name: \t\t " << username << std::endl; std::cerr << "\tAdded features: " << added.size() << std::endl; for (auto it = std::begin(added); it != std::end(added); ++it) { @@ -732,7 +732,7 @@ OsmChangeFile::validateWays(const multipolygon_t &poly, std::shared_ptrrefs.front() == way->refs.back()) { status->timestamp = boost::posix_time::microsec_clock::universal_time(); - status->user_id = way->uid; + status->uid = way->uid; // Overlapping if (plugin->overlaps(change->ways, *way)) { diff --git a/src/osm/osmchange.hh b/src/osm/osmchange.hh index 0f67a52c..e66c4984 100644 --- a/src/osm/osmchange.hh +++ b/src/osm/osmchange.hh @@ -68,8 +68,8 @@ typedef enum { empty, node, way, relation, member } osmtype_t; /// which later gets added to the database statistics. class ChangeStats { public: - long change_id = 0; ///< The ID of this change - long user_id = 0; ///< The User ID + long changeset = 0; ///< The ID of this change + long uid = 0; ///< The User ID std::string username; ///< The User Name ptime created_at; ///< The starting timestamp ptime closed_at; ///< The finished timestamp diff --git a/src/osm/osmobjects.cc b/src/osm/osmobjects.cc index a14bcc7b..9f9c0ae3 100644 --- a/src/osm/osmobjects.cc +++ b/src/osm/osmobjects.cc @@ -79,8 +79,8 @@ OsmObject::dump(void) const } else { std::cerr << "\tNot in Priority area" << std::endl; } - if (change_id > 0) { - std::cerr << "\tChange ID: " << std::to_string(change_id) << std::endl; + if (changeset > 0) { + std::cerr << "\tChange ID: " << std::to_string(changeset) << std::endl; } if (tags.size() > 0) { std::cerr << "\tTags: " << tags.size() << std::endl; diff --git a/src/osm/osmobjects.hh b/src/osm/osmobjects.hh index 92ac88e6..5f5db82a 100644 --- a/src/osm/osmobjects.hh +++ b/src/osm/osmobjects.hh @@ -72,7 +72,7 @@ class OsmObject { void setAction(action_t act) { action = act; }; void setUID(long val) { uid = val; }; - void setChangeID(long val) { change_id = val; }; + void setChangeID(long val) { changeset = val; }; action_t action = none; ///< the action that contains this object osmtype_t type = empty; ///< The type of this object, node, way, or relation @@ -81,7 +81,7 @@ class OsmObject { ptime timestamp; ///< The timestamp of this object's creation or modification long uid = 0; ///< The User ID of the mapper of this object std::string user; ///< The User name of the mapper of this object - long change_id = 0; ///< The changeset ID this object is contained in + long changeset = 0; ///< The changeset ID this object is contained in std::map tags; ///< OSM metadata tags bool priority = false; ///< Whether it's in the priority area diff --git a/src/raw/queryraw.cc b/src/raw/queryraw.cc index d4b1cc10..d7312e39 100644 --- a/src/raw/queryraw.cc +++ b/src/raw/queryraw.cc @@ -63,10 +63,10 @@ QueryRaw::applyChange(const OsmNode &node) const { std::string query; if (node.action == osmobjects::create || node.action == osmobjects::modify) { - query = "INSERT INTO raw_node as r (osm_id, geometry, tags, timestamp, version) VALUES("; - std::string format = "%d, ST_GeomFromText(\'%s\', 4326), %s, \'%s\', %d \ - ) ON CONFLICT (osm_id) DO UPDATE SET geometry = ST_GeomFromText(\'%s\', \ - 4326), tags = %s, timestamp = \'%s\', version = %d WHERE r.version < %d;"; + query = "INSERT INTO nodes as r (osm_id, geom, tags, timestamp, version, \"user\", uid, changeset) VALUES("; + std::string format = "%d, ST_GeomFromText(\'%s\', 4326), %s, \'%s\', %d, \'%s\', %d, %d \ + ) ON CONFLICT (osm_id) DO UPDATE SET geom = ST_GeomFromText(\'%s\', \ + 4326), tags = %s, timestamp = \'%s\', version = %d, \"user\" = \'%s\', uid = %d, changeset = %d WHERE r.version < %d;"; boost::format fmt(format); // osm_id @@ -99,36 +99,59 @@ QueryRaw::applyChange(const OsmNode &node) const fmt % timestamp; // version fmt % node.version; + // user + fmt % node.user; + // uid + fmt % node.uid; + // changeset + fmt % node.changeset; // ON CONFLICT fmt % geometry; fmt % tags; fmt % timestamp; fmt % node.version; + fmt % node.user; + fmt % node.uid; + fmt % node.changeset; fmt % node.version; query += fmt.str(); } else if (node.action == osmobjects::remove) { - query = "DELETE from raw_node where osm_id = " + std::to_string(node.id) + ";"; + query = "DELETE from nodes where osm_id = " + std::to_string(node.id) + ";"; } return query; } + +const std::string QueryRaw::polyTable = "ways_poly"; +const std::string QueryRaw::lineTable = "ways_line"; + std::string QueryRaw::applyChange(const OsmWay &way) const { std::string query = ""; - - if (way.refs.size() > 3 && (way.refs.front() == way.refs.back()) + const std::string* tableName; + + std::stringstream ss; + if (way.refs.size() > 3 && (way.refs.front() == way.refs.back())) { + tableName = &QueryRaw::polyTable; + ss << std::setprecision(12) << boost::geometry::wkt(way.polygon); + } else { + tableName = &QueryRaw::lineTable; + ss << std::setprecision(12) << boost::geometry::wkt(way.linestring); + } + std::string geostring = ss.str(); + + if (way.refs.size() > 2 && (way.action == osmobjects::create || way.action == osmobjects::modify)) { - if (way.refs.size() == boost::geometry::num_points(way.linestring)) { - query = "INSERT INTO raw_poly as r (osm_id, tags, refs, geometry, timestamp, version) VALUES("; + query = "INSERT INTO " + *tableName + " as r (osm_id, tags, refs, geom, timestamp, version) VALUES("; std::string format = "%d, %s, %s, %s, \'%s\', %d) \ - ON CONFLICT (osm_id) DO UPDATE SET tags = %s, refs = %s, geometry = %s, timestamp = \'%s\', version = %d WHERE r.version <= %d;"; + ON CONFLICT (osm_id) DO UPDATE SET tags = %s, refs = %s, geom = %s, timestamp = \'%s\', version = %d WHERE r.version <= %d;"; boost::format fmt(format); // osm_id @@ -163,9 +186,6 @@ QueryRaw::applyChange(const OsmWay &way) const // geometry std::string geometry; - std::stringstream ss; - ss << std::setprecision(12) << boost::geometry::wkt(way.polygon); - std::string geostring = ss.str(); geometry = "ST_GeomFromText(\'" + geostring + "\', 4326)"; fmt % geometry; @@ -192,7 +212,7 @@ QueryRaw::applyChange(const OsmWay &way) const } } else if (way.action == osmobjects::remove) { query += "DELETE FROM way_refs WHERE way_id=" + std::to_string(way.id) + ";"; - query += "DELETE FROM raw_poly where osm_id = " + std::to_string(way.id) + ";"; + query += "DELETE FROM " + *tableName + " where osm_id = " + std::to_string(way.id) + ";"; } return query; @@ -273,7 +293,7 @@ void QueryRaw::getNodeCache(std::shared_ptr osmchanges, const mul if (referencedNodeIds.size() > 1) { referencedNodeIds.erase(referencedNodeIds.size() - 1); // Get Nodes from DB - std::string nodesQuery = "SELECT osm_id, st_x(geometry) as lat, st_y(geometry) as lon FROM raw_node where osm_id in (" + referencedNodeIds + ");"; + std::string nodesQuery = "SELECT osm_id, st_x(geom) as lat, st_y(geom) as lon FROM nodes where osm_id in (" + referencedNodeIds + ");"; auto result = dbconn->query(nodesQuery); // Fill nodecache for (auto node_it = result.begin(); node_it != result.end(); ++node_it) { @@ -321,7 +341,7 @@ QueryRaw::getNodeCacheFromWays(std::shared_ptr> ways, std::m nodeIds.erase(nodeIds.size() - 1); // Get Nodes from DB - std::string nodesQuery = "SELECT osm_id, st_x(geometry) as lat, st_y(geometry) as lon FROM raw_node where osm_id in (" + nodeIds + ") and st_x(geometry) is not null and st_y(geometry) is not null;"; + std::string nodesQuery = "SELECT osm_id, st_x(geom) as lat, st_y(geom) as lon FROM nodes where osm_id in (" + nodeIds + ") and st_x(geom) is not null and st_y(geom) is not null;"; auto result = dbconn->query(nodesQuery); // Fill nodecache for (auto node_it = result.begin(); node_it != result.end(); ++node_it) { @@ -361,7 +381,7 @@ QueryRaw::getWaysByNodesRefs(std::string &nodeIds) const // Get all ways that have references to nodes std::list> ways; - std::string waysQuery = "SELECT distinct(osm_id), refs, version, tags from way_refs join raw_poly rp on rp.osm_id = way_id where node_id = any(ARRAY[" + nodeIds + "]);"; + std::string waysQuery = "SELECT distinct(osm_id), refs, version, tags from way_refs join ways_poly rp on rp.osm_id = way_id where node_id = any(ARRAY[" + nodeIds + "]);"; auto ways_result = dbconn->query(waysQuery); // Fill vector of OsmWay objects @@ -387,14 +407,14 @@ QueryRaw::getWaysByNodesRefs(std::string &nodeIds) const } int QueryRaw::getWaysCount() { - std::string query = "select count(osm_id) from raw_poly"; + std::string query = "select count(osm_id) from ways_poly"; auto result = dbconn->query(query); return result[0][0].as(); } std::shared_ptr> QueryRaw::getWaysFromDB(int lastid) { - std::string waysQuery = "SELECT osm_id, refs, ST_AsText(ST_ExteriorRing(geometry), 4326), version, tags FROM raw_poly where osm_id > " + std::to_string(lastid) + " order by osm_id asc limit 500;"; + std::string waysQuery = "SELECT osm_id, refs, ST_AsText(ST_ExteriorRing(geom), 4326), version, tags FROM ways_poly where osm_id > " + std::to_string(lastid) + " order by osm_id asc limit 500;"; auto ways_result = dbconn->query(waysQuery); // Fill vector of OsmWay objects @@ -426,7 +446,7 @@ QueryRaw::getWaysFromDB(int lastid) { std::shared_ptr QueryRaw::getWayById(long id) { - std::string waysQuery = "SELECT osm_id, refs, version FROM raw_poly where osm_id=" + std::to_string(id) + ";"; + std::string waysQuery = "SELECT osm_id, refs, version FROM ways_poly where osm_id=" + std::to_string(id) + ";"; auto result = dbconn->query(waysQuery); // Fill vector of OsmWay objects diff --git a/src/raw/queryraw.hh b/src/raw/queryraw.hh index 7ba79eae..51961b3f 100644 --- a/src/raw/queryraw.hh +++ b/src/raw/queryraw.hh @@ -56,6 +56,9 @@ class QueryRaw { ~QueryRaw(void){}; QueryRaw(std::shared_ptr db); + static const std::string polyTable; + static const std::string lineTable; + /// Build query for processed Node std::string applyChange(const OsmNode &node) const; /// Build query for processed Way diff --git a/src/replicator/planetreplicator.cc b/src/replicator/planetreplicator.cc index 5430cf52..94731950 100644 --- a/src/replicator/planetreplicator.cc +++ b/src/replicator/planetreplicator.cc @@ -75,7 +75,7 @@ using namespace underpassconfig; using namespace logger; // Forward declarations -namespace changeset { +namespace changesets { class ChangeSet; }; diff --git a/src/replicator/planetreplicator.hh b/src/replicator/planetreplicator.hh index 544ab41f..c6deff59 100644 --- a/src/replicator/planetreplicator.hh +++ b/src/replicator/planetreplicator.hh @@ -53,7 +53,7 @@ class PlanetReplicator : public replication::Planet { private: std::vector default_minutes; std::vector default_changesets; - std::shared_ptr changes; ///< All the changes in the file + std::shared_ptr changes; ///< All the changes in the file std::shared_ptr> hashes; ///< Existing hashtags }; diff --git a/src/replicator/replication.cc b/src/replicator/replication.cc index afac0afe..6f1174ca 100644 --- a/src/replicator/replication.cc +++ b/src/replicator/replication.cc @@ -217,7 +217,7 @@ StateFile::isValid() const bool Replication::readChanges(const std::string &file) { - changeset::ChangeSetFile changeset; + changesets::ChangeSetFile changeset; std::ifstream stream; stream.open(file, std::ifstream::in); changeset.readXML(stream); diff --git a/src/replicator/threads.cc b/src/replicator/threads.cc index 90c6987f..f0790845 100644 --- a/src/replicator/threads.cc +++ b/src/replicator/threads.cc @@ -374,7 +374,7 @@ threadChangeSet(std::shared_ptr &remote, task.status = file.status; if (file.status == reqfile_t::success) { - auto changeset = std::make_unique(); + auto changeset = std::make_unique(); log_debug("Processing ChangeSet: %1%", remote->filespec); auto xml = planet->processData(remote->filespec, *file.data); std::istream& input(xml); diff --git a/src/stats/querystats.cc b/src/stats/querystats.cc index 20a23e80..21011723 100644 --- a/src/stats/querystats.cc +++ b/src/stats/querystats.cc @@ -105,7 +105,7 @@ QueryStats::applyChange(const osmchange::ChangeStats &change) const // Some of the data field in the changset come from a different file, // which may not be downloaded yet. ptime now = boost::posix_time::microsec_clock::universal_time(); - std::string aquery = "INSERT INTO changesets (id, user_id, closed_at, updated_at, "; + std::string aquery = "INSERT INTO changesets (id, uid, closed_at, updated_at, "; if (change.added.size() > 0) { aquery += "added, "; @@ -116,8 +116,8 @@ QueryStats::applyChange(const osmchange::ChangeStats &change) const aquery.erase(aquery.size() - 2); aquery += ")"; - aquery += " VALUES(" + std::to_string(change.change_id) + ", "; - aquery += std::to_string(change.user_id) + ", "; + aquery += " VALUES(" + std::to_string(change.changeset) + ", "; + aquery += std::to_string(change.uid) + ", "; aquery += "\'" + to_simple_string(change.closed_at) + "\', "; aquery += "\'" + to_simple_string(now) + "\', "; @@ -152,13 +152,13 @@ QueryStats::applyChange(const osmchange::ChangeStats &change) const } std::string -QueryStats::applyChange(const changeset::ChangeSet &change) const +QueryStats::applyChange(const changesets::ChangeSet &change) const { #ifdef TIMING_DEBUG_X boost::timer::auto_cpu_timer timer("applyChange(changeset): took %w seconds\n"); #endif - std::string query = "INSERT INTO changesets (id, editor, user_id, created_at, closed_at, updated_at"; + std::string query = "INSERT INTO changesets (id, editor, uid, created_at, closed_at, updated_at"; if (change.hashtags.size() > 0) { query += ", hashtags "; diff --git a/src/stats/querystats.hh b/src/stats/querystats.hh index c52ce370..2ab7c4f0 100644 --- a/src/stats/querystats.hh +++ b/src/stats/querystats.hh @@ -51,7 +51,7 @@ using namespace boost::gregorian; using namespace pq; // Forward declarations -namespace changeset { +namespace changesets { class ChangeSet; }; namespace osmchange { @@ -74,7 +74,7 @@ class QueryStats { ~QueryStats(void){}; QueryStats(std::shared_ptr db); /// Build query for processed ChangeSet - std::string applyChange(const changeset::ChangeSet &change) const; + std::string applyChange(const changesets::ChangeSet &change) const; /// Build query for processed OsmChange std::string applyChange(const osmchange::ChangeStats &change) const; // Database connection, used for escape strings diff --git a/src/testsuite/libunderpass.all/areafilter-test.cc b/src/testsuite/libunderpass.all/areafilter-test.cc index efd24497..da34a20a 100644 --- a/src/testsuite/libunderpass.all/areafilter-test.cc +++ b/src/testsuite/libunderpass.all/areafilter-test.cc @@ -28,7 +28,7 @@ TestState runtest; using namespace logger; -class TestChangeset : public changeset::ChangeSetFile {}; +class TestChangeset : public changesets::ChangeSetFile {}; class TestOsmChange : public osmchange::OsmChangeFile {}; int @@ -92,7 +92,7 @@ main(int argc, char *argv[]) osmchangeFile += "/testsuite/testdata/areafilter-test.osm"; TestChangeset changeset; TestOsmChange osmchange; - changeset::ChangeSet *testChangeset; + changesets::ChangeSet *testChangeset; // ChangeSet - Whole world changeset.readChanges(changesetFile); diff --git a/src/testsuite/libunderpass.all/change-test.cc b/src/testsuite/libunderpass.all/change-test.cc index edc28a0d..f3a319ff 100644 --- a/src/testsuite/libunderpass.all/change-test.cc +++ b/src/testsuite/libunderpass.all/change-test.cc @@ -39,7 +39,7 @@ namespace opts = boost::program_options; using namespace logger; -using namespace changeset; +using namespace changesets; using namespace boost::posix_time; using namespace boost::gregorian; @@ -67,7 +67,7 @@ using namespace boost::gregorian; TestState runtest; -class TestCS : public changeset::ChangeSetFile { +class TestCS : public changesets::ChangeSetFile { }; class TestCO : public osmchange::OsmChangeFile { @@ -110,7 +110,7 @@ main(int argc, char *argv[]) if (vm.count("changefile")) { std::string file = vm["changefile"].as(); std::cout << "Importing change file " << file << std::endl; - auto changeset = std::make_shared(); + auto changeset = std::make_shared(); changeset->readChanges(file); changeset->dump(); exit(0); @@ -174,13 +174,13 @@ main(int argc, char *argv[]) std::map changeset_ids_found; for (const auto &change: testco.changes) { for (const auto &node: change->nodes) { - changeset_ids_found.insert(std::pair(node->change_id, node->change_id)); + changeset_ids_found.insert(std::pair(node->changeset, node->changeset)); } for (const auto &way: change->ways) { - changeset_ids_found.insert(std::pair(way->change_id, way->change_id)); + changeset_ids_found.insert(std::pair(way->changeset, way->changeset)); } for (const auto &relation: change->relations) { - changeset_ids_found.insert(std::pair(relation->change_id, relation->change_id)); + changeset_ids_found.insert(std::pair(relation->changeset, relation->changeset)); } } bool all_changesets_tracked = true; @@ -198,14 +198,14 @@ main(int argc, char *argv[]) auto tf = testco.changes.front(); auto tnf = tf->nodes.front(); - if (tnf->change_id == 99069702 && tnf->id == 5776216755) { + if (tnf->changeset == 99069702 && tnf->id == 5776216755) { runtest.pass("ChangeSetFile::readChanges(first change, first node)"); } else { runtest.fail("ChangeSetFile::readChanges(first change, first node)"); } auto twf = tf->ways.front(); // twf->dump(); - if (twf->change_id == 99069879L && twf->id == 474556695L && + if (twf->changeset == 99069879L && twf->id == 474556695L && twf->uid == 1041828L) { runtest.pass("ChangeSetFile::readChanges(first change, first way)"); } else { @@ -213,14 +213,14 @@ main(int argc, char *argv[]) } auto tnb = tf->nodes.back(); // tnb->dump(); - if (tnb->change_id == 94802322L && tnb->id == 289112823L) { + if (tnb->changeset == 94802322L && tnb->id == 289112823L) { runtest.pass("ChangeSetFile::readChanges(first change, last node)"); } else { runtest.fail("ChangeSetFile::readChanges(first change, last node)"); } auto twb = tf->ways.back(); // twb->dump(); - if (twb->change_id == 99063443L && twb->id == 67365141L && + if (twb->changeset == 99063443L && twb->id == 67365141L && twb->uid == 1137406L) { runtest.pass("ChangeSetFile::readChanges(first change, last way)"); } else { diff --git a/src/testsuite/libunderpass.all/hashtags-test.cc b/src/testsuite/libunderpass.all/hashtags-test.cc index 15545818..e117826d 100644 --- a/src/testsuite/libunderpass.all/hashtags-test.cc +++ b/src/testsuite/libunderpass.all/hashtags-test.cc @@ -28,7 +28,7 @@ TestState runtest; using namespace logger; -class TestChangeset : public changeset::ChangeSetFile {}; +class TestChangeset : public changesets::ChangeSetFile {}; int main(int argc, char *argv[]) @@ -38,7 +38,7 @@ main(int argc, char *argv[]) std::string changesetFile(DATADIR); changesetFile += "/testsuite/testdata/hashtags-test.osc"; TestChangeset changeset; - changeset::ChangeSet *change; + changesets::ChangeSet *change; changeset.readChanges(changesetFile); multipolygon_t polyEmpty; changeset.areaFilter(polyEmpty); diff --git a/src/testsuite/libunderpass.all/stats-test.cc b/src/testsuite/libunderpass.all/stats-test.cc index 164e7a8b..b119670c 100644 --- a/src/testsuite/libunderpass.all/stats-test.cc +++ b/src/testsuite/libunderpass.all/stats-test.cc @@ -65,7 +65,7 @@ class TestStats { std::string jsonstr = ""; for (auto it = std::begin(*stats); it != std::end(*stats); ++it) { auto changestats = it->second; - jsonstr += "{\"changeset_id\":" + std::to_string(changestats->change_id) + ", "; + jsonstr += "{\"changeset_id\":" + std::to_string(changestats->changeset) + ", "; jsonstr += "\"filespec\": \"" + filespec + "\" "; if (changestats->added.size() > 0) { @@ -233,9 +233,9 @@ class TestStats { for (auto it = std::begin(*stats); it != std::end(*stats); ++it) { auto changestats = it->second; - if (changestats->change_id == validation.at("change_id")) { + if (changestats->changeset == validation.at("changeset")) { if (this->verbose) { - std::cout << "change_id: " << changestats->change_id << std::endl; + std::cout << "changeset: " << changestats->changeset << std::endl; } // TODO: get values to test from YAML validation file testStat(changestats, validation, "highway"); diff --git a/src/testsuite/libunderpass.all/val-test.cc b/src/testsuite/libunderpass.all/val-test.cc index aa4afcce..36bf7fa2 100644 --- a/src/testsuite/libunderpass.all/val-test.cc +++ b/src/testsuite/libunderpass.all/val-test.cc @@ -159,7 +159,7 @@ test_semantic_building(std::shared_ptr &plugin) { osmobjects::OsmNode node; node.id = 11111; - node.change_id = 22222; + node.changeset = 22222; // Node with no tags status = plugin->checkPOI(node, "building"); @@ -171,7 +171,7 @@ test_semantic_building(std::shared_ptr &plugin) { osmobjects::OsmNode node_place; node_place.id = 11111; - node_place.change_id = 22222; + node_place.changeset = 22222; // Has valid tags, but it's incomplete node_place.addTag("place", "city"); diff --git a/src/testsuite/testdata/stats/107235440.yaml b/src/testsuite/testdata/stats/107235440.yaml index ae6245de..81fbdd36 100644 --- a/src/testsuite/testdata/stats/107235440.yaml +++ b/src/testsuite/testdata/stats/107235440.yaml @@ -1,4 +1,4 @@ -- change_id: +- changeset: - 107235440 - modified_highway: - 8 diff --git a/src/testsuite/testdata/stats/highway.yaml b/src/testsuite/testdata/stats/highway.yaml index 92c44f9a..f6ce0077 100644 --- a/src/testsuite/testdata/stats/highway.yaml +++ b/src/testsuite/testdata/stats/highway.yaml @@ -1,4 +1,4 @@ -- change_id: +- changeset: - 1 - added_highway: - 2 diff --git a/src/testsuite/testdata/stats/test_stats.yaml b/src/testsuite/testdata/stats/test_stats.yaml index 676ced58..6623d303 100644 --- a/src/testsuite/testdata/stats/test_stats.yaml +++ b/src/testsuite/testdata/stats/test_stats.yaml @@ -1,4 +1,4 @@ -- change_id: +- changeset: - 1 - modified_highway: - 2 diff --git a/src/testsuite/testdata/stats/test_statsconfig2.yaml b/src/testsuite/testdata/stats/test_statsconfig2.yaml index a83e3e45..75f146b4 100644 --- a/src/testsuite/testdata/stats/test_statsconfig2.yaml +++ b/src/testsuite/testdata/stats/test_statsconfig2.yaml @@ -1,4 +1,4 @@ -- change_id: +- changeset: - 1 - added_building: - 1 diff --git a/src/testsuite/testdata/stats/test_statsconfig3.yaml b/src/testsuite/testdata/stats/test_statsconfig3.yaml index fdca3065..d0374c56 100644 --- a/src/testsuite/testdata/stats/test_statsconfig3.yaml +++ b/src/testsuite/testdata/stats/test_statsconfig3.yaml @@ -1,4 +1,4 @@ -- change_id: +- changeset: - 1 - added_building: - 2 diff --git a/src/testsuite/testdata/test_stats.yaml b/src/testsuite/testdata/test_stats.yaml index 7da36cd5..b150dc05 100644 --- a/src/testsuite/testdata/test_stats.yaml +++ b/src/testsuite/testdata/test_stats.yaml @@ -1,4 +1,4 @@ -- change_id: +- changeset: - 1 - modified_highway: - 2 diff --git a/src/underpass.cc b/src/underpass.cc index a3351af9..2de1e3dd 100644 --- a/src/underpass.cc +++ b/src/underpass.cc @@ -77,7 +77,7 @@ using namespace underpassconfig; using namespace logger; // Forward declarations -namespace changeset { +namespace changesets { class ChangeSet; }; diff --git a/src/validate/hotosm.cc b/src/validate/hotosm.cc index f1b58989..37639837 100644 --- a/src/validate/hotosm.cc +++ b/src/validate/hotosm.cc @@ -110,7 +110,7 @@ Hotosm::checkPOI(const osmobjects::OsmNode &node, const std::string &type) { auto status = std::make_shared(node); status->timestamp = boost::posix_time::microsec_clock::universal_time(); - status->user_id = node.uid; + status->uid = node.uid; if (yamls.size() == 0) { log_error("No config files!"); @@ -176,7 +176,7 @@ Hotosm::checkWay(const osmobjects::OsmWay &way, const std::string &type) auto status = std::make_shared(way); status->timestamp = boost::posix_time::microsec_clock::universal_time(); - status->user_id = way.uid; + status->uid = way.uid; if (yamls.size() == 0) { log_error("No config files!"); diff --git a/src/validate/queryvalidate.cc b/src/validate/queryvalidate.cc index b90a4fa3..dcbe8933 100644 --- a/src/validate/queryvalidate.cc +++ b/src/validate/queryvalidate.cc @@ -129,17 +129,17 @@ QueryValidate::applyChange(const ValidateStatus &validation, const valerror_t &s std::string query; if (validation.values.size() > 0) { - query = "INSERT INTO validation as v (osm_id, change_id, user_id, type, status, values, timestamp, location, source, version) VALUES("; + query = "INSERT INTO validation as v (osm_id, changeset, uid, type, status, values, timestamp, location, source, version) VALUES("; format = "%d, %d, %g, \'%s\', \'%s\', ARRAY[%s], \'%s\', ST_GeomFromText(\'%s\', 4326), \'%s\', %s) "; } else { - query = "INSERT INTO validation as v (osm_id, change_id, user_id, type, status, timestamp, location, source, version) VALUES("; + query = "INSERT INTO validation as v (osm_id, changeset, uid, type, status, timestamp, location, source, version) VALUES("; format = "%d, %d, %g, \'%s\', \'%s\', \'%s\', ST_GeomFromText(\'%s\', 4326), \'%s\', %s) "; } format += "ON CONFLICT (osm_id, status, source) DO UPDATE SET version = %d, timestamp = \'%s\' WHERE v.version < %d;"; boost::format fmt(format); fmt % validation.osm_id; - fmt % validation.change_id; - fmt % validation.user_id; + fmt % validation.changeset; + fmt % validation.uid; fmt % objtypes[validation.objtype]; std::string valtmp; fmt % status_list[status]; diff --git a/src/validate/validate.hh b/src/validate/validate.hh index ea0555e2..5ec3788a 100644 --- a/src/validate/validate.hh +++ b/src/validate/validate.hh @@ -104,16 +104,16 @@ class ValidateStatus { ValidateStatus(void){}; ValidateStatus(const osmobjects::OsmNode &node) { osm_id = node.id; - user_id = node.uid; - change_id = node.change_id; + uid = node.uid; + changeset = node.changeset; version = node.version; objtype = osmobjects::node; timestamp = node.timestamp; } ValidateStatus(const osmobjects::OsmWay &way) { osm_id = way.id; - user_id = way.uid; - change_id = way.change_id; + uid = way.uid; + changeset = way.changeset; objtype = osmobjects::way; version = way.version; timestamp = way.timestamp; @@ -130,8 +130,8 @@ class ValidateStatus { void dump(void) const { std::cerr << "Dumping Validation Statistics" << std::endl; std::cerr << "\tOSM ID: " << osm_id << std::endl; - std::cerr << "\tUser ID: " << user_id << std::endl; - std::cerr << "\tChange ID: " << change_id << std::endl; + std::cerr << "\tUser ID: " << uid << std::endl; + std::cerr << "\tChangeset: " << changeset << std::endl; std::map results; results[notags] = "No tags"; @@ -156,8 +156,8 @@ class ValidateStatus { std::unordered_set status; osmobjects::osmtype_t objtype; long osm_id = 0; ///< The OSM ID of the feature - long user_id = 0; ///< The user ID of the mapper creating/modifying this feature - long change_id = 0; ///< The changeset ID + long uid = 0; ///< The user ID of the mapper creating/modifying this feature + long changeset = 0; ///< The changeset ID long version = 0; ///< The object version ptime timestamp; ///< The timestamp when this validation was performed point_t center; ///< The centroid of the building polygon diff --git a/src/wrappers/python.cc b/src/wrappers/python.cc index 12ff30ea..83ac89ad 100644 --- a/src/wrappers/python.cc +++ b/src/wrappers/python.cc @@ -69,8 +69,8 @@ std::string dumpJSON(ValidateStatus& self) { std::string output = ""; output += "{\n"; output += "\t\"osm_id\":" + std::to_string(self.osm_id) + ",\n"; - output += "\t\"user_id\":" + std::to_string(self.user_id) + ",\n"; - output += "\t\"change_id\":" + std::to_string(self.change_id) + ",\n"; + output += "\t\"uid\":" + std::to_string(self.uid) + ",\n"; + output += "\t\"changeset\":" + std::to_string(self.changeset) + ",\n"; if (self.status.size() > 0) { output += "\t\"results\": ["; diff --git a/utils/clean-osmchanges.sql b/utils/clean-osmchanges.sql index 430fc65a..78b7b97c 100644 --- a/utils/clean-osmchanges.sql +++ b/utils/clean-osmchanges.sql @@ -6,7 +6,7 @@ We run this query every 24 hours for cleaning the database when running the replicator process with areaFilter disabled for osmchanges (--osmnoboundary). */ -DELETE FROM validation v where v.change_id in ( +DELETE FROM validation v where v.changeset in ( SELECT id from changesets WHERE bbox is NULL AND closed_at < NOW() - INTERVAL '24 HOURS' diff --git a/utils/raw-underpass.lua b/utils/raw-underpass.lua index 578eb589..09018f52 100644 --- a/utils/raw-underpass.lua +++ b/utils/raw-underpass.lua @@ -1,65 +1,117 @@ --- Copyright (c) 2020, 2021, 2023 Humanitarian OpenStreetMap Team --- --- This file is part of Underpass. --- --- Underpass is free software: you can redistribute it and/or modify --- it under the terms of the GNU General Public License as published by --- the Free Software Foundation, either version 3 of the License, or --- (at your option) any later version. --- --- Underpass is distributed in the hope that it will be useful, --- but WITHOUT ANY WARRANTY; without even the implied warranty of --- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the --- GNU General Public License for more details. --- --- You should have received a copy of the GNU General Public License --- along with Underpass. If not, see . +-- # Copyright (c) 2020, 2021, 2023 Humanitarian OpenStreetMap Team +-- # This program is free software: you can redistribute it and/or modify +-- # it under the terms of the GNU Affero General Public License as +-- # published by the Free Software Foundation, either version 3 of the +-- # License, or (at your option) any later version. + +-- # This program is distributed in the hope that it will be useful, +-- # but WITHOUT ANY WARRANTY; without even the implied warranty of +-- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-- # GNU Affero General Public License for more details. + +-- # You should have received a copy of the GNU Affero General Public License +-- # along with this program. If not, see . + +-- # Humanitarian OpenStreetmap Team +-- # 1100 13th Street NW Suite 800 Washington, D.C. 20005 +-- # -- This is lua script for osm2pgsql in order to create and process custom schema to store incoming osm data efficiently --- osm2pgsql -H postgis -U underpass -p underpass -P 5432 -d underpass --extra-attributes --output=flex --style ./raw-underpass.lua pokhara_all.osm.pbf +-- osm2pgsql --create -H localhost -U admin -P 5432 -d postgres -W --extra-attributes --output=flex --style ./raw.lua nepal-latest-internal.osm.pbf + -- Set projection to 4326 local srid = 4326 local tables = {} -tables.raw_node = osm2pgsql.define_table{ - name="raw_node", +tables.nodes = osm2pgsql.define_table{ + name="nodes", -- This will generate a derived nodes table which stores all the nodes feature with their point geometry - ids = {type='node',id_column = 'osm_id'}, + ids = {type='node',id_column = 'osm_id' }, columns = { + { column = 'uid', type = 'int' }, + { column = 'user', type = 'text' }, { column = 'version', type = 'int' }, + { column = 'changeset', type = 'int' }, { column = 'timestamp', sql_type = 'timestamp' }, + -- { column = 'tags', type = 'jsonb' }, { column = 'tags', sql_type = 'public.hstore' }, - { column = 'geometry', type = 'point', projection = srid }, + { column = 'geom', type = 'point', projection = srid }, + -- { column = 'country', sql_type= 'int[]', create_only = true }, } } -tables.raw_poly = osm2pgsql.define_table{ - name="raw_poly", - -- This will generate a derived polygon table which stores all the ways feature without their geometry - ids = {type='way',id_column = 'osm_id'}, +tables.ways_line = osm2pgsql.define_table{ + name="ways_line", + -- This will generate a derived ways line table which stores all the ways feature with linestring geometry + ids = {type='way',id_column = 'osm_id' }, columns = { + { column = 'uid', type = 'int' }, + { column = 'user', type = 'text' }, { column = 'version', type = 'int' }, + { column = 'changeset', type = 'int' }, { column = 'timestamp', sql_type = 'timestamp' }, + -- { column = 'tags', type = 'jsonb' }, { column = 'tags', sql_type = 'public.hstore' }, - { column = 'refs', type= 'text', sql_type = 'int8[]'}, - { column = 'geometry', type = 'polygon', projection = srid } + { column = 'refs', type= 'text', sql_type = 'bigint[]'}, + { column = 'geom', type = 'linestring', projection = srid }, + -- { column = 'country', sql_type= 'int[]', create_only = true }, + } + } ---tables.raw_line = osm2pgsql.define_table{ --- name="raw_line", - -- This will generate a derived nodes table which stores all the nodes feature with their point geometry --- ids = {type='way',id_column = 'osm_id'}, --- columns = { --- { column = 'version', type = 'int' }, --- { column = 'timestamp', sql_type = 'timestamp' }, --- { column = 'tags', sql_type = 'public.hstore' }, --- { column = 'refs', type= 'text', sql_type = 'int8[]'}, --- } ---} +tables.ways_poly = osm2pgsql.define_table{ + name="ways_poly", + -- This will generate a derived ways poly table which stores all the ways feature with polygon geometry + ids = {type='way',id_column = 'osm_id' }, + columns = { + { column = 'uid', type = 'int' }, + { column = 'user', type = 'text' }, + { column = 'version', type = 'int' }, + { column = 'changeset', type = 'int' }, + { column = 'timestamp', sql_type = 'timestamp' }, + -- This will store tags as jsonb type + -- { column = 'tags', type = 'jsonb' }, + { column = 'tags', sql_type = 'public.hstore' }, + { column = 'refs', type= 'text', sql_type = 'bigint[]'}, + { column = 'geom', type = 'polygon', projection = srid }, + -- { column = 'grid', type = 'int', create_only = true }, + -- { column = 'country', sql_type= 'int[]', create_only = true }, + } + +} + +tables.rels = osm2pgsql.define_table{ + name="relations", + -- This will generate a derived realtion table which stores all the relation feature to query without storing meta data parts and members + + ids = {type='relation', id_column = 'osm_id' }, + columns = { + + { column = 'uid', type = 'int' }, + { column = 'user', type = 'text' }, + { column = 'version', type = 'int' }, + { column = 'changeset', type = 'int' }, + { column = 'timestamp', sql_type = 'timestamp' }, + -- { column = 'tags', type = 'jsonb' }, + { column = 'tags', sql_type = 'public.hstore' }, + { column = 'refs', type = 'jsonb'}, + { column = 'geom', type = 'geometry', projection = srid }, + -- { column = 'country',sql_type= 'int[]', create_only = true }, + + } +} + +-- Returns true if there are no tags left. +function clean_tags(tags) + tags.odbl = nil + -- tags.created_by = nil + tags['source:ref'] = nil + return next(tags) == nil +end function tags_to_hstore(tags) local hstore = '' @@ -67,32 +119,90 @@ function tags_to_hstore(tags) hstore = hstore .. string.format('%s=>%s,', string.format('%q', k), string.format('%q', v)) end return hstore:sub(1, -2) - end +end function osm2pgsql.process_node(object) - tables.raw_node:add_row({ + + if clean_tags(object.tags) then + return + end + + tables.nodes:add_row({ + uid = object.uid, + user = object.user, version = object.version, - tags = tags_to_hstore(object.tags), + changeset = object.changeset, timestamp = os.date('!%Y-%m-%dT%H:%M:%SZ', object.timestamp), - geometry = { create = 'point' }, + -- tags = object.tags, + tags = tags_to_hstore(object.tags), + geom = { create = 'point' } }) end -function osm2pgsql.process_way(object) +function osm2pgsql.process_way(object) + if clean_tags(object.tags) then + return + end + if object.is_closed and #object.nodes>3 then - tables.raw_poly:add_row({ + tables.ways_poly:add_row({ + uid = object.uid, + user = object.user, version = object.version, + changeset = object.changeset, + timestamp = os.date('!%Y-%m-%dT%H:%M:%SZ', object.timestamp), + -- tags = object.tags, tags = tags_to_hstore(object.tags), + refs = '{' .. table.concat(object.nodes, ',') .. '}', + geom = { create = 'area' }, + + }) + else + tables.ways_line:add_row({ + uid = object.uid, + user = object.user, + version = object.version, + changeset = object.changeset, timestamp = os.date('!%Y-%m-%dT%H:%M:%SZ', object.timestamp), + -- tags = object.tags, + tags = tags_to_hstore(object.tags), refs = '{' .. table.concat(object.nodes, ',') .. '}', - geometry = { create = 'area' }, + geom = { create = 'line' }, + + }) + end +end + +function osm2pgsql.process_relation(object) + if clean_tags(object.tags) then + return + end + if object.tags.type == 'multipolygon' or object.tags.type == 'boundary' then + tables.rels:add_row({ + uid = object.uid, + user = object.user, + version = object.version, + changeset = object.changeset, + timestamp = os.date('!%Y-%m-%dT%H:%M:%SZ', object.timestamp), + -- tags = object.tags, + tags = tags_to_hstore(object.tags), + geom = { create = 'area' }, + refs = object.members + + }) + else + tables.rels:add_row({ + uid = object.uid, + user = object.user, + version = object.version, + changeset = object.changeset, + timestamp = os.date('!%Y-%m-%dT%H:%M:%SZ', object.timestamp), + -- tags = object.tags, + tags = tags_to_hstore(object.tags), + geom= { create = 'line' }, + refs = object.members + }) - --else - -- tables.raw_line:add_row({ - -- version = object.version, - -- tags = tags_to_hstore(object.tags), - -- timestamp = os.date('!%Y-%m-%dT%H:%M:%SZ', object.timestamp), - -- refs = '{' .. table.concat(object.nodes, ',') .. '}', - -- }) end -end \ No newline at end of file +end + diff --git a/utils/raw-underpass.sql b/utils/raw-underpass.sql index d69ba8d7..20fdd286 100644 --- a/utils/raw-underpass.sql +++ b/utils/raw-underpass.sql @@ -1,10 +1,10 @@ -CREATE UNIQUE INDEX raw_osm_id_idx ON public.raw_node (osm_id); -CREATE UNIQUE INDEX raw_poly_osm_id_idx ON public.raw_poly (osm_id); +CREATE UNIQUE INDEX nodes_id_idx ON public.nodes (osm_id); +CREATE UNIQUE INDEX ways_poly_id_idx ON public.ways_poly (osm_id); CREATE INDEX way_refs_node_id_idx ON public.way_refs (node_id); CREATE INDEX way_refs_way_id_idx ON public.way_refs (way_id); -CREATE INDEX node_version_idx ON public.raw_node (version); -CREATE INDEX way_version_idx ON public.raw_poly (version); +CREATE INDEX nodes_version_idx ON public.nodes (version); +CREATE INDEX ways_poly_version_idx ON public.ways_poly (version); -CREATE INDEX node_timestamp_idx ON public.raw_node(timestamp DESC); -CREATE INDEX way_timestamp_idx ON public.raw_poly(timestamp DESC); +CREATE INDEX nodes_timestamp_idx ON public.nodes(timestamp DESC); +CREATE INDEX ways_poly_timestamp_idx ON public.ways_poly(timestamp DESC); From 4b2e7ed264b835a3f019cd68f1af8e4ee4a48c1e Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Thu, 7 Sep 2023 17:16:32 -0300 Subject: [PATCH 2/8] Fix for field type --- setup/underpass.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/underpass.sql b/setup/underpass.sql index 7911367b..089de92d 100644 --- a/setup/underpass.sql +++ b/setup/underpass.sql @@ -74,7 +74,7 @@ CREATE TABLE IF NOT EXISTS public.ways_poly ( CREATE TABLE IF NOT EXISTS public.ways_line ( osm_id int8, changeset int8, - geom public.geometry(Line,4326), + geom public.geometry(LineString,4326), tags public.hstore, refs int8[], timestamp timestamp with time zone, From 7c47e157d6987184a21b7fa43d496e173fb79be6 Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Thu, 7 Sep 2023 17:40:45 -0300 Subject: [PATCH 3/8] Fix for deleting way from both line and poly tables --- src/raw/queryraw.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/raw/queryraw.cc b/src/raw/queryraw.cc index d7312e39..7c45ccb5 100644 --- a/src/raw/queryraw.cc +++ b/src/raw/queryraw.cc @@ -212,7 +212,8 @@ QueryRaw::applyChange(const OsmWay &way) const } } else if (way.action == osmobjects::remove) { query += "DELETE FROM way_refs WHERE way_id=" + std::to_string(way.id) + ";"; - query += "DELETE FROM " + *tableName + " where osm_id = " + std::to_string(way.id) + ";"; + query += "DELETE FROM " + QueryRaw::polyTable + " where osm_id = " + std::to_string(way.id) + ";"; + query += "DELETE FROM " + QueryRaw::lineTable + " where osm_id = " + std::to_string(way.id) + ";"; } return query; From b6e38b691efc4f3bcf8b9de4b9278e122260e7da Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Sat, 9 Sep 2023 23:19:13 -0300 Subject: [PATCH 4/8] Add missing code for processing lines along polygons --- src/bootstrap/bootstrap.cc | 63 +++++++++++++++++++++++++++++++++++--- src/bootstrap/bootstrap.hh | 3 +- src/raw/queryraw.cc | 31 ++++++------------- src/raw/queryraw.hh | 6 ++-- utils/bootstrap.sh | 25 ++++++++++++--- 5 files changed, 91 insertions(+), 37 deletions(-) diff --git a/src/bootstrap/bootstrap.cc b/src/bootstrap/bootstrap.cc index ea070d1d..f431eb05 100644 --- a/src/bootstrap/bootstrap.cc +++ b/src/bootstrap/bootstrap.cc @@ -68,8 +68,31 @@ void startProcessingWays(const underpassconfig::UnderpassConfig &config) { auto queryvalidate = std::make_shared(db); auto queryraw = std::make_shared(db); - int total = queryraw->getWaysCount(); - + // TODO: improve duplicate code + std::cout << "Processing polygons ..." << std::endl; + int total = queryraw->getWaysCount(QueryRaw::polyTable); + if (total > 0) { + int count = 0; + long lastid = 0; + while (count < total) { + int percentage = (count * 100) / total; + auto task = std::make_shared(); + WayTask wayTask; + wayTask.plugin = validator; + wayTask.queryvalidate = queryvalidate; + wayTask.queryraw = queryraw; + wayTask.task = task; + wayTask.lastid = lastid; + processWaysPoly(wayTask); + db->query(task->query); + lastid = wayTask.lastid; + count += wayTask.processed; + std::cout << "\r" << "Processing : " << count << "/" << total << " (" << percentage << "%)"; + } + } + + std::cout << "Processing lines ..." << std::endl; + total = queryraw->getWaysCount(QueryRaw::lineTable); if (total > 0) { int count = 0; long lastid = 0; @@ -82,7 +105,7 @@ void startProcessingWays(const underpassconfig::UnderpassConfig &config) { wayTask.queryraw = queryraw; wayTask.task = task; wayTask.lastid = lastid; - processWays(wayTask); + processWaysLine(wayTask); db->query(task->query); lastid = wayTask.lastid; count += wayTask.processed; @@ -94,7 +117,7 @@ void startProcessingWays(const underpassconfig::UnderpassConfig &config) { // This thread get started for every page of way void -processWays(WayTask &wayTask) +processWaysPoly(WayTask &wayTask) { #ifdef TIMING_DEBUG boost::timer::auto_cpu_timer timer("bootstrap::processWays(wayTask): took %w seconds\n"); @@ -106,7 +129,7 @@ processWays(WayTask &wayTask) auto queryraw = wayTask.queryraw; auto lastid = wayTask.lastid; - auto ways = queryraw->getWaysFromDB(lastid); + auto ways = queryraw->getWaysFromDB(lastid, QueryRaw::polyTable); wayTask.processed = ways->size(); // Proccesing ways @@ -135,4 +158,34 @@ processWays(WayTask &wayTask) } +void +processWaysLine(WayTask &wayTask) +{ +#ifdef TIMING_DEBUG + boost::timer::auto_cpu_timer timer("bootstrap::processWays(wayTask): took %w seconds\n"); +#endif + + auto plugin = wayTask.plugin; + auto task = wayTask.task; + auto queryvalidate = wayTask.queryvalidate; + auto queryraw = wayTask.queryraw; + auto lastid = wayTask.lastid; + + auto ways = queryraw->getWaysFromDB(lastid, QueryRaw::lineTable); + wayTask.processed = ways->size(); + + // Proccesing ways + for (auto way = ways->begin(); way != ways->end(); ++way) { + if (way->refs.front() == way->refs.back()) { + log_debug("Way Id: %1%", way->id); + // Fill the way_refs table + for (auto ref = way->refs.begin(); ref != way->refs.end(); ++ref) { + task->query += "INSERT INTO way_refs (way_id, node_id) VALUES (" + std::to_string(way->id) + "," + std::to_string(*ref) + "); "; + } + } + } + wayTask.lastid = ways->back().id; + +} + } \ No newline at end of file diff --git a/src/bootstrap/bootstrap.hh b/src/bootstrap/bootstrap.hh index daf66839..3debf2d6 100644 --- a/src/bootstrap/bootstrap.hh +++ b/src/bootstrap/bootstrap.hh @@ -47,6 +47,7 @@ struct WayTask { void startProcessingWays(const underpassconfig::UnderpassConfig &config); // This thread get started for every page of way -void processWays(WayTask &wayTask); +void processWaysPoly(WayTask &wayTask); +void processWaysLine(WayTask &wayTask); } \ No newline at end of file diff --git a/src/raw/queryraw.cc b/src/raw/queryraw.cc index 7c45ccb5..e128c44b 100644 --- a/src/raw/queryraw.cc +++ b/src/raw/queryraw.cc @@ -317,7 +317,9 @@ void QueryRaw::getNodeCache(std::shared_ptr osmchanges, const mul boost::geometry::append(way->linestring, osmchanges->nodecache.at(*rit)); } } - way->polygon = { {std::begin(way->linestring), std::end(way->linestring)} }; + if (way->isClosed()) { + way->polygon = { {std::begin(way->linestring), std::end(way->linestring)} }; + } } } @@ -382,7 +384,8 @@ QueryRaw::getWaysByNodesRefs(std::string &nodeIds) const // Get all ways that have references to nodes std::list> ways; - std::string waysQuery = "SELECT distinct(osm_id), refs, version, tags from way_refs join ways_poly rp on rp.osm_id = way_id where node_id = any(ARRAY[" + nodeIds + "]);"; + // TODO: add ways_line to the query + std::string waysQuery = "SELECT distinct(osm_id), refs, version, tags from way_refs join ways_poly rp on rp.osm_id = way_id where node_id = any(ARRAY[" + nodeIds + "]);"; auto ways_result = dbconn->query(waysQuery); // Fill vector of OsmWay objects @@ -407,15 +410,15 @@ QueryRaw::getWaysByNodesRefs(std::string &nodeIds) const return ways; } -int QueryRaw::getWaysCount() { - std::string query = "select count(osm_id) from ways_poly"; +int QueryRaw::getWaysCount(const std::string &tableName) { + std::string query = "select count(osm_id) from " + tableName; auto result = dbconn->query(query); return result[0][0].as(); } std::shared_ptr> -QueryRaw::getWaysFromDB(int lastid) { - std::string waysQuery = "SELECT osm_id, refs, ST_AsText(ST_ExteriorRing(geom), 4326), version, tags FROM ways_poly where osm_id > " + std::to_string(lastid) + " order by osm_id asc limit 500;"; +QueryRaw::getWaysFromDB(int lastid, const std::string &tableName) { + std::string waysQuery = "SELECT osm_id, refs, ST_AsText(ST_ExteriorRing(geom), 4326), version, tags FROM " + tableName + " where osm_id > " + std::to_string(lastid) + " order by osm_id asc limit 500;"; auto ways_result = dbconn->query(waysQuery); // Fill vector of OsmWay objects @@ -445,22 +448,6 @@ QueryRaw::getWaysFromDB(int lastid) { return ways; } -std::shared_ptr -QueryRaw::getWayById(long id) { - std::string waysQuery = "SELECT osm_id, refs, version FROM ways_poly where osm_id=" + std::to_string(id) + ";"; - auto result = dbconn->query(waysQuery); - - // Fill vector of OsmWay objects - OsmWay way; - way.id = result[0][0].as(); - way.version = result[0][2].as(); - std::string refs_str = result[0][1].as(); - if (refs_str.size() > 1) { - way.refs = arrayStrToVector(refs_str); - } - return std::make_shared(way); -} - } // namespace queryraw // local Variables: diff --git a/src/raw/queryraw.hh b/src/raw/queryraw.hh index 51961b3f..b47559f6 100644 --- a/src/raw/queryraw.hh +++ b/src/raw/queryraw.hh @@ -74,11 +74,9 @@ class QueryRaw { // DB connection std::shared_ptr dbconn; // Get ways count - int getWaysCount(); + int getWaysCount(const std::string &tableName); // Get ways by page - std::shared_ptr> getWaysFromDB(int page); - // Get single way by id - std::shared_ptr getWayById(long id); + std::shared_ptr> getWaysFromDB(int page, const std::string &tableName); }; diff --git a/utils/bootstrap.sh b/utils/bootstrap.sh index 043e2581..a60115ff 100644 --- a/utils/bootstrap.sh +++ b/utils/bootstrap.sh @@ -25,7 +25,7 @@ localfiles='false' -while getopts r:c:h::u:p:d:l flag +while getopts r:c:h::u:p:d:l:k flag do case "${flag}" in r) region=${OPTARG};; @@ -35,6 +35,7 @@ do p) port=${OPTARG};; d) database=${OPTARG};; l) localfiles=${OPTARG};; + k) use_docker=${OPTARG};; esac done @@ -65,6 +66,10 @@ then then echo "Use local files?: yes" fi + if [ -z "${use_docker}" ] + then + echo "Use Docker?: yes" + fi echo " " echo "*** WARNING: THIS ACTION WILL OVERWRITE DATA IN THE CURRENT DATABASE ***" @@ -99,11 +104,20 @@ then PGPASSWORD=$PASS psql --host $HOST --user $USER --port $PORT $DB < raw-underpass.sql echo "Configuring Underpass ..." - python3 poly2geojson.py $COUNTRY.poly && \ - docker cp $COUNTRY.geojson underpass:/usr/local/lib/underpass/config/ - docker cp $COUNTRY.geojson underpass:/code/config + python3 poly2geojson.py $COUNTRY.poly + if [ -z "${use_docker}" ] + docker cp $COUNTRY.geojson underpass:/usr/local/lib/underpass/config/priority.geojson + docker cp $COUNTRY.geojson underpass:/code/config/priority.geojson + then + cp $COUNTRY.geojson underpass:/usr/local/lib/underpass/config/priority.geojson + cp $COUNTRY.geojson underpass:/code/config/priority.geojson + fi echo "Bootstrapping database ..." - docker exec -w /code/build -t underpass ./underpass --bootstrap + if [ -z "${use_docker}" ] + docker exec -w /code/build -t underpass ./underpass --bootstrap + then + cd ../build ./underpass --bootstrap + fi echo "Done." echo " " fi @@ -125,4 +139,5 @@ else echo " -p port (Database port)" echo " -d database (Database name)" echo " -l (Use local files instead of download them)" + echo " -k (Use Docker Underpass installation)" fi From 9d1cb9a13b2d4bc5a1d74e08e36913134e3abb91 Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Sat, 9 Sep 2023 23:27:38 -0300 Subject: [PATCH 5/8] Fix to bootstrap script --- utils/bootstrap.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/bootstrap.sh b/utils/bootstrap.sh index a60115ff..ff4e149a 100644 --- a/utils/bootstrap.sh +++ b/utils/bootstrap.sh @@ -24,6 +24,7 @@ # ----- localfiles='false' +use_docker='false' while getopts r:c:h::u:p:d:l:k flag do @@ -87,7 +88,7 @@ then fi echo "Cleaning database ..." - PGPASSWORD=$PASS psql --host $HOST --user $USER --port $PORT $DB -c 'DROP TABLE IF EXISTS raw_poly; DROP TABLE IF EXISTS raw_node; DROP TABLE IF EXISTS way_refs; DROP TABLE IF EXISTS validation; DROP TABLE IF EXISTS changesets;' + PGPASSWORD=$PASS psql --host $HOST --user $USER --port $PORT $DB -c 'DROP TABLE IF EXISTS ways_poly; DROP TABLE IF EXISTS ways_line; DROP TABLE IF EXISTS nodes; DROP TABLE IF EXISTS way_refs; DROP TABLE IF EXISTS validation; DROP TABLE IF EXISTS changesets;' PGPASSWORD=$PASS psql --host $HOST --user $USER --port $PORT $DB --file '../setup/underpass.sql' if [ -z "${localfiles}" ] From d105542adfb0c3174c09c80638ab4df29e51cae8 Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Sat, 9 Sep 2023 23:36:55 -0300 Subject: [PATCH 6/8] Fix for bootstrap script --- utils/bootstrap.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/utils/bootstrap.sh b/utils/bootstrap.sh index ff4e149a..accd051e 100644 --- a/utils/bootstrap.sh +++ b/utils/bootstrap.sh @@ -107,16 +107,18 @@ then echo "Configuring Underpass ..." python3 poly2geojson.py $COUNTRY.poly if [ -z "${use_docker}" ] + then docker cp $COUNTRY.geojson underpass:/usr/local/lib/underpass/config/priority.geojson docker cp $COUNTRY.geojson underpass:/code/config/priority.geojson - then + else cp $COUNTRY.geojson underpass:/usr/local/lib/underpass/config/priority.geojson cp $COUNTRY.geojson underpass:/code/config/priority.geojson fi echo "Bootstrapping database ..." if [ -z "${use_docker}" ] - docker exec -w /code/build -t underpass ./underpass --bootstrap then + docker exec -w /code/build -t underpass ./underpass --bootstrap + else cd ../build ./underpass --bootstrap fi echo "Done." From 0345563fe758626c43365d29f8850fd1c2071f68 Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Sun, 10 Sep 2023 08:30:14 -0300 Subject: [PATCH 7/8] Fixes for adding lines --- src/bootstrap/bootstrap.cc | 52 ++++++++++++++++++-------------------- src/raw/queryraw.cc | 38 ++++++++++++++++++---------- utils/raw-underpass.sql | 4 +++ 3 files changed, 53 insertions(+), 41 deletions(-) diff --git a/src/bootstrap/bootstrap.cc b/src/bootstrap/bootstrap.cc index f431eb05..4185baf9 100644 --- a/src/bootstrap/bootstrap.cc +++ b/src/bootstrap/bootstrap.cc @@ -69,30 +69,30 @@ void startProcessingWays(const underpassconfig::UnderpassConfig &config) { auto queryraw = std::make_shared(db); // TODO: improve duplicate code - std::cout << "Processing polygons ..." << std::endl; int total = queryraw->getWaysCount(QueryRaw::polyTable); - if (total > 0) { - int count = 0; - long lastid = 0; - while (count < total) { - int percentage = (count * 100) / total; - auto task = std::make_shared(); - WayTask wayTask; - wayTask.plugin = validator; - wayTask.queryvalidate = queryvalidate; - wayTask.queryraw = queryraw; - wayTask.task = task; - wayTask.lastid = lastid; - processWaysPoly(wayTask); - db->query(task->query); - lastid = wayTask.lastid; - count += wayTask.processed; - std::cout << "\r" << "Processing : " << count << "/" << total << " (" << percentage << "%)"; - } - } + std::cout << "Processing polygons (" + std::to_string(total) + ")..." << std::endl; + // if (total > 0) { + // int count = 0; + // long lastid = 0; + // while (count < total) { + // int percentage = (count * 100) / total; + // auto task = std::make_shared(); + // WayTask wayTask; + // wayTask.plugin = validator; + // wayTask.queryvalidate = queryvalidate; + // wayTask.queryraw = queryraw; + // wayTask.task = task; + // wayTask.lastid = lastid; + // processWaysPoly(wayTask); + // db->query(task->query); + // lastid = wayTask.lastid; + // count += wayTask.processed; + // std::cout << "\r" << "Processing : " << count << "/" << total << " (" << percentage << "%)"; + // } + // } - std::cout << "Processing lines ..." << std::endl; total = queryraw->getWaysCount(QueryRaw::lineTable); + std::cout << "Processing lines (" + std::to_string(total) + ")..." << std::endl; if (total > 0) { int count = 0; long lastid = 0; @@ -176,12 +176,10 @@ processWaysLine(WayTask &wayTask) // Proccesing ways for (auto way = ways->begin(); way != ways->end(); ++way) { - if (way->refs.front() == way->refs.back()) { - log_debug("Way Id: %1%", way->id); - // Fill the way_refs table - for (auto ref = way->refs.begin(); ref != way->refs.end(); ++ref) { - task->query += "INSERT INTO way_refs (way_id, node_id) VALUES (" + std::to_string(way->id) + "," + std::to_string(*ref) + "); "; - } + log_debug("Way Id: %1%", way->id); + // Fill the way_refs table + for (auto ref = way->refs.begin(); ref != way->refs.end(); ++ref) { + task->query += "INSERT INTO way_refs (way_id, node_id) VALUES (" + std::to_string(way->id) + "," + std::to_string(*ref) + "); "; } } wayTask.lastid = ways->back().id; diff --git a/src/raw/queryraw.cc b/src/raw/queryraw.cc index e128c44b..c0134960 100644 --- a/src/raw/queryraw.cc +++ b/src/raw/queryraw.cc @@ -418,9 +418,15 @@ int QueryRaw::getWaysCount(const std::string &tableName) { std::shared_ptr> QueryRaw::getWaysFromDB(int lastid, const std::string &tableName) { - std::string waysQuery = "SELECT osm_id, refs, ST_AsText(ST_ExteriorRing(geom), 4326), version, tags FROM " + tableName + " where osm_id > " + std::to_string(lastid) + " order by osm_id asc limit 500;"; + std::string waysQuery; + if (tableName == QueryRaw::polyTable) { + waysQuery = "SELECT osm_id, refs, ST_AsText(ST_ExteriorRing(geom), 4326)"; + } else { + waysQuery = "SELECT osm_id, refs, ST_AsText(geom, 4326)"; + } + waysQuery += ", version, tags FROM " + tableName + " where osm_id > " + std::to_string(lastid) + " order by osm_id asc limit 500;"; + auto ways_result = dbconn->query(waysQuery); - // Fill vector of OsmWay objects auto ways = std::make_shared>(); for (auto way_it = ways_result.begin(); way_it != ways_result.end(); ++way_it) { @@ -429,20 +435,24 @@ QueryRaw::getWaysFromDB(int lastid, const std::string &tableName) { std::string refs_str = (*way_it)[1].as(); if (refs_str.size() > 1) { way.refs = arrayStrToVector(refs_str); - } - std::string poly = (*way_it)[2].as(); - boost::geometry::read_wkt(poly, way.linestring); - way.polygon = { {std::begin(way.linestring), std::end(way.linestring)} }; - way.version = (*way_it)[3].as(); - auto tags = (*way_it)[4]; - if (!tags.is_null()) { - auto tags = parseTagsString((*way_it)[4].as()); - for (auto const& [key, val] : tags) - { - way.addTag(key, val); + + std::string poly = (*way_it)[2].as(); + boost::geometry::read_wkt(poly, way.linestring); + + if (tableName == QueryRaw::polyTable) { + way.polygon = { {std::begin(way.linestring), std::end(way.linestring)} }; + } + way.version = (*way_it)[3].as(); + auto tags = (*way_it)[4]; + if (!tags.is_null()) { + auto tags = parseTagsString((*way_it)[4].as()); + for (auto const& [key, val] : tags) + { + way.addTag(key, val); + } } + ways->push_back(way); } - ways->push_back(way); } return ways; diff --git a/utils/raw-underpass.sql b/utils/raw-underpass.sql index 20fdd286..6639ad24 100644 --- a/utils/raw-underpass.sql +++ b/utils/raw-underpass.sql @@ -1,10 +1,14 @@ CREATE UNIQUE INDEX nodes_id_idx ON public.nodes (osm_id); CREATE UNIQUE INDEX ways_poly_id_idx ON public.ways_poly (osm_id); +CREATE UNIQUE INDEX ways_line_id_idx ON public.ways_line(osm_id); + CREATE INDEX way_refs_node_id_idx ON public.way_refs (node_id); CREATE INDEX way_refs_way_id_idx ON public.way_refs (way_id); CREATE INDEX nodes_version_idx ON public.nodes (version); CREATE INDEX ways_poly_version_idx ON public.ways_poly (version); +CREATE INDEX ways_line_version_idx ON public.ways_line (version); CREATE INDEX nodes_timestamp_idx ON public.nodes(timestamp DESC); CREATE INDEX ways_poly_timestamp_idx ON public.ways_poly(timestamp DESC); +CREATE INDEX ways_line_timestamp_idx ON public.ways_line(timestamp DESC); From e60eca17a886ceb17a8f303f44cf2f1ca05efcbb Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Sun, 10 Sep 2023 08:37:29 -0300 Subject: [PATCH 8/8] Fixes for adding lines --- src/bootstrap/bootstrap.cc | 38 +++++++++++++++++++------------------- utils/bootstrap.sh | 6 +++--- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/bootstrap/bootstrap.cc b/src/bootstrap/bootstrap.cc index 4185baf9..7cc658b0 100644 --- a/src/bootstrap/bootstrap.cc +++ b/src/bootstrap/bootstrap.cc @@ -71,25 +71,25 @@ void startProcessingWays(const underpassconfig::UnderpassConfig &config) { // TODO: improve duplicate code int total = queryraw->getWaysCount(QueryRaw::polyTable); std::cout << "Processing polygons (" + std::to_string(total) + ")..." << std::endl; - // if (total > 0) { - // int count = 0; - // long lastid = 0; - // while (count < total) { - // int percentage = (count * 100) / total; - // auto task = std::make_shared(); - // WayTask wayTask; - // wayTask.plugin = validator; - // wayTask.queryvalidate = queryvalidate; - // wayTask.queryraw = queryraw; - // wayTask.task = task; - // wayTask.lastid = lastid; - // processWaysPoly(wayTask); - // db->query(task->query); - // lastid = wayTask.lastid; - // count += wayTask.processed; - // std::cout << "\r" << "Processing : " << count << "/" << total << " (" << percentage << "%)"; - // } - // } + if (total > 0) { + int count = 0; + long lastid = 0; + while (count < total) { + int percentage = (count * 100) / total; + auto task = std::make_shared(); + WayTask wayTask; + wayTask.plugin = validator; + wayTask.queryvalidate = queryvalidate; + wayTask.queryraw = queryraw; + wayTask.task = task; + wayTask.lastid = lastid; + processWaysPoly(wayTask); + db->query(task->query); + lastid = wayTask.lastid; + count += wayTask.processed; + std::cout << "\r" << "Processing : " << count << "/" << total << " (" << percentage << "%)"; + } + } total = queryraw->getWaysCount(QueryRaw::lineTable); std::cout << "Processing lines (" + std::to_string(total) + ")..." << std::endl; diff --git a/utils/bootstrap.sh b/utils/bootstrap.sh index accd051e..30471312 100644 --- a/utils/bootstrap.sh +++ b/utils/bootstrap.sh @@ -111,15 +111,15 @@ then docker cp $COUNTRY.geojson underpass:/usr/local/lib/underpass/config/priority.geojson docker cp $COUNTRY.geojson underpass:/code/config/priority.geojson else - cp $COUNTRY.geojson underpass:/usr/local/lib/underpass/config/priority.geojson - cp $COUNTRY.geojson underpass:/code/config/priority.geojson + cp $COUNTRY.geojson /usr/local/lib/underpass/config/priority.geojson + cp $COUNTRY.geojson ../config/priority.geojson fi echo "Bootstrapping database ..." if [ -z "${use_docker}" ] then docker exec -w /code/build -t underpass ./underpass --bootstrap else - cd ../build ./underpass --bootstrap + cd ../build && ./underpass --bootstrap fi echo "Done." echo " "