Skip to content
This repository has been archived by the owner on Oct 5, 2024. It is now read-only.

Commit

Permalink
+ Compatibility between Underpass and Raw Data API. Fix for bootstrap…
Browse files Browse the repository at this point in the history 
… basg script.
  • Loading branch information
@emi420
emi420 committed Oct 10, 2023
1 parent d029b1a commit 13422f2
Show file tree
Hide file tree
Showing 9 changed files with 132 additions and 211 deletions.
9 changes: 4 additions & 5 deletions python/dbapi/api/raw.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,23 +22,22 @@
RESULTS_PER_PAGE = 500
RESULTS_PER_PAGE_LIST = 100

# TODO: improve this code
def tagsQueryFilter(tagsQuery, table):
query = ""
tags = tagsQuery.split(",")
keyValue = tags[0].split("=")

if len(keyValue) == 2:
query += "{0}.tags->'{1}' ~* '^{2}'".format(table, keyValue[0], keyValue[1])
query += "{0}.tags->>'{1}' ~* '^{2}'".format(table, keyValue[0], keyValue[1])
else:
query += "{0}.tags ? '{1}'".format(table, keyValue[0])
query += "{0}.tags->>'{1}' IS NOT NULL".format(table, keyValue[0])

for tag in tags[1:]:
keyValue = tag.split("=")
if len(keyValue) == 2:
query += "OR {0}.tags->'{!}' ~* '^{2}'".format(table, keyValue[0], keyValue[1])
query += "OR {0}.tags->>'{1}' ~* '^{2}'".format(table, keyValue[0], keyValue[1])
else:
query += "OR {0}.tags ? '{1}'".format(table, keyValue[0])
query += "OR {0}.tags->>'{1}' IS NOT NULL".format(table, keyValue[0])
return query

class Raw:
Expand Down
6 changes: 3 additions & 3 deletions setup/underpass.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ CREATE TABLE IF NOT EXISTS public.ways_poly (
osm_id int8,
changeset int8,
geom public.geometry(Polygon,4326),
tags public.hstore,
tags JSONB,
refs int8[],
timestamp timestamp with time zone,
version int,
Expand All @@ -75,7 +75,7 @@ CREATE TABLE IF NOT EXISTS public.ways_line (
osm_id int8,
changeset int8,
geom public.geometry(LineString,4326),
tags public.hstore,
tags JSONB,
refs int8[],
timestamp timestamp with time zone,
version int,
Expand All @@ -87,7 +87,7 @@ CREATE TABLE IF NOT EXISTS public.nodes (
osm_id int8,
changeset int8,
geom public.geometry(Point,4326),
tags public.hstore,
tags JSONB,
timestamp timestamp with time zone,
version int,
"user" text,
Expand Down
132 changes: 46 additions & 86 deletions src/bootstrap/bootstrap.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,55 +67,40 @@ void startProcessingWays(const underpassconfig::UnderpassConfig &config) {

auto queryvalidate = std::make_shared<QueryValidate>(db);
auto queryraw = std::make_shared<QueryRaw>(db);
std::vector<const std::string> tables = {
QueryRaw::polyTable,
QueryRaw::lineTable
};

// TODO: improve duplicate code
int total = queryraw->getWaysCount(QueryRaw::polyTable);
if (total > 0) {
int count = 0;
long lastid = 0;
while (count < total) {
int percentage = (count * 100) / total;
auto task = std::make_shared<BootstrapTask>();
WayTask wayTask;
wayTask.plugin = validator;
wayTask.queryvalidate = queryvalidate;
wayTask.queryraw = queryraw;
wayTask.task = task;
wayTask.lastid = lastid;
processWaysPoly(wayTask);
db->query(task->query);
lastid = wayTask.lastid;
count += wayTask.processed;
std::cout << "\r" << "Processing polygons: " << count << "/" << total << " (" << percentage << "%)";
}
}

total = queryraw->getWaysCount(QueryRaw::lineTable);
if (total > 0) {
int count = 0;
long lastid = 0;
while (count < total) {
int percentage = (count * 100) / total;
auto task = std::make_shared<BootstrapTask>();
WayTask wayTask;
wayTask.plugin = validator;
wayTask.queryvalidate = queryvalidate;
wayTask.queryraw = queryraw;
wayTask.task = task;
wayTask.lastid = lastid;
processWaysLine(wayTask);
db->query(task->query);
lastid = wayTask.lastid;
count += wayTask.processed;
std::cout << "\r" << "Processing lines: " << count << "/" << total << " (" << percentage << "%)";
for (auto table_it = tables.begin(); table_it != tables.end(); ++table_it) {
int total = queryraw->getWaysCount(*table_it);
if (total > 0) {
int count = 0;
long lastid = 0;
while (count < total) {
int percentage = (count * 100) / total;
auto task = std::make_shared<BootstrapTask>();
WayTask wayTask;
wayTask.plugin = validator;
wayTask.queryvalidate = queryvalidate;
wayTask.queryraw = queryraw;
wayTask.task = task;
wayTask.lastid = lastid;

processWays(wayTask, *table_it);
db->query(task->query);
lastid = wayTask.lastid;
count += wayTask.processed;
std::cout << "\r" << "Processing " << *table_it << ": " << count << "/" << total << " (" << percentage << "%)";
}
}
}

}

// This thread get started for every page of way
void
processWaysPoly(WayTask &wayTask)
processWays(WayTask &wayTask, const std::string &tableName)
{
#ifdef TIMING_DEBUG
boost::timer::auto_cpu_timer timer("bootstrap::processWays(wayTask): took %w seconds\n");
Expand All @@ -127,60 +112,35 @@ processWaysPoly(WayTask &wayTask)
auto queryraw = wayTask.queryraw;
auto lastid = wayTask.lastid;

auto ways = queryraw->getWaysFromDB(lastid, QueryRaw::polyTable);
auto ways = queryraw->getWaysFromDB(lastid, tableName);
wayTask.processed = ways->size();

// Proccesing ways
for (auto way = ways->begin(); way != ways->end(); ++way) {
if (way->refs.front() == way->refs.back()) {
log_debug("Way Id: %1%", way->id);

// Bad geometry
if (way->containsKey("building") && (boost::geometry::num_points(way->linestring) - 1 < 4 ||
plugin->unsquared(way->linestring))
) {
auto status = ValidateStatus(*way);
status.timestamp = boost::posix_time::microsec_clock::universal_time();
status.source = "building";
boost::geometry::centroid(way->linestring, status.center);
task->query += queryvalidate->applyChange(status, badgeom);
if (wayTask.processed > 0) {
// Proccesing ways
for (auto way = ways->begin(); way != ways->end(); ++way) {

// If it's closed polygon
if (way->refs.front() == way->refs.back()) {
log_debug("Way Id: %1%", way->id);

// Bad geometry
if (way->containsKey("building") && (boost::geometry::num_points(way->linestring) - 1 < 4 ||
plugin->unsquared(way->linestring))
) {
auto status = ValidateStatus(*way);
status.timestamp = boost::posix_time::microsec_clock::universal_time();
status.source = "building";
boost::geometry::centroid(way->linestring, status.center);
task->query += queryvalidate->applyChange(status, badgeom);
}
}

// Fill the way_refs table
for (auto ref = way->refs.begin(); ref != way->refs.end(); ++ref) {
task->query += "INSERT INTO way_refs (way_id, node_id) VALUES (" + std::to_string(way->id) + "," + std::to_string(*ref) + "); ";
}
}
wayTask.lastid = ways->back().id;
}
wayTask.lastid = ways->back().id;

}

void
processWaysLine(WayTask &wayTask)
{
#ifdef TIMING_DEBUG
boost::timer::auto_cpu_timer timer("bootstrap::processWays(wayTask): took %w seconds\n");
#endif

auto plugin = wayTask.plugin;
auto task = wayTask.task;
auto queryvalidate = wayTask.queryvalidate;
auto queryraw = wayTask.queryraw;
auto lastid = wayTask.lastid;

auto ways = queryraw->getWaysFromDB(lastid, QueryRaw::lineTable);
wayTask.processed = ways->size();

// Proccesing ways
for (auto way = ways->begin(); way != ways->end(); ++way) {
log_debug("Way Id: %1%", way->id);
// Fill the way_refs table
for (auto ref = way->refs.begin(); ref != way->refs.end(); ++ref) {
task->query += "INSERT INTO way_refs (way_id, node_id) VALUES (" + std::to_string(way->id) + "," + std::to_string(*ref) + "); ";
}
}
wayTask.lastid = ways->back().id;

}

Expand Down
3 changes: 1 addition & 2 deletions src/bootstrap/bootstrap.hh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@ struct WayTask {
void startProcessingWays(const underpassconfig::UnderpassConfig &config);

// This thread get started for every page of way
void processWaysPoly(WayTask &wayTask);
void processWaysLine(WayTask &wayTask);
void processWays(WayTask &wayTask, const std::string &tableName);

}
2 changes: 1 addition & 1 deletion src/data/pq.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ Pq::escapedString(std::string text)
}
i++;
}
return sdb->esc(boost::locale::conv::to_utf<char>(newstr, "Latin1"));
return sdb->esc(newstr);
}

} // namespace pq
16 changes: 9 additions & 7 deletions src/raw/queryraw.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@
/// includes querying existing data in the database, as well as
/// updating the database.

// TODO: add support for relations/multipolygon

// This is generated by autoconf
#ifdef HAVE_CONFIG_H
#include "unconfig.h"
Expand Down Expand Up @@ -82,14 +84,14 @@ QueryRaw::applyChange(const OsmNode &node) const
std::string tags = "";
if (node.tags.size() > 0) {
for (auto it = std::begin(node.tags); it != std::end(node.tags); ++it) {
std::string tag_format = "\"%s\" => \"%s\",";
std::string tag_format = "\"%s\" : \"%s\",";
boost::format tag_fmt(tag_format);
tag_fmt % dbconn->escapedString(it->first);
tag_fmt % dbconn->escapedString(it->second);
tags += tag_fmt.str();
}
tags.erase(tags.size() - 1);
tags = "'" + tags + "'";
tags = "'{" + tags + "}'";
} else {
tags = "null";
}
Expand Down Expand Up @@ -161,14 +163,14 @@ QueryRaw::applyChange(const OsmWay &way) const
std::string tags = "";
if (way.tags.size() > 0) {
for (auto it = std::begin(way.tags); it != std::end(way.tags); ++it) {
std::string tag_format = "\"%s\" => \"%s\",";
std::string tag_format = "\"%s\" : \"%s\",";
boost::format tag_fmt(tag_format);
tag_fmt % dbconn->escapedString(it->first);
tag_fmt % dbconn->escapedString(it->second);
tags += tag_fmt.str();
}
tags.erase(tags.size() - 1);
tags = "'" + tags + "'";
tags = "'{" + tags + "}'";
} else {
tags = "null";
}
Expand Down Expand Up @@ -372,7 +374,7 @@ std::map<std::string, std::string> parseTagsString(const std::string& input) {
std::string token;
while (std::getline(ss, token, ',')) {
// Find the position of the arrow
size_t arrowPos = token.find("=>");
size_t arrowPos = token.find(":");
if (arrowPos != std::string::npos) {
std::string key = token.substr(1, arrowPos - 1);
std::string value = token.substr(arrowPos + 2);
Expand All @@ -393,8 +395,8 @@ QueryRaw::getWaysByNodesRefs(std::string &nodeIds) const
// Get all ways that have references to nodes
std::list<std::shared_ptr<osmobjects::OsmWay>> ways;

// TODO: add ways_line to the query
std::string waysQuery = "SELECT distinct(osm_id), refs, version, tags from way_refs join ways_poly rp on rp.osm_id = way_id where node_id = any(ARRAY[" + nodeIds + "]);";
std::string waysQuery = "SELECT distinct(osm_id), refs, version, tags from way_refs join ways_poly wp on wp.osm_id = way_id where node_id = any(ARRAY[" + nodeIds + "])";
waysQuery += " UNION SELECT distinct(osm_id), refs, version, tags from way_refs join ways_line wl on wl.osm_id = way_id where node_id = any(ARRAY[" + nodeIds + "]);";
auto ways_result = dbconn->query(waysQuery);

// Fill vector of OsmWay objects
Expand Down
40 changes: 21 additions & 19 deletions utils/bootstrap.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@
# database with OSM data for a country
# -----

localfiles='false'
use_docker='false'
localfiles=false
use_docker=false

while getopts r:c:h::u:p:d:l:k flag
do
Expand All @@ -35,8 +35,8 @@ do
u) user=${OPTARG};;
p) port=${OPTARG};;
d) database=${OPTARG};;
l) localfiles=${OPTARG};;
k) use_docker=${OPTARG};;
l) localfiles=true;;
k) use_docker=true;;
esac
done

Expand All @@ -53,6 +53,7 @@ else
USER=underpass
fi


if [ -n "${REGION}" ] && [ -n "${COUNTRY}" ]
then

Expand All @@ -63,11 +64,12 @@ then
echo Port: $PORT
echo Database: $DB

if [ -z "${localfiles}" ]
if "$localfiles";
then
echo "Use local files?: yes"
fi
if [ -z "${use_docker}" ]

if "$use_docker";
then
echo "Use Docker?: yes"
fi
Expand All @@ -91,7 +93,7 @@ then
PGPASSWORD=$PASS psql --host $HOST --user $USER --port $PORT $DB -c 'DROP TABLE IF EXISTS ways_poly; DROP TABLE IF EXISTS ways_line; DROP TABLE IF EXISTS nodes; DROP TABLE IF EXISTS way_refs; DROP TABLE IF EXISTS validation; DROP TABLE IF EXISTS changesets;'
PGPASSWORD=$PASS psql --host $HOST --user $USER --port $PORT $DB --file '../setup/underpass.sql'

if [ -z "${localfiles}" ]
if "$localfiles";
then
echo "(Using local files)"
else
Expand All @@ -106,23 +108,23 @@ then

echo "Configuring Underpass ..."
python3 poly2geojson.py $COUNTRY.poly
if [ -z "${use_docker}" ]
if "$use_docker";
then
docker cp $COUNTRY.geojson underpass:/usr/local/lib/underpass/config/priority.geojson
docker cp $COUNTRY.geojson underpass:/code/config/priority.geojson
else
cp $COUNTRY.geojson /usr/local/lib/underpass/config/priority.geojson
cp $COUNTRY.geojson ../config/priority.geojson
fi
echo "Bootstrapping database ..."
if [ -z "${use_docker}" ]
then
docker exec -w /code/build -t underpass ./underpass --bootstrap
else
cd ../build && ./underpass --bootstrap
fi
echo "Done."
echo " "
# echo "Bootstrapping database ..."
# if "$use_docker";
# then
# docker exec -w /code/build -t underpass ./underpass --bootstrap
# else
# cd ../build && ./underpass --bootstrap
# fi
# echo "Done."
# echo " "
fi

else
Expand All @@ -141,6 +143,6 @@ else
echo " -u user (Database user)"
echo " -p port (Database port)"
echo " -d database (Database name)"
echo " -l (Use local files instead of download them)"
echo " -k (Use Docker Underpass installation)"
echo " -l yes (Use local files instead of download them)"
echo " -k yes (Use Docker Underpass installation)"
fi
Loading

0 comments on commit 13422f2

Please sign in to comment.