From 0d0dd47fb75c4a73350647d75857cb2d201508b9 Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Thu, 7 Mar 2024 16:35:12 -0300 Subject: [PATCH] + Relations support for Python DB/REST API --- python/dbapi/api/db.py | 7 ++- python/dbapi/api/raw.py | 98 +++++++++++++++++++-------------------- python/dbapi/api/stats.py | 48 ++++++++++++++----- src/raw/queryraw.cc | 2 +- 4 files changed, 92 insertions(+), 63 deletions(-) diff --git a/python/dbapi/api/db.py b/python/dbapi/api/db.py index db647593f..05418acdc 100644 --- a/python/dbapi/api/db.py +++ b/python/dbapi/api/db.py @@ -61,9 +61,12 @@ async def run(self, query, singleObject = False): return result[0] return json.loads((result[0]['result'])) except Exception as e: - print("\n******* \n" + query + "\n******* \n") + # print("\n******* \n" + query + "\n******* \n") print(e) - return None + if singleObject: + return {} + else: + return [] finally: await self.pool.release(conn) return None diff --git a/python/dbapi/api/raw.py b/python/dbapi/api/raw.py index 632b2f302..783166a96 100644 --- a/python/dbapi/api/raw.py +++ b/python/dbapi/api/raw.py @@ -29,6 +29,11 @@ def getGeoType(table): return "LineString" return "Node" +def getRelationsGeoType(table): + if table == "ways_poly": + return "MultiPolygon" + return "MultiLineString" + def geoFeaturesQuery( area = None, tags = None, @@ -40,21 +45,29 @@ def geoFeaturesQuery( table = None): geoType = getGeoType(table) - query = "with t_ways AS ( \ + query = "with t_data AS ( \ SELECT '" + geoType + "' as type, " + table + ".osm_id as id, " + table + ".timestamp, geom as geometry, tags, status, hashtags, editor, created_at FROM " + table + " \ LEFT JOIN validation ON validation.osm_id = " + table + ".osm_id \ - LEFT JOIN changesets c ON c.id = " + table + ".changeset \ - WHERE \ + LEFT JOIN changesets c ON c.id = " + table + ".changeset" + + if table != "nodes": + query += " UNION \ + SELECT '" + getRelationsGeoType(table) + "' as type, relations.osm_id as id, relations.timestamp, geom as geometry, tags, status, hashtags, editor, created_at FROM relations \ + LEFT JOIN validation ON validation.osm_id = relations.osm_id \ + LEFT JOIN changesets c ON c.id = relations.changeset \ + WHERE (tags->>'type' = '" + getRelationsGeoType(table).lower() + "')" + + query += "), t_results as (select * from t_data WHERE \ {0} {1} {2} {3} {4} {5} \ ), \ t_features AS ( \ - SELECT jsonb_build_object( 'type', 'Feature', 'id', id, 'properties', to_jsonb(t_ways) \ - - 'geometry' , 'geometry', ST_AsGeoJSON(geometry)::jsonb ) AS feature FROM t_ways \ + SELECT jsonb_build_object( 'type', 'Feature', 'id', id, 'properties', to_jsonb(t_results) \ + - 'geometry' , 'geometry', ST_AsGeoJSON(geometry)::jsonb ) AS feature FROM t_results \ ) SELECT jsonb_build_object( 'type', 'FeatureCollection', 'features', jsonb_agg(t_features.feature) ) \ as result FROM t_features;".format( - "ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({0})))', 4326) )".format(area) if area else "1=1 ", - "AND (" + tagsQueryFilter(tags, table) + ")" if tags else "", - "AND " + hashtagQueryFilter(hashtag, table) if hashtag else "", + "ST_Intersects(\"geometry\", ST_GeomFromText('MULTIPOLYGON((({0})))', 4326) )".format(area) if area else "1=1 ", + "AND (" + tagsQueryFilter(tags, "t_data") + ")" if tags else "", + "AND " + hashtagQueryFilter(hashtag, "t_data") if hashtag else "", "AND created_at >= {0} AND created_at <= {1}".format(dateFrom, dateTo) if dateFrom and dateTo else "", "AND status = '{0}'".format(status) if (status) else "", "LIMIT " + str(RESULTS_PER_PAGE), @@ -74,25 +87,33 @@ def listAllFeaturesQuery( ): geoType = getGeoType(table) + relationsGeoType = getRelationsGeoType(table) if table == "nodes": osmType = "node" else: osmType = "way" - query = "\ - ( \ + query = "with t_data AS ( \ SELECT '" + osmType + "' as type, '" + geoType + "' as geotype, " + table + ".osm_id as id, ST_X(ST_Centroid(geom)) as lat, ST_Y(ST_Centroid(geom)) as lon, " + table + ".timestamp, tags, " + table + ".changeset, c.created_at, v.status FROM " + table + " \ LEFT JOIN changesets c ON c.id = " + table + ".changeset \ - LEFT JOIN validation v ON v.osm_id = " + table + ".osm_id \ + LEFT JOIN validation v ON v.osm_id = " + table + ".osm_id" + + if table != "nodes": + query += " UNION \ + SELECT '" + osmType + "' as type, '" + relationsGeoType + "' as geotype, relations.osm_id as id, ST_X(ST_Centroid(geom)) as lat, ST_Y(ST_Centroid(geom)) as lon, relations.timestamp, tags, relations.changeset, c.created_at, v.status FROM relations \ + LEFT JOIN validation v ON v.osm_id = relations.osm_id \ + LEFT JOIN changesets c ON c.id = relations.changeset \ + WHERE (tags->>'type' = '" + getRelationsGeoType(table).lower() + "')" + + query += ") select * from t_data \ WHERE {0} {1} {2} {3} {4} {5} {6} \ - )\ ".format( "created_at >= '{0}'".format(dateFrom) if (dateFrom) else "1=1", "AND created_at <= '{0}'".format(dateTo) if (dateTo) else "", "AND status = '{0}'".format(status) if (status) else "", - "AND " + hashtagQueryFilter(hashtag, table) if hashtag else "", + "AND " + hashtagQueryFilter(hashtag, "t_data") if hashtag else "", "AND ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({0})))', 4326) )".format(area) if area else "", - "AND (" + tagsQueryFilter(tags, table) + ")" if tags else "", + "AND (" + tagsQueryFilter(tags, "t_data") + ")" if tags else "", "AND " + orderBy + " IS NOT NULL ORDER BY " + orderBy + " DESC LIMIT " + str(RESULTS_PER_PAGE_LIST) + (" OFFSET {0}" \ .format(page * RESULTS_PER_PAGE_LIST) if page else ""), ).replace("WHERE 1=1 AND", "WHERE") @@ -152,19 +173,8 @@ def getList( orderBy, page ) - elif featureType == "polygon": - return self.getPolygonsList( - area, - tags, - hashtag, - dateFrom, - dateTo, - status, - orderBy, - page - ) - else: - return self.getAllList( + + return self.getPolygonsList( area, tags, hashtag, @@ -206,18 +216,8 @@ def getFeatures( status, page ) - elif featureType == "polygon": - return self.getPolygons( - area, - tags, - hashtag, - dateFrom, - dateTo, - status, - page - ) - else: - return self.getAll( + + return self.getPolygons( area, tags, hashtag, @@ -362,7 +362,7 @@ async def getPolygonsList( tags, hashtag, status, - orderBy or "ways_poly.osm_id", + orderBy or "id", page or 0, dateFrom, dateTo, @@ -372,7 +372,7 @@ async def getPolygonsList( " UNION ".join([queryPolygons]), dateFrom, dateTo, - orderBy or "osm_id" + orderBy or "id" ) return await self.underpassDB.run(query) @@ -393,7 +393,7 @@ async def getLinesList( tags, hashtag, status, - orderBy or "ways_line.osm_id", + orderBy or "id", page or 0, dateFrom, dateTo, @@ -403,7 +403,7 @@ async def getLinesList( " UNION ".join([queryLines]), dateFrom, dateTo, - orderBy or "osm_id" + orderBy or "id" ) return await self.underpassDB.run(query) @@ -424,7 +424,7 @@ async def getNodesList( tags, hashtag, status, - orderBy or "nodes.osm_id", + orderBy or "id", page or 0, dateFrom, dateTo, @@ -434,7 +434,7 @@ async def getNodesList( " UNION ".join([queryNodes]), dateFrom, dateTo, - orderBy or "osm_id" + orderBy or "id" ) return await self.underpassDB.run(query) @@ -455,7 +455,7 @@ async def getAllList( tags, hashtag, status, - orderBy or "ways_poly.osm_id", + orderBy or "id", page or 0, dateFrom, dateTo, @@ -466,7 +466,7 @@ async def getAllList( tags, hashtag, status, - orderBy or "ways_line.osm_id", + orderBy or "id", page or 0, dateFrom, dateTo, @@ -477,7 +477,7 @@ async def getAllList( tags, hashtag, status, - orderBy or "nodes.osm_id", + orderBy or "id", page or 0, dateFrom, dateTo, @@ -487,6 +487,6 @@ async def getAllList( " UNION ".join([queryPolygons, queryLines, queryNodes]), dateFrom, dateTo, - orderBy or "osm_id" + orderBy or "id" ) return await self.underpassDB.run(query) diff --git a/python/dbapi/api/stats.py b/python/dbapi/api/stats.py index 8338f2d7c..55d877632 100644 --- a/python/dbapi/api/stats.py +++ b/python/dbapi/api/stats.py @@ -45,25 +45,39 @@ async def getCount( select {0}.osm_id from {0} \ left join changesets c on changeset = c.id \ where {1} {2} {3} {4} {5} \ - ), \ - count_validated_features as ( \ - select count(distinct(all_features.osm_id)) as count from all_features \ - left join validation v on all_features.osm_id = v.osm_id \ - where v.status = '{6}' \ - ), count_features as (\ - select count(distinct(all_features.osm_id)) as total from all_features \ - ) \ - select count, total from count_validated_features, count_features".format( + ".format( table, "created_at >= '{0}'".format(dateFrom) if (dateFrom) else "1=1", "AND created_at <= '{0}'".format(dateTo) if (dateTo) else "", "AND ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({0})))', 4326) )".format(area) if area else "", "AND (" + tagsQueryFilter(tags, table) + ")" if tags else "", "AND " + hashtagQueryFilter(hashtag, table) if hashtag else "", - status ) + + query += " union \ + select relations.osm_id from relations \ + left join changesets c on changeset = c.id \ + where {0} {1} {2} {3} {4} \ + ".format( + "created_at >= '{0}'".format(dateFrom) if (dateFrom) else "1=1", + "AND created_at <= '{0}'".format(dateTo) if (dateTo) else "", + "AND ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({0})))', 4326) )".format(area) if area else "", + "AND (" + tagsQueryFilter(tags, "relations") + ")" if tags else "", + "AND " + hashtagQueryFilter(hashtag, "relations") if hashtag else "", + ) + + query += "), count_validated_features as ( \ + select count(distinct(all_features.osm_id)) as count from all_features \ + left join validation v on all_features.osm_id = v.osm_id \ + where v.status = '{0}' \ + ), count_features as (\ + select count(distinct(all_features.osm_id)) as total from all_features \ + ) \ + select count, total from count_validated_features, count_features".format(status) + else: - query = "select count(distinct {0}.osm_id) as count from {0} \ + query = "WITH counts AS (" + query += "select distinct {0}.osm_id from {0} \ left join changesets c on changeset = c.id \ where {1} {2} {3} {4} {5}".format( table, @@ -73,6 +87,18 @@ async def getCount( "AND (" + tagsQueryFilter(tags, table) + ")" if tags else "", "AND " + hashtagQueryFilter(hashtag, table) if hashtag else "" ) + query += " union " + query += "select distinct relations.osm_id from relations \ + left join changesets c on changeset = c.id \ + where {0} {1} {2} {3} {4}".format( + "created_at >= '{0}'".format(dateFrom) if (dateFrom) else "1=1", + "AND created_at <= '{0}'".format(dateTo) if (dateTo) else "", + "AND ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({0})))', 4326) )".format(area) if area else "", + "AND (" + tagsQueryFilter(tags, "relations") + ")" if tags else "", + "AND " + hashtagQueryFilter(hashtag, "relations") if hashtag else "" + ) + query += ") SELECT COUNT(osm_id) FROM counts;" + return(await self.underpassDB.run(query, True)) \ No newline at end of file diff --git a/src/raw/queryraw.cc b/src/raw/queryraw.cc index 0a8467423..bb1db731f 100644 --- a/src/raw/queryraw.cc +++ b/src/raw/queryraw.cc @@ -919,7 +919,7 @@ QueryRaw::getRelationsFromDB(long lastid, int pageSize) { std::string geometry = (*rel_it)[2].as(); if (geometry.substr(0, 12) == "MULTIPOLYGON") { boost::geometry::read_wkt(geometry, relation.multipolygon); - } else if (geometry.substr(0, 15) == "MULTILINESTRING") { + } else if (geometry.substr(0, 15) == "MULTILINESTRING") { boost::geometry::read_wkt(geometry, relation.multilinestring); } relation.version = (*rel_it)[3].as();