+ Lines for raw data. Compatibility between Underpass and Raw Data API

docs/osmstats.md

@@ -57,7 +57,7 @@ buildings_modified | This value is updated by counting the existing buildings mo
 pois_added | This value is updated by counting the POIs added by the user in the change file
 pois_modified | This value is updated by counting the existing POIs modified by the user in the change file
 editor | The editor used, and comes from the changeset
-user_id | The user ID, comes from the changeset
+uid | The user ID, comes from the changeset
 created_at | The timestamp this changeset was created, comes from the changeset
 closed_at | The timestamp this changeset was closed, comes from the changeset
 verified | Whether this data has been validated
@@ -128,7 +128,7 @@ level | The badge level
 
 Keyword | Description
 --------|------------
-user_id | The OSM user ID
+uid | The OSM user ID
 badge_id | The badge ID
 updated_at | The timestamp of the user receiving this badge
 

docs/statistics.md

@@ -211,7 +211,7 @@ backend and the frontend, without having modify the database schema.
 An example query to count the total number of buildings added by the
 user **4321** for a Tasking Manager project **1234** would be this:
 > SELECT SUM(CAST(added::hstore->'building' AS DOUBLE precision)) FROM
-changesets WHERE 'hotosm-project-1234' = ANY(hashtags) AND user_id=4321;
+changesets WHERE 'hotosm-project-1234' = ANY(hashtags) AND uid=4321;
 
 The source is the satellite imagery used for remote mapping.
 
@@ -221,7 +221,7 @@ Keyword | Description
 --------|------------
 id | The ID of this changeset
 editor | The editor used for this changeset
-user_id | The OSM User ID of the mapper
+uid | The OSM User ID of the mapper
 created_at | The timestamp when this changes was uploaded
 closed_at | The timestamp when this uploaded change completed processing
 updated_at | The timestamp when this last had data updated

python/dbapi/api/raw.py

@@ -35,8 +35,8 @@ def getPolygons(
         page = None
     ):
         query = "with t_ways AS ( \
-            SELECT raw_poly.osm_id as id, raw_poly.timestamp, geometry, tags, status FROM raw_poly \
-            LEFT JOIN validation ON validation.osm_id = raw_poly.osm_id \
+            SELECT ways_poly.osm_id as id, ways_poly.timestamp, geom as geometry, tags, status FROM ways_poly \
+            LEFT JOIN validation ON validation.osm_id = ways_poly.osm_id \
             WHERE \
             {0} {1} {2} {3} \
         ), \
@@ -45,10 +45,37 @@ def getPolygons(
             - 'geometry' , 'geometry', ST_AsGeoJSON(geometry)::jsonb ) AS feature FROM t_ways  \
         ) SELECT jsonb_build_object( 'type', 'FeatureCollection', 'features', jsonb_agg(t_features.feature) ) \
         as result FROM t_features;".format(
-            "ST_Intersects(\"geometry\", ST_GeomFromText('POLYGON(({0}))', 4326) )".format(area) if area else "1=1 ",
-            "and raw_poly.tags ? '{0}'".format(key) if key and not value else "",
-            "and raw_poly.tags->'{0}' ~* '^{1}'".format(key, value) if key and value else "",
-            "ORDER BY raw_poly.timestamp DESC LIMIT " + str(RESULTS_PER_PAGE) + " OFFSET {0}".format(page * RESULTS_PER_PAGE) if page else "",
+            "ST_Intersects(\"geom\", ST_GeomFromText('POLYGON(({0}))', 4326) )".format(area) if area else "1=1 ",
+            "and ways_poly.tags ? '{0}'".format(key) if key and not value else "",
+            "and ways_poly.tags->'{0}' ~* '^{1}'".format(key, value) if key and value else "",
+            "ORDER BY ways_poly.timestamp DESC LIMIT " + str(RESULTS_PER_PAGE) + " OFFSET {0}".format(page * RESULTS_PER_PAGE) if page else "",
+        )
+        return self.underpassDB.run(query, responseType, True)
+
+    def getLines(
+        self, 
+        area = None,
+        key = None,
+        value = None,
+        hashtag = None,
+        responseType = "json",
+        page = None
+    ):
+        query = "with t_ways AS ( \
+            SELECT ways_line.osm_id as id, ways_line.timestamp, geom as geometry, tags, status FROM ways_line \
+            LEFT JOIN validation ON validation.osm_id = ways_line.osm_id \
+            WHERE \
+            {0} {1} {2} {3} \
+        ), \
+        t_features AS (  \
+            SELECT jsonb_build_object( 'type', 'Feature', 'id', id, 'properties', to_jsonb(t_ways) \
+            - 'geometry' , 'geometry', ST_AsGeoJSON(geometry)::jsonb ) AS feature FROM t_ways  \
+        ) SELECT jsonb_build_object( 'type', 'FeatureCollection', 'features', jsonb_agg(t_features.feature) ) \
+        as result FROM t_features;".format(
+            "ST_Intersects(\"geom\", ST_GeomFromText('POLYGON(({0}))', 4326) )".format(area) if area else "1=1 ",
+            "and ways_line.tags ? '{0}'".format(key) if key and not value else "",
+            "and ways_line.tags->'{0}' ~* '^{1}'".format(key, value) if key and value else "",
+            "ORDER BY ways_line.timestamp DESC LIMIT " + str(RESULTS_PER_PAGE) + " OFFSET {0}".format(page * RESULTS_PER_PAGE) if page else "",
         )
         return self.underpassDB.run(query, responseType, True)
 
@@ -61,10 +88,10 @@ def getNodes(
         responseType = "json"
     ):
         query = "with t_nodes AS ( \
-            SELECT raw_node.osm_id as id, geometry, tags, status FROM raw_node \
-            LEFT JOIN validation ON validation.osm_id = raw_node.osm_id \
+            SELECT nodes.osm_id as id, geom as geometry, tags, status FROM nodes \
+            LEFT JOIN validation ON validation.osm_id = nodes.osm_id \
             WHERE \
-            ST_Intersects(\"geometry\", \
+            ST_Intersects(\"geom\", \
             ST_GeomFromText('POLYGON(({0}))', 4326) \
             ) {1} {2} \
         ), \
@@ -74,8 +101,8 @@ def getNodes(
         ) SELECT jsonb_build_object( 'type', 'FeatureCollection', 'features', jsonb_agg(t_features.feature) ) \
         as result FROM t_features;".format(
             area,
-            "and raw_node.tags ? '{0}'".format(key) if key and not value else "",
-            "and raw_node.tags->'{0}' ~* '^{1}'".format(key, value) if key and value else "",
+            "and nodes.tags ? '{0}'".format(key) if key and not value else "",
+            "and nodes.tags->'{0}' ~* '^{1}'".format(key, value) if key and value else "",
         )
         return self.underpassDB.run(query, responseType, True)
 
@@ -92,17 +119,17 @@ def getPolygonsList(
             page = 1
 
         query = "with t_ways AS ( \
-            SELECT raw_poly.osm_id as id, ST_X(ST_Centroid(geometry)) as lat, ST_Y(ST_Centroid(geometry)) as lon, raw_poly.timestamp, tags, status FROM raw_poly \
-            LEFT JOIN validation ON validation.osm_id = raw_poly.osm_id \
+            SELECT ways_poly.osm_id as id, ST_X(ST_Centroid(geom)) as lat, ST_Y(ST_Centroid(geom)) as lon, ways_poly.timestamp, tags, status FROM ways_poly \
+            LEFT JOIN validation ON validation.osm_id = ways_poly.osm_id \
             WHERE \
             {0} {1} {2} {3} \
         ), t_features AS ( \
             SELECT to_jsonb(t_ways) as feature from t_ways \
         ) SELECT jsonb_agg(t_features.feature) as result FROM t_features;".format(
-            "ST_Intersects(\"geometry\", ST_GeomFromText('POLYGON(({0}))', 4326) )".format(area) if area else "1=1 ",
-            "and raw_poly.tags ? '{0}'".format(key) if key and not value else "",
-            "and raw_poly.tags->'{0}' ~* '^{1}'".format(key, value) if key and value else "",
-            "ORDER BY raw_poly.timestamp DESC LIMIT " + str(RESULTS_PER_PAGE) + " OFFSET {0}".format(page * RESULTS_PER_PAGE) if page else "",
+            "ST_Intersects(\"geom\", ST_GeomFromText('POLYGON(({0}))', 4326) )".format(area) if area else "1=1 ",
+            "and ways_poly.tags ? '{0}'".format(key) if key and not value else "",
+            "and ways_poly.tags->'{0}' ~* '^{1}'".format(key, value) if key and value else "",
+            "ORDER BY ways_poly.timestamp DESC LIMIT " + str(RESULTS_PER_PAGE) + " OFFSET {0}".format(page * RESULTS_PER_PAGE) if page else "",
         )
         return self.underpassDB.run(query, responseType, True)
 
@@ -119,16 +146,16 @@ def getNodesList(
             page = 1
 
         query = "with t_nodes AS ( \
-            SELECT raw_node.osm_id as id, ST_X(ST_Centroid(geometry)) as lat, ST_Y(ST_Centroid(geometry)) as lon, tags, status FROM raw_node \
-            LEFT JOIN validation ON validation.osm_id = raw_node.osm_id \
+            SELECT nodes.osm_id as id, ST_X(ST_Centroid(geom)) as lat, ST_Y(ST_Centroid(geom)) as lon, tags, status FROM nodes \
+            LEFT JOIN validation ON validation.osm_id = nodes.osm_id \
             WHERE {0} {1} {2} {3} \
         ), \
         t_features AS (  \
             SELECT to_jsonb(t_nodes) AS feature FROM t_nodes  \
         ) SELECT jsonb_agg(t_features.feature) as result FROM t_features;".format(
-            "ST_Intersects(\"geometry\", ST_GeomFromText('POLYGON(({0}))', 4326) )".format(area) if area else "1=1 ",
-            "and raw_node.tags ? '{0}'".format(key) if key and not value else "",
-            "and raw_node.tags->'{0}' ~* '^{1}'".format(key, value) if key and value else "",
-            "ORDER BY raw_node.timestamp DESC LIMIT " + str(RESULTS_PER_PAGE) + " OFFSET {0}".format(page * RESULTS_PER_PAGE) if page else "",
+            "ST_Intersects(\"geom\", ST_GeomFromText('POLYGON(({0}))', 4326) )".format(area) if area else "1=1 ",
+            "and nodes.tags ? '{0}'".format(key) if key and not value else "",
+            "and nodes.tags->'{0}' ~* '^{1}'".format(key, value) if key and value else "",
+            "ORDER BY nodes.timestamp DESC LIMIT " + str(RESULTS_PER_PAGE) + " OFFSET {0}".format(page * RESULTS_PER_PAGE) if page else "",
         )
         return self.underpassDB.run(query, responseType, True)

python/dbapi/api/report.py

@@ -42,7 +42,7 @@ def getDataQualityGeo(
             st_y(location) as lon \
             from changesets \
             INNER JOIN validation \
-            ON validation.change_id = changesets.id \
+            ON validation.changeset = changesets.id \
             where validation.status = 'badgeom' \
             {0} {1} {2} {3} \
             order by osm_id \
@@ -91,16 +91,16 @@ def getDataQualityTag(
                 {0} {1} {2} {3} \
             ), \
             t1 AS ( \
-                SELECT change_id, source, osm_id, type, \
+                SELECT changeset, source, osm_id, type, \
                 unnest(values) as unnest_values \
                 from validation, t2 \
-                where change_id = t2.id \
+                where changeset = t2.id \
             ) \
             select \
                 'https://osm.org/' || t1.type || '/' || t1.osm_id as link, \
                 t1.unnest_values as tag, t1.source \
                 FROM t1, t2 \
-                where t1.change_id = t2.id \
+                where t1.changeset = t2.id \
                 limit {4} offset {5}".format(
                     "and closed_at >= '{0}'".format(fromDate) if fromDate else "",
                     "and closed_at <= '{0}'".format(toDate) if toDate else "",
@@ -127,15 +127,15 @@ def getDataQualityTagStats(
                 {0} {1} {2} {3} \
             ), \
             t1 AS ( \
-                SELECT change_id, source, \
+                SELECT changeset, source, \
                 unnest(values) as unnest_values \
                 from validation, t2 \
-                where change_id = t2.id \
+                where changeset = t2.id \
             ) \
             SELECT \
                 t1.unnest_values as tag, t1.source, count(t1.unnest_values) \
                 FROM t1, t2  \
-                where t1.change_id = t2.id  \
+                where t1.changeset = t2.id  \
                 group by t1.unnest_values, t1.source \
                 order by count desc  \
                 limit {4} offset {5}".format(

python/restapi/main.py

@@ -51,6 +51,7 @@
 origins = [
     "http://localhost",
     "http://localhost:5000",
+    "http://localhost:3000",
     "http://127.0.0.1",
     "http://127.0.0.1:5000"
 ]
@@ -171,6 +172,15 @@ def getNodes(request: RawRequest):
     )
     return results
 
+@app.post("/raw/lines")
+def getLines(request: RawRequest):
+    results = rawer.getLines(
+        area = request.area,
+        key = request.key or "",
+        value = request.value or ""
+    )
+    return results
+
 @app.post("/raw/polygonsList")
 def getPolygonsList(request: RawRequest):
     results = rawer.getPolygonsList(

setup/underpass.sql

@@ -23,7 +23,7 @@ SET default_tablespace = '';
 CREATE TABLE IF NOT EXISTS public.changesets (
     id int8 NOT NULL,
     editor text,
-    user_id integer NOT NULL,
+    uid integer NOT NULL,
     created_at timestamptz,
     closed_at timestamptz,
     updated_at timestamptz,
@@ -46,8 +46,8 @@ CREATE TYPE public.status AS ENUM ('notags', 'complete', 'incomplete', 'badvalue
 
 CREATE TABLE IF NOT EXISTS public.validation (
     osm_id int8,
-    user_id int8,
-    change_id int8,
+    uid int8,
+    changeset int8,
     type public.objtype,
     status public.status,
     values text[],
@@ -59,31 +59,56 @@ CREATE TABLE IF NOT EXISTS public.validation (
 ALTER TABLE ONLY public.validation
     ADD CONSTRAINT validation_pkey PRIMARY KEY (osm_id, status, source);
 
-CREATE TABLE IF NOT EXISTS public.raw_poly (
+CREATE TABLE IF NOT EXISTS public.ways_poly (
     osm_id int8,
-    change_id int8,
-    geometry public.geometry(Polygon,4326),
+    changeset int8,
+    geom public.geometry(Polygon,4326),
     tags public.hstore,
     refs int8[],
     timestamp timestamp with time zone,
-    version int
+    version int,
+    "user" text,
+    uid int8
 );
 
-CREATE TABLE IF NOT EXISTS public.raw_node (
+CREATE TABLE IF NOT EXISTS public.ways_line (
     osm_id int8,
-    change_id int8,
-    geometry public.geometry(Point,4326),
+    changeset int8,
+    geom public.geometry(LineString,4326),
+    tags public.hstore,
+    refs int8[],
+    timestamp timestamp with time zone,
+    version int,
+    "user" text,
+    uid int8
+);
+
+CREATE TABLE IF NOT EXISTS public.nodes (
+    osm_id int8,
+    changeset int8,
+    geom public.geometry(Point,4326),
     tags public.hstore,
     timestamp timestamp with time zone,
-    version int
+    version int,
+    "user" text,
+    uid int8
 );
 
 CREATE TABLE IF NOT EXISTS public.way_refs (
     way_id int8,
     node_id int8
 );
 
-CREATE UNIQUE INDEX IF NOT EXISTS raw_osm_id_idx ON public.raw_node (osm_id);
-CREATE UNIQUE INDEX IF NOT EXISTS raw_poly_osm_id_idx ON public.raw_poly (osm_id);
-CREATE INDEX IF NOT EXISTS way_refs_nodes_idx ON public.way_refs (node_id);
-CREATE INDEX IF NOT EXISTS way_refs_ways_idx ON public.way_refs (way_id);
+CREATE UNIQUE INDEX nodes_id_idx ON public.nodes (osm_id);
+CREATE UNIQUE INDEX ways_poly_id_idx ON public.ways_poly (osm_id);
+CREATE UNIQUE INDEX ways_line_id_idx ON public.ways_line(osm_id);
+CREATE INDEX way_refs_node_id_idx ON public.way_refs (node_id);
+CREATE INDEX way_refs_way_id_idx ON public.way_refs (way_id);
+
+CREATE INDEX nodes_version_idx ON public.nodes (version);
+CREATE INDEX ways_poly_version_idx ON public.ways_poly (version);
+CREATE INDEX ways_line_version_idx ON public.ways_line (version);
+
+CREATE INDEX nodes_timestamp_idx ON public.nodes(timestamp DESC);
+CREATE INDEX ways_poly_timestamp_idx ON public.ways_poly(timestamp DESC);
+CREATE INDEX ways_line_timestamp_idx ON public.ways_line(timestamp DESC);