Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

pre-queries for geojson geoms in bounding box before cluster query re… #10663

Merged
merged 18 commits into from
Apr 17, 2024
Merged

pre-queries for geojson geoms in bounding box before cluster query re… #10663

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
bddfed8
pre-queries for geojson geoms in bounding box before cluster query re…
whatisgalen Mar 7, 2024
4e80fd6
fixes type issue re #10452
whatisgalen Mar 7, 2024
9ad015d
creates cache_key create method, including db snapshot of user_x_edit…
whatisgalen Mar 8, 2024
c1efb72
calls create_mvt_cache_key to make cache_key for mvt request re #10452
whatisgalen Mar 8, 2024
2eb521a
caches an empty mvt so as to not re-query for non-existent mvt re #10452
whatisgalen Mar 8, 2024
3f5901c
alters pre-query to only return count of matching geoms re #10452
whatisgalen Mar 8, 2024
1151cd3
store only search_geom count instead of geoms themselves re #10452
whatisgalen Mar 8, 2024
071f144
narrows query of matching geoms in cluster query to those within Tile…
whatisgalen Mar 8, 2024
b9aea9a
adds control flow for when search_geom_count < min_points to avoid cl…
whatisgalen Mar 8, 2024
53bf50b
rm redundant 404 exception re #10452
whatisgalen Mar 8, 2024
1fcda6e
creates receiver method on post_save for EditLog to cache instance co…
whatisgalen Mar 29, 2024
2e7c6e8
gets cached editlog_nodegroupid_count if available to make mvt_cache_…
whatisgalen Mar 29, 2024
e4688a6
restyles to conform to line char limit, sets new editlog snapshot in …
whatisgalen Mar 29, 2024
9687708
removes refs to editlog or mvt_snapshot in mvt cache key create metho…
whatisgalen Apr 5, 2024
bbd6f90
rm editlog receiver due to perform implicns re #10452
whatisgalen Apr 5, 2024
e2a3a94
rm CACHE_BY_USER entry for editlog re #10452
whatisgalen Apr 5, 2024
801bc3b
nit in settings.py re #10452
whatisgalen Apr 5, 2024
72bc030
rm redundant cache.set statement, also retain tile from earlier query…
whatisgalen Apr 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
132 changes: 84 additions & 48 deletions arches/app/views/api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -279,8 +279,9 @@ def get(self, request, nodeid, zoom, x, y):
node = models.Node.objects.get(nodeid=nodeid, nodegroup_id__in=viewable_nodegroups)
except models.Node.DoesNotExist:
raise Http404()
search_geom_count = 0
config = node.config
cache_key = f"mvt_{nodeid}_{zoom}_{x}_{y}"
cache_key = MVT.create_mvt_cache_key(node, zoom, x, y, request.user)
whatisgalen marked this conversation as resolved.
Show resolved Hide resolved
tile = cache.get(cache_key)
if tile is None:
resource_ids = get_restricted_instances(request.user, allresources=True)
Expand All @@ -293,57 +294,90 @@ def get(self, request, nodeid, zoom, x, y):
distance = arc * float(config["clusterDistance"])
min_points = int(config["clusterMinPoints"])
distance = settings.CLUSTER_DISTANCE_MAX if distance > settings.CLUSTER_DISTANCE_MAX else distance
cursor.execute(
"""WITH clusters(tileid, resourceinstanceid, nodeid, geom, cid)
AS (
SELECT m.*,
ST_ClusterDBSCAN(geom, eps := %s, minpoints := %s) over () AS cid
FROM (
SELECT tileid,
resourceinstanceid,
nodeid,
geom
FROM geojson_geometries
WHERE nodeid = %s and resourceinstanceid not in %s
) m
)

SELECT ST_AsMVT(
tile,
%s,
4096,
'geom',
'id'
) FROM (
SELECT resourceinstanceid::text,
row_number() over () as id,
1 as total,
count_query = """
SELECT count(*) FROM geojson_geometries
WHERE
ST_Intersects(geom, TileBBox(%s, %s, %s, 3857))
AND
nodeid = %s and resourceinstanceid not in %s
"""

# get the count of matching geometries
cursor.execute(count_query, [zoom, x, y, nodeid, resource_ids])
search_geom_count = cursor.fetchone()[0]

if search_geom_count >= min_points:
cursor.execute(
"""WITH clusters(tileid, resourceinstanceid, nodeid, geom, cid)
AS (
SELECT m.*,
ST_ClusterDBSCAN(geom, eps := %s, minpoints := %s) over () AS cid
FROM (
SELECT tileid,
resourceinstanceid,
nodeid,
geom
FROM geojson_geometries
WHERE
ST_Intersects(geom, TileBBox(%s, %s, %s, 3857))
AND
nodeid = %s and resourceinstanceid not in %s
) m
)
SELECT ST_AsMVT(
tile,
%s,
4096,
'geom',
'id'
) FROM (
SELECT resourceinstanceid::text,
row_number() over () as id,
1 as total,
ST_AsMVTGeom(
geom,
TileBBox(%s, %s, %s, 3857)
) AS geom,
'' AS extent
FROM clusters
WHERE cid is NULL
UNION
SELECT NULL as resourceinstanceid,
row_number() over () as id,
count(*) as total,
ST_AsMVTGeom(
ST_Centroid(
ST_Collect(geom)
),
TileBBox(%s, %s, %s, 3857)
) AS geom,
ST_AsGeoJSON(
ST_Extent(geom)
) AS extent
FROM clusters
WHERE cid IS NOT NULL
GROUP BY cid
) as tile;""",
[distance, min_points, zoom, x, y, nodeid, resource_ids, nodeid, zoom, x, y, zoom, x, y],
)
elif search_geom_count:
cursor.execute(
"""SELECT ST_AsMVT(tile, %s, 4096, 'geom', 'id') FROM (SELECT tileid,
id,
resourceinstanceid,
nodeid,
ST_AsMVTGeom(
geom,
TileBBox(%s, %s, %s, 3857)
) AS geom,
'' AS extent
FROM clusters
WHERE cid is NULL
UNION
SELECT NULL as resourceinstanceid,
row_number() over () as id,
count(*) as total,
ST_AsMVTGeom(
ST_Centroid(
ST_Collect(geom)
),
TileBBox(%s, %s, %s, 3857)
) AS geom,
ST_AsGeoJSON(
ST_Extent(geom)
) AS extent
FROM clusters
WHERE cid IS NOT NULL
GROUP BY cid
) as tile;""",
[distance, min_points, nodeid, resource_ids, nodeid, zoom, x, y, zoom, x, y],
)
1 AS total
FROM geojson_geometries
WHERE nodeid = %s and resourceinstanceid not in %s) AS tile;""",
[nodeid, zoom, x, y, nodeid, resource_ids],
)
else:
tile = ""
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It seems like this alone will prevent a requery for the MVT because the value saved to the cache isn't None which is what is being checked for prior to the query.

else:
cursor.execute(
"""SELECT ST_AsMVT(tile, %s, 4096, 'geom', 'id') FROM (SELECT tileid,
Expand All @@ -359,12 +393,14 @@ def get(self, request, nodeid, zoom, x, y):
WHERE nodeid = %s and resourceinstanceid not in %s) AS tile;""",
[nodeid, zoom, x, y, nodeid, resource_ids],
)
tile = bytes(cursor.fetchone()[0])
tile = bytes(cursor.fetchone()[0]) if tile is None else tile
cache.set(cache_key, tile, settings.TILE_CACHE_TIMEOUT)
if not len(tile):
raise Http404()
return HttpResponse(tile, content_type="application/x-protobuf")

def create_mvt_cache_key(node, zoom, x, y, user):
return f"mvt_{str(node.nodeid)}_{zoom}_{x}_{y}_{user.id}"

@method_decorator(csrf_exempt, name="dispatch")
class Graphs(APIBase):
Expand Down
2 changes: 1 addition & 1 deletion arches/settings.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -500,7 +500,7 @@
CACHE_BY_USER = {
"default": 3600 * 24, #24hrs
"anonymous": 3600 * 24 #24hrs
}
}

BYPASS_UNIQUE_CONSTRAINT_TILE_VALIDATION = False
BYPASS_REQUIRED_VALUE_TILE_VALIDATION = False
Expand Down