From 738a633d406ce84ed4b918c124a9bed43aa08553 Mon Sep 17 00:00:00 2001 From: Esteban Zimanyi Date: Tue, 9 Jul 2024 16:09:42 +0200 Subject: [PATCH] Improve overall typesetting --- docs/AIS.xml | 6 +- docs/AIS_Dashboard.xml | 1966 +++++++++++++-------------- docs/FlightDataDashboard.xml | 2406 ++++++++++++++++------------------ docs/GPX.xml | 2 +- docs/location_history.xml | 2 +- 5 files changed, 1985 insertions(+), 2397 deletions(-) diff --git a/docs/AIS.xml b/docs/AIS.xml index fc9557a..c82b238 100644 --- a/docs/AIS.xml +++ b/docs/AIS.xml @@ -210,9 +210,9 @@ SELECT COUNT(*) FROM AISInputFiltered; CREATE TABLE Ships(MMSI, Trip, SOG, COG) AS SELECT MMSI, - tgeompoint_seq(array_agg(tgeompoint_inst(ST_Transform(Geom, 25832), T) ORDER BY T)), - tfloat_seq(array_agg(tfloat_inst(SOG, T) ORDER BY T) FILTER (WHERE SOG IS NOT NULL)), - tfloat_seq(array_agg(tfloat_inst(COG, T) ORDER BY T) FILTER (WHERE COG IS NOT NULL)) + tgeompointSeq(array_agg(tgeompoint(ST_Transform(Geom, 25832), T) ORDER BY T)), + tfloatSeq(array_agg(tfloat(SOG, T) ORDER BY T) FILTER (WHERE SOG IS NOT NULL)), + tfloatSeq(array_agg(tfloat(COG, T) ORDER BY T) FILTER (WHERE COG IS NOT NULL)) FROM AISInputFiltered GROUP BY MMSI; -- Query returned successfully: 6264 rows affected, 00:52 minutes execution time. diff --git a/docs/AIS_Dashboard.xml b/docs/AIS_Dashboard.xml index d67b0b1..bfe4ac6 100644 --- a/docs/AIS_Dashboard.xml +++ b/docs/AIS_Dashboard.xml @@ -1,114 +1,98 @@ - Dashboard and Visualization of Ship Trajectories (AIS) - - This module builds on the Managing Ship Trajectories (AIS) module by - creating a business intelligence dashboard to visualize and - manipulate data. The module shows how to set up a Grafana dashboard - with an existing database, create basic visualizations, set - properties for different outputs, and use Variables to create - dynamic visuals. - -
- Contents - - The module covers the following topics: - - - - - Setting up a Grafana dashboard and connecting to a database - - - - - Visualize a statistic from simple aggregations - - - - - Visualize spatial frequency with a heat-map (not aggregated) - - - - - Visualize frequency in spatial extent with a heat-map - (pre-aggregated) - - - - - Visualize spatio-temporal proximate objects - - - - - Create dynamic queries with variables - - - -
-
- Tools - - The tools used in this module are as follows: - - - - - MobilityDB, on top of PostgreSQL and PostGIS - - - - - Grafana (version 9.0.7) - - - -
-
- Setting up the Data Source - - Data for the workshop is loaded into a MobilityDB database hosted - on Azure, with all login information provided in the [Sign-in and - Connect to Data Source] section below. - - - Alternatively, you can set up your own MobilityDB database as described in the previous module. - The raw data in CSV format is also available on the - MobilityDB-workshop - repository. - -
-
- Setting up the Visualization Dashboard - - We can use Grafana, - an open-source technology, to create a business intelligence - dashboard. This will allow different users to set up their own - queries and visualizations, or easily slice through data in a - visual way for non-technical users. - - - Start by setting up Grafana on your system: - - - - - macOS - - + Dashboard and Visualization of Ship Trajectories (AIS) + + This module builds on the Managing Ship Trajectories (AIS) module by creating a business intelligence dashboard to visualize and manipulate data. The module shows how to set up a Grafana dashboard with an existing database, create basic visualizations, set properties for different outputs, and use Variables to create dynamic visuals. + +
+ Contents + + The module covers the following topics: + + + + + Setting up a Grafana dashboard and connecting to a database + + + + + Visualize a statistic from simple aggregations + + + + + Visualize spatial frequency with a heat-map (not aggregated) + + + + + Visualize frequency in spatial extent with a heat-map (pre-aggregated) + + + + + Visualize spatio-temporal proximate objects + + + + + Create dynamic queries with variables + + + +
+
+ Tools + + The tools used in this module are as follows: + + + + + MobilityDB, on top of PostgreSQL and PostGIS + + + + + Grafana (version 9.0.7) + + + +
+
+ Setting up the Data Source + + Data for the workshop is loaded into a MobilityDB database hosted on Azure, with all login information provided in the [Sign-in and Connect to Data Source] section below. + + + Alternatively, you can set up your own MobilityDB database as described in the previous module. The raw data in CSV format is also available on the MobilityDB-workshop repository. + +
+
+ Setting up the Visualization Dashboard + + We can use Grafana, an open-source technology, to create a business intelligence dashboard. This will allow different users to set up their own queries and visualizations, or easily slice through data in a visual way for non-technical users. + + + Start by setting up Grafana on your system: + + + + + macOS + + brew update brew install grafana brew services start grafana - - - - Debian - or Ubuntu - - + + + + Debian or Ubuntu + + # Note These are instructions for Grafana Enterprise Edition (via APT repository), # which they recommend. It includes all the Open Source features and can also use # Enterprise features if you have a License. @@ -120,1008 +104,828 @@ wget -q -O - https://packages.grafana.com/gpg.key | sudo apt-key add - # Add repository for stable releases echo "deb https://packages.grafana.com/enterprise/deb stable main" - | sudo tee -a /etc/apt/sources.list.d/grafana.list + | sudo tee -a /etc/apt/sources.list.d/grafana.list # Install Grafana sudo apt-get update sudo apt-get install grafana-enterprise - - - - Windows - - - Use the Windows installer available at the Grafana website. - - - -
-
- Sign in and Connect to Data Source - - We can now sign in to Grafana by going to - http://localhost:3000/. - Set up a new account if needed. Additional instructions to login - can be found here following the - build - your first dashboard instructions. - - - Next, we add a data source for - Grafana to interact with. In this case, we can follow the - Grafana - instructions for adding a data source and search for - PostgreSQL - as the data source. - - - The workshop is using the following settings to connect to the - postgres server on Azure. - - - - - Name: DanishAIS - - - - - Host: 20.79.254.53:5432 - - - - - Database: danishais - - - - - User: mobilitydb-guest - - - - - Password: mobilitydb@guest - - - - - TLS/SSL Mode: disable - - - - - Version: 12+ - - - - - Then press save and test. - -
- Data Source settings - - - - - Data Source settings - -
-
-
- Creating a Dashboard - - With the dashboard configured, and a datasource added, we can now - build different panels to visualize data in intuitive ways. - -
- Speed of Individual Ships - - Let's visualize the speed of the ships using the previously - built query. Here we will represent it as a statistic with a - color gradient. - - - - - Add a new panel - - - - - Select DanishAIS as the data source - - - - - In Format as, change Time series to - Table and choose Edit SQL - - - - - Here you can add your SQL queries. Let’s replace the existing - query with the following SQL script: - - -SELECT mmsi, ABS( twavg(SOG) * 1.852 - twavg( speed(Trip))* 3.6 ) AS SpeedDifference + + + + Windows + + + Use the Windows installer available at the Grafana website. + + + +
+
+ Sign in and Connect to Data Source + + We can now sign in to Grafana by going to http://localhost:3000/. Set up a new account if needed. Additional instructions to login can be found here following the build your first dashboard instructions. + + + Next, we add a data source for Grafana to interact with. In this case, we can follow the Grafana instructions for adding a data source and search for PostgreSQL as the data source. + + + The workshop is using the following settings to connect to the postgres server on Azure. + + + + + Name: DanishAIS + + + + + Host: 20.79.254.53:5432 + + + + + Database: danishais + + + + + User: mobilitydb-guest + + + + + Password: mobilitydb@guest + + + + + TLS/SSL Mode: disable + + + + + Version: 12+ + + + + + Then press save and test. + +
+ Data Source settings + + + + + Data Source settings + +
+
+
+ Creating a Dashboard + + With the dashboard configured, and a datasource added, we can now build different panels to visualize data in intuitive ways. + +
+ Speed of Individual Ships + + Let's visualize the speed of the ships using the previously built query. Here we will represent it as a statistic with a color gradient. + + + + + Add a new panel + + + + + Select DanishAIS as the data source + + + + + In Format as, change Time series to Table and choose Edit SQL + + + + + Here you can add your SQL queries. Let's replace the existing query with the following SQL script: + + +SELECT MMSI, ABS( twavg(SOG) * 1.852 - twAvg( speed(Trip))* 3.6 ) AS SpeedDifference FROM Ships ORDER BY SpeedDifference DESC LIMIT 5; - - - - We can also quickly do some datatype transformations to help - Grafana correctly interpret the incoming data. Next to the - Query button, select Transform, add - Convert field type and choose - mmsi as String. - -
- Datatype transformations in Grafana - - - - - Datatype transformations in - Grafana - -
- - - - We will modify some visualization options in the - panel on the right. - - - First, choose stat as the visualization - -
- Choosing visualization type - - - - - Choosing visualization - type - -
- - Panel Options: Give the - panel the title Incorrect AIS Boat Speed - Reporting - - - Value Options: - - - - - Show: All values - - - - - Fields: speeddifference - -
- Value options dialogue box - - - - - Value options dialogue - box - -
- - - - Note: we can include a limit here instead of in - our SQL query as well. - - - Stat Styles: - - - - - Orientation: Horizontal - -
- Stat styles dialogue box - - - - - Stat styles dialogue - box - -
- - - - Standard Options: - - - - - Unit: Velocity → meter/second (m/s). Note: you - can scroll in the drop-down menu to see all - options. - - - - - Color scheme: Green-Yellow-Red (by value) - - - -
- Standard options dialogue box - - - - - Standard options dialogue - box - -
- - Thresholds: - - - - - remove the existing threshold by clicking the little - trash can icon on the right. Adding a threshold will - force the visualization to color the data a specific - color if the threshold is met. - - - -
- Thresholds dialogue box - - - - - Thresholds dialogue - box - -
- - - - The final visualization will look like the screenshot below. - -
- Individual ship speed statistics visualization - - - - - Individual ship speed statistics - visualization - -
-
-
- Routes Used Most Frequently Visualized with a Static Heat - Map - - We can visualize the routes used by ships with a heat map - generated from individual GPS points of the ships. This approach - is quite costly, so we will use TABLESAMPLE SYSTEM to specify an - approximate percentage of the data to use. If the frequency of - locations returned varies in different areas, a heatmap using - individual datapoints could be misleading without further data - pre-processing. An alternative approach could be to use the - postGIS - ST_AsGeoJSON - to generate shapes in geoJSON format which can be used in - Grafana’s - World Map Panel plugin. - - - - - Add a panel, select DanishAIS as the data source and Format - As Table. - - - - - Using Edit SQL, add the following SQL code: - - + + + + We can also quickly do some datatype transformations to help Grafana correctly interpret the incoming data. Next to the Query button, select Transform, add Convert field type and choose mmsi as String. + +
+ Datatype transformations in Grafana + + + + + Datatype transformations in Grafana + +
+ + + + We will modify some visualization options in the panel on the right. + + + First, choose stat as the visualization + +
+ Choosing visualization type + + + + + Choosing visualization type + +
+ + Panel Options: Give the panel the title Incorrect AIS Boat Speed Reporting + + + Value Options: + + + + + Show: All values + + + + + Fields: speeddifference + +
+ Value options dialogue box + + + + + Value options dialogue box + +
+ + + + Note: we can include a limit here instead of in our SQL query as well. + + + Stat Styles: + + + + + Orientation: Horizontal + +
+ Stat styles dialogue box + + + + + Stat styles dialogue box + +
+ + + + Standard Options: + + + + + Unit: Velocity → meter/second (m/s). Note: you can scroll in the drop-down menu to see all options. + + + + + Color scheme: Green-Yellow-Red (by value) + + + +
+ Standard options dialogue box + + + + + Standard options dialogue + box + +
+ + Thresholds: + + + + + remove the existing threshold by clicking the little trash can icon on the right. Adding a threshold will force the visualization to color the data a specific color if the threshold is met. + + + +
+ Thresholds dialogue box + + + + + Thresholds dialogue box + +
+ + + + The final visualization will look like the screenshot below. + +
+ Individual ship speed statistics visualization + + + + + Individual ship speed statistics visualization + +
+
+
+ Routes Used Most Frequently Visualized with a Static Heat Map + + We can visualize the routes used by ships with a heat map generated from individual GPS points of the ships. This approach is quite costly, so we will use TABLESAMPLE SYSTEM to specify an approximate percentage of the data to use. If the frequency of locations returned varies in different areas, a heatmap using individual datapoints could be misleading without further data pre-processing. An alternative approach could be to use the + PostGIS function ST_AsGeoJSON to generate shapes in geoJSON format which can be used in Grafana's World Map Panel plugin. + + + + + Add a panel, select DanishAIS as the data source and Format As Table. + + + + + Using Edit SQL, add the following SQL code: + + -- NOTE: TABLESAMPLE SYSTEM(40) returns ~40% of the data. -SELECT - latitude, - longitude, - mmsi +SELECT latitude, longitude, mmsi FROM aisinputfiltered TABLESAMPLE SYSTEM (40) - - - - Change the visualization type to - Geomap. - - - - - On the map, zoom in to fit the data points into the frame - and modify the following visualization options: - - - Panel Options: - - - - - Title: Route Usage Frequency - - - - - Map View: - - - - - Use current map setting (this will use the current zoom - and positioning level as default) - - - - - Share View: enable (this will sync up the movement and - zoom across multiple maps on the same dashboard) - - - -
- Setting initial view in map view dialogue box - - - - - Setting initial view in map view - dialogue box - -
- - Data Layer: - - - - - Layer type: Heatmap - - - - - Location: Coords - - - - - Latitude field: latitude - - - - - Longitude field: longitude - - - - - Weight values: 0.1 - - - - - Radius: 1 - - - - - Blur: 5 - - - -
- Setting up heat-map in data layer dialogue - box - - - - - Setting up heat-map in data layer - dialogue box - -
- - Standard Options: - - - - - Color scheme: Blue-Yellow-Red (by value). - - - -
- Choosing color scheme in standard options dialogue - box - - - - - Choosing color scheme in standard - options dialogue box - -
- - - - The final visualization will look like the screenshot below. - - - Note: The number of datapoints rendered can be manipulated by - changing the parameter of the TABLESAMPLE SYSTEM() call in the - query. - -
- Route usage frequency heat-map visualization - - - - - Route usage frequency heat-map - visualization - -
-
-
- Number of Boats Moving Through a Given Area - - - - Create a new panel, and set DanishAIS as the Source, Format - as: Table. - - - - - Select visualization as: Geomap - - - - - Add this SQL in the SQL editor section - - + + + + Change the visualization type to Geomap. + + + + + On the map, zoom in to fit the data points into the frame and modify the following visualization options: + + + Panel Options: + + + + + Title: Route Usage Frequency + + + + + Map View: + + + + + Use current map setting (this will use the current zoom and positioning level as default) + + + + + Share View: enable (this will sync up the movement and zoom across multiple maps on the same dashboard) + + + +
+ Setting initial view in map view dialogue box + + + + + Setting initial view in map view dialogue box + +
+ + Data Layer: + + + + + Layer type: Heatmap + + + + + Location: Coords + + + + + Latitude field: latitude + + + + + Longitude field: longitude + + + + + Weight values: 0.1 + + + + + Radius: 1 + + + + + Blur: 5 + + + +
+ Setting up heat-map in data layer dialogue box + + + + + Setting up heat-map in data layer dialogue box + +
+ + Standard Options: + + + + + Color scheme: Blue-Yellow-Red (by value). + + + +
+ Choosing color scheme in standard options dialogue box + + + + + Choosing color scheme in standard options dialogue box + +
+ + + + The final visualization will look like the screenshot below. + + + Note: The number of datapoints rendered can be manipulated by changing the parameter of the TABLESAMPLE SYSTEM() call in the query. + +
+ Route usage frequency heat-map visualization + + + + + Route usage frequency heat-map visualization + +
+
+
+ Number of Boats Moving Through a Given Area + + + + Create a new panel, and set DanishAIS as the Source, Format as: Table. + + + + + Select visualization as: Geomap + + + + + Add this SQL in the SQL editor section + + -- Table with bounding boxes over regions of interest -WITH ports(port_name, port_geom, lat, lng) - AS (SELECT p.port_name, p.port_geom, lat, lng - FROM - -- ST_MakeEnvelope creates geometry against which to check intersection - (VALUES ('Rodby', - ST_MakeEnvelope(651135, 6058230, 651422, 6058548, 25832)::geometry, - 54.53, 11.06), - ('Puttgarden', - ST_MakeEnvelope(644339, 6042108, 644896, 6042487, 25832)::geometry, - 54.64, 11.36)) AS p(port_name, port_geom, lat, lng)) - +WITH ports(port_name, port_geom, lat, lng) AS ( + SELECT p.port_name, p.port_geom, lat, lng + FROM (VALUES + -- ST_MakeEnvelope creates geometry against which to check intersection + ('Rodby', ST_MakeEnvelope( + 651135, 6058230, 651422, 6058548, 25832)::geometry, 54.53, 11.06), + ('Puttgarden', ST_MakeEnvelope( + 644339, 6042108, 644896, 6042487, 25832)::geometry, 54.64, 11.36)) + AS p(port_name, port_geom, lat, lng)) -- p.lat and p.lng will be used to place the port location on the visualization SELECT P.port_name, - sum(numSequences(atGeometry(S.Trip, P.port_geom))) AS trips_intersect_with_port, - p.lat, - p.lng -FROM ports AS P, - Ships AS S -WHERE eintersects(S.Trip, P.port_geom) + sum(numSequences(atGeometry(S.Trip, P.port_geom))) AS trips_intersect_with_port, + p.lat, p.lng +FROM ports AS P, Ships AS S +WHERE eIntersects(S.Trip, P.port_geom) GROUP BY P.port_name, P.lat, P.lng - - Note: You will see queries are build using the - WITH statement (common table expressions - CTE). This helps - to break the query down into parts, and also helps make it - easier to understand by others. - - - - - The options (visualization settings - on the right side of - the screen) should be as follows: - - - Data Layer - - - - - Layer type: → markers - - - - - Style Size: → Fixed and value: 20 - - - - - Color: → trips_intersect_with_port (This - will color points on the map based on this value) - - - - - Standard options - - - - - Min → 88 - - - - - Max → 97 - - - - - Color scheme → Green-Yellow-Red (by - value) - - - - - Note: At the writing of this tutorial, the Geomap - plugin is in beta and has some minor bugs with how colors - are rendered based when the Min and - Max values are auto calculated. - - - - - In the visualization below we can see port Rodby has a higher - number of ships coming and going to it and that’s why it is - colored red. This visualization can show relative activity of - ships in regions and ports. - -
- Frequency intersecting with geometric envelop - visualization - - - - - Frequency intersecting with geometric - envelop visualization - -
-
-
- Boats in Proximity in a Given Time Range - - Follow the similar steps to add a Geomap panel as before, we - include the following SQL script: - - --- 2 CTEs are help to make these queries user-friendly; TimeShips and TimeClosestShips. -WITH - -- The TimeShips CTE returns the data for a time period from 1am to 6:30am - TimeShips AS ( - SELECT - MMSI, - atTime(S.Trip, tstzspan '[2018-01-04 01:00:00, 2018-01-04 06:30:00)' ) AS trip - FROM - Ships S -), - -- The TimeClosestShips CTE returns the time, location, and closest distance of the boats - -- that are within 300m of each other. Note the use of dwithin in the WHERE clause - -- improves performance by limiting the computation to only those ships that were within - -- 300m. - TimeClosestShips AS ( - SELECT - S1.MMSI AS "boat1", S2.MMSI AS "boat_2", - startValue( atMin(S1.trip <-> S2.trip)) AS closet_distance, - startTimestamp( atMin(S1.trip <-> S2.trip)) AS time_at_closest_dist, - S1.trip AS "b1_trip", - S2.trip AS "b2_trip" - FROM - TimeShips S1, TimeShips S2 - WHERE - S1.MMSI > S2.MMSI AND - edwithin(S1.Trip, S2.Trip, 300) -) --- The final SELECT is used to project the time_at_closest_distance onto the sequence of --- locations to return the lat and long of both ships. -SELECT t.boat1, t.boat_2, t.closet_distance, t.time_at_closest_dist, - ST_X( ST_Transform( valueAtTimestamp(b1_trip, time_at_closest_dist), 4326) ) AS b1_lng, - ST_Y( ST_Transform( valueAtTimestamp(b1_trip, time_at_closest_dist), 4326) ) AS b1_lat, - ST_X( ST_Transform( valueAtTimestamp(b2_trip, time_at_closest_dist), 4326) ) AS b2_lng, - ST_Y( ST_Transform( valueAtTimestamp(b2_trip, time_at_closest_dist), 4326) ) AS b2_lat - + + Note: You will see queries are build using the WITH statement (common table expressions - CTE). This helps to break the query down into parts, and also helps make it easier to understand by others. + + + + + The options (visualization settings - on the right side of the screen) should be as follows: + + + Data Layer + + + + + Layer type: → markers + + + + + Style Size: → Fixed and value: 20 + + + + + Color: → trips_intersect_with_port (This will color points on the map based on this value) + + + + + Standard options + + + + + Min → 88 + + + + + Max → 97 + + + + + Color scheme → Green-Yellow-Red (by value) + + + + + Note: At the writing of this tutorial, the Geomap plugin is in beta and has some minor bugs with how colors are rendered based when the Min and Max values are auto calculated. + + + + + In the visualization below we can see port Rodby has a higher number of ships coming and going to it and that's why it is colored red. This visualization can show relative activity of ships in regions and ports. + +
+ Frequency intersecting with geometric envelop + visualization + + + + + Frequency intersecting with geometric envelop visualization + +
+
+
+ Boats in Proximity in a Given Time Range + + Follow the similar steps to add a Geomap panel as before, we include the following SQL script: + + +WITH TimeShips(MMSI, Trip) AS ( + SELECT MMSI, + atTime(Trip, tstzspan '[2018-01-04 01:00:00, 2018-01-04 06:30:00)' ) + FROM Ships ), +TimeClosestShips(Boat1, Boat2, closestDistance, timeAtClosestDist, tripB1, tripB2) AS ( + SELECT S1.MMSI, S2.MMSI, + startValue(atMin(S1.trip <-> S2.trip)), + startTimestamp(atMin(S1.trip <-> S2.trip)), + S1.trip AS tripB1, S2.trip + FROM TimeShips S1, TimeShips S2 + WHERE S1.MMSI > S2.MMSI AND edwithin(S1.Trip, S2.Trip, 300) ) +SELECT t.boat1, t.boat_2, t.closet_distance, t.timeAtClosestDist, + ST_X(ST_Transform(valueAtTimestamp(tripB1, timeAtClosestDist), 4326) ) AS b1_lng, + ST_Y(ST_Transform(valueAtTimestamp(tripB1, timeAtClosestDist), 4326) ) AS b1_lat, + ST_X(ST_Transform(valueAtTimestamp(tripB2, timeAtClosestDist), 4326) ) AS b2_lng, + ST_Y(ST_Transform(valueAtTimestamp(tripB2, timeAtClosestDist), 4326) ) AS b2_lat FROM TimeClosestShips t; - - To add the points to the map modify the following options: - - - Panel Options: - - - - - Title: Ships within 300m - - - - - Map View: - - - - - Share view: enabled - - - - - Data Layer: - - - - - Layer 1: rename to Boat1 - - - - - Layer type: Heatmap - - - - - Location: Coords - - - - - Latitude field: b1_lat - - - - - Longitude field: b1_lng - - - - - Radius: 5 - - - - - Blur: 15 - - - - - Click on + Add layer to add another heat map - layer to the data, this time using b2_lat and b2_long as the - coordinates. We can also add a layer to show the precise - locations with markers for both ships (using b1_lat, b1_lng, - b2_lat and b2_long), setting each marker to a different color. - For the Boat 1 and Boat 2 Locations, we use the following - options: - - - Data Layer: - - - - - Value: 1 - - - - - Color: select different color for each boat. - - - -
- Multiple layers in data layers dialogue box - - - - - Multiple layers in data layers dialogue - box - -
- - The final visualization looks like the below. - -
- Visualization of ships within 300m using heat-map - - - - - Visualization of ships within 300m using - heat-map - -
- - It’s helpful to include the tooltip for layers to allow users to - see the data behind the visualization, which helps in - interpretation and is a good way for subject-matter-experts to - provide concrete feedback. Using the tooltip, we can quickly see - that the same ship can be within 300m to multiple other ships in - the same time frame (as seen in the screenshot below). This can - result in a higher frequency of results in a heat map view than - expected. SQL queries should be modified to ensure they are - correctly interpreted. - - - Not surprisingly, we see there are lots of results for proximity - within ports. We could avoid including results in ports by - excluding all results that occur within envelopes defined by - ST_MakeEnvelope, as seen in the previous queries. - -
- Multiple results for the same ship at various times while - in a port - - - - - Multiple results for the same ship at - various times while in a port - -
-
-
-
- Dynamic Dashboards - Creating Variables - - We can use variables in Grafana to manipulate time-ranges that are - used as inputs to MobilityDB queries. We’ll create a drop-down - type variable called - FromTime that - will be used as an input for the time period within which a query - returns results. - - - - - In the dashboard window, click Dashboard - settings icon; the gear symbol, on the - top-slightly-right of the window. - -
- Dashboard settings gear box - - - - - Dashboard settings gear - box - -
- - - - Click on the Variables in the next window on - the top-left side of the screen. - -
- Selecting Variables in dashboard settings - - - - - Selecting Variables in dashboard - settings - -
- - - - You’ll see a screen that explains the variables in Grafana and - also points to the - Templates - and variables documentation. Click on the Add - variable button. - - - - - In General - - - - - Name → FromTime - - - - - Type → Custom - - - - - - - In Custom options we will manually add all the - time ranges with 1 hour increment. e.g. 2018-01-04 - 00:00:00, 2018-01-04 01:00:00 … 2018-01-04 23:00:00 - - - - - You get a screen like below. Towards the bottom there is also - a Preview of values that shows what the - drop-down options will look like for the variable you created. - In this case, we are creating the timestamps in the same - format that MobilityDB will accept. - -
- Creating user-defined list of custom variables - - - - - Creating user-defined list of custom - variables - -
- - - - We can create another variable called ToTime - with values shifted 1 hour. So the starting value would be - 2018-01-04 01:00:00 and the final value will be - 2018-01-05 00:00:00. - - - - - Now we can modify some queries by including the newly - created variables which will return results from a specific time - window. We have now provided a user with the ability to - dynamically modify visualization queries and slice through time. - -
- Dynamic Query: Number of Boats Moving Through a Given Area - in a Certain Time Period - - In the query code we just need to make slight changes for it to - take time values from the variables. In the original query, - shown below: - - + + We explain next the above query. The TimeShips CTE returns the data for a time period from 1 am to 6:30 am The TimeClosestShips CTE returns the time, location, and closest distance of the boats that are within 300 m of each other. Note the use of dWithin in the WHERE clause improves performance by limiting the computation to only those ships that were within 300 m. The final SELECT is used to project the timeAtClosestDistance onto the sequence of locations to return the lat and long of both ships. + + + To add the points to the map modify the following options: + + + Panel Options: + + + + + Title: Ships within 300m + + + + + Map View: + + + + + Share view: enabled + + + + + Data Layer: + + + + + Layer 1: rename to Boat1 + + + + + Layer type: Heatmap + + + + + Location: Coords + + + + + Latitude field: b1_lat + + + + + Longitude field: b1_lng + + + + + Radius: 5 + + + + + Blur: 15 + + + + + Click on + Add layer to add another heat map layer to the data, this time using b2_lat and b2_long as the coordinates. We can also add a layer to show the precise locations with markers for both ships (using b1_lat, b1_lng, b2_lat and b2_long), setting each marker to a different color. For the Boat 1 and Boat 2 Locations, we use the following options: + + + Data Layer: + + + + + Value: 1 + + + + + Color: select different color for each boat. + + + +
+ Multiple layers in data layers dialogue box + + + + + Multiple layers in data layers dialogue box + +
+ + The final visualization looks like the below. + +
+ Visualization of ships within 300m using heat-map + + + + + Visualization of ships within 300m using heat-map + +
+ + It's helpful to include the tooltip for layers to allow users to see the data behind the visualization, which helps in interpretation and is a good way for subject-matter-experts to provide concrete feedback. Using the tooltip, we can quickly see that the same ship can be within 300m to multiple other ships in the same time frame (as seen in the screenshot below). This can result in a higher frequency of results in a heat map view than expected. SQL queries should be modified to ensure they are correctly interpreted. + + + Not surprisingly, we see there are lots of results for proximity within ports. We could avoid including results in ports by excluding all results that occur within envelopes defined by ST_MakeEnvelope, as seen in the previous queries. + +
+ Multiple results for the same ship at various times while in a port + + + + + Multiple results for the same ship at various times while in a port + +
+
+
+
+ Dynamic Dashboards - Creating Variables + + We can use variables in Grafana to manipulate time-ranges that are used as inputs to MobilityDB queries. We'll create a drop-down type variable called FromTime that will be used as an input for the time period within which a query returns results. + + + + + In the dashboard window, click Dashboard settings icon; the gear symbol, on the top-slightly-right of the window. + +
+ Dashboard settings gear box + + + + + Dashboard settings gear box + +
+ + + + Click on the Variables in the next window on the top-left side of the screen. + +
+ Selecting Variables in dashboard settings + + + + + Selecting Variables in dashboard settings + +
+ + + + You'll see a screen that explains the variables in Grafana and also points to the Templates and variables documentation. Click on the Add variable button. + + + + + In General + + + + + Name → FromTime + + + + + Type → Custom + + + + + + + In Custom options we will manually add all the time ranges with 1 hour increment. e.g. 2018-01-04 00:00:00, 2018-01-04 01:00:00 … 2018-01-04 23:00:00 + + + + + You get a screen like below. Towards the bottom there is also a Preview of values that shows what the drop-down options will look like for the variable you created. In this case, we are creating the timestamps in the same format that MobilityDB will accept. + +
+ Creating user-defined list of custom variables + + + + + Creating user-defined list of custom variables + +
+ + + + We can create another variable called ToTime with values shifted 1 hour. So the starting value would be 2018-01-04 01:00:00 and the final value will be 2018-01-05 00:00:00. + + + + + Now we can modify some queries by including the newly created variables which will return results from a specific time window. We have now provided a user with the ability to dynamically modify visualization queries and slice through time. + +
+ Dynamic Query: Number of Boats Moving Through a Given Area in a Certain Time Period + + In the query code we just need to make slight changes for it to take time values from the variables. In the original query, shown below: + + SELECT P.port_name, - sum( numSequences( atGeometry( S.Trip, P.port_geom))) AS trips_intersect_with_port, - p.lat, - p.lng + sum( numSequences( atGeometry( S.Trip, P.port_geom))) AS trips_intersect_with_port, + p.lat, p.lng FROM ports AS P, Ships AS S -WHERE eintersects(S.Trip, P.port_geom) +WHERE eIntersects(S.Trip, P.port_geom) GROUP BY P.port_name, P.lat, P.lng - - We just need to modify the trips_intersect_with_port parameter - in the SELECT statement to look like: - - -sum -(numSequences(atGeometry( atTime(S.Trip, tstzspan '[$FromTime, $ToTime)'), P.port_geom))) - AS trips_intersect_with_port + + We just need to modify the trips_intersect_with_port parameter in the SELECT statement to look like: + + +sum(numSequences(atGeometry(atTime(S.Trip, tstzspan '[$FromTime, $ToTime)'), P.port_geom))) + AS trips_intersect_with_port - - Essentially we just wrapped S.Trip with - atTime() and passed our custom tstzspan range. - The full query with this modification is below: - - + + Essentially we just wrapped S.Trip with atTime() and passed our custom tstzspan range. The full query with this modification is below: + + -- Table with bounding boxes over regions of interest -WITH ports(port_name, port_geom, lat, lng) - AS (SELECT p.port_name, p.port_geom, lat, lng - FROM - (VALUES ('Rodby', - ST_MakeEnvelope(651135, 6058230, 651422, 6058548, 25832)::geometry, - 54.53, 11.06), - ('Puttgarden', - ST_MakeEnvelope(644339, 6042108, 644896, 6042487, 25832)::geometry, - 54.64, 11.36)) AS p(port_name, port_geom, lat, lng)) - +WITH ports(port_name, port_geom, lat, lng) AS ( + SELECT p.port_name, p.port_geom, lat, lng + FROM (VALUES + ('Rodby', ST_MakeEnvelope(651135, 6058230, 651422, 6058548, 25832)::geometry, + 54.53, 11.06), + ('Puttgarden', ST_MakeEnvelope(644339, 6042108, 644896, 6042487, 25832)::geometry, + 54.64, 11.36)) AS p(port_name, port_geom, lat, lng)) SELECT P.port_name, - sum(numSequences(atGeometry(atTime(S.Trip, tstzspan '[$FromTime, $ToTime)'), - P.port_geom))) AS trips_intersect_with_port, - p.lat, - p.lng -FROM ports AS P, - Ships AS S -WHERE eintersects(S.Trip, P.port_geom) + sum(numSequences(atGeometry(atTime(S.Trip, tstzspan '[$FromTime, $ToTime)'), + P.port_geom))) AS trips_intersect_with_port, + p.lat, p.lng +FROM Ports AS P, Ships AS S +WHERE eIntersects(S.Trip, P.port_geom) GROUP BY P.port_name, P.lat, P.lng - - We can select the start time, FromTime → - 2018-01-04 02:00:00 & ToTime → - 2018-01-04 06:00:00 . As we can see below, the - port Rodby has less activity during this period and that’s why - it is green now. But overall Rodby has more activity so when we - look at the entire days data it is colored red. - -
- Visualization of geometry intersection using dynamic - variables - - - - - Visualization of geometry intersection - using dynamic variables - -
-
-
- Global Variables - - Grafana also has some - built-in - variables (global variables) that can be used to - accomplish the same thing we did with custom variables. We can - use the global variables ${__from:date} and ${__to:date} instead - of the $FromTime and $ToTime we created. The time range can then - be modified with the time range options in the top right of the - dashboard. - -
- Assigning time range using global variables - - - - - Assigning time range using global - variables - -
- - Note: It is important to be aware of the timezone used - for the underlying data relative to the queries for global - variables. Time zones can be adjusted at the bottom of the time - range selection, Change time settings. For this - example, we change the time zone to UTC to match our - dataset. - -
-
-
- Final Dashboard - - The final dashboard will look like this. Note there are a couple - additional query views that were not covered explicitly in the - workshop. - -
- Full Dashboard - - - - - Full Dashboard - -
-
+ + We can select the start time, FromTime → 2018-01-04 02:00:00 and ToTime → 2018-01-04 06:00:00. As we can see below, the port Rodby has less activity during this period and that's why it is green now. But overall Rodby has more activity so when we look at the entire days data it is colored red. + +
+ Visualization of geometry intersection using dynamic variables + + + + + Visualization of geometry intersection using dynamic variables + +
+
+
+ Global Variables + + Grafana also has some built-in variables (global variables) that can be used to accomplish the same thing we did with custom variables. We can use the global variables ${__from:date} and ${__to:date} instead of the $FromTime and $ToTime we created. The time range can then be modified with the time range options in the top right of the dashboard. + +
+ Assigning time range using global variables + + + + + Assigning time range using global variables + +
+ + Note: It is important to be aware of the timezone used for the underlying data relative to the queries for global variables. Time zones can be adjusted at the bottom of the time range selection, Change time settings. For this example, we change the time zone to UTC to match our dataset. + +
+
+
+ Final Dashboard + + The final dashboard will look like this. Note there are a couple additional query views that were not covered explicitly in the workshop. + +
+ Full Dashboard + + + + + Full Dashboard + +
+
diff --git a/docs/FlightDataDashboard.xml b/docs/FlightDataDashboard.xml index f43d5c0..bd1f41e 100644 --- a/docs/FlightDataDashboard.xml +++ b/docs/FlightDataDashboard.xml @@ -1,1389 +1,1173 @@ - Managing Flight Data and Creating Dashboard with Grafana -
- Contents - - The module covers the following topics in 3 parts: - - - Part 1 - Data and Environment Preparation - - - - - Preparing the Database - - - - - Data Cleaning - - - - - Setting up the Dashboard and Connecting to Data Source - - - - - Part 2 - Working with Discrete Points - + Managing Flight Data and Creating Dashboard with Grafana +
+ Contents + + The module covers the following topics in 3 parts: + + + Part 1 - Data and Environment Preparation + + + + + Preparing the Database + + + + + Data Cleaning + + + + + Setting up the Dashboard and Connecting to Data Source + + + + + Part 2 - Working with Discrete Points + - - - - Visualizing time-series data for a single airplane - - - - - Visualizing discrete geographic points on a map - - - - - Part 3 - Working with Continuous Trajectories in MobilityDB - - - - - Creating trajectories for individual flights - - - - - Visualizing statistics from temporal aggregations - - - - - Visualizing statistics from multiple queries returning temporal aggregations - - - - - Returning value changes from temporal data - - - - - Visualizing spatial statistics from nested temporal conditions (intrinsic and dynamic) - - - -
-
- Tools - - The tools used in this module are as follows: - - - - - MobilityDB, on top of PostgreSQL and PostGIS - - - - - Grafana (version 9.0.7) - - - -
-
- Part 1 - Data and Environment Preparation -
- Preparing the Database - - The opensky data can be found in this Dataset - link - . - - - Create a new database opensky, then use your SQL - editor to create the MobilityDB extension as follows: - - + + + + Visualizing time-series data for a single airplane + + + + + Visualizing discrete geographic points on a map + + + + + Part 3 - Working with Continuous Trajectories in MobilityDB + + + + + Creating trajectories for individual flights + + + + + Visualizing statistics from temporal aggregations + + + + + Visualizing statistics from multiple queries returning temporal aggregations + + + + + Returning value changes from temporal data + + + + + Visualizing spatial statistics from nested temporal conditions (intrinsic and dynamic) + + + +
+
+ Tools + + The tools used in this module are as follows: + + + + + MobilityDB, on top of PostgreSQL and PostGIS + + + + + Grafana (version 9.0.7) + + + +
+
+ Part 1 - Data and Environment Preparation +
+ Preparing the Database + + The OpenSky data can be found in this Dataset + link. + + + Create a new database opensky, then use your SQL editor to create the MobilityDB extension as follows: + + CREATE EXTENSION MobilityDB CASCADE; - - - The CASCADE command will additionally create the PostGIS - extension. - - - Now create a table in which the CSV file will be loaded: - - + + + The CASCADE command will additionally create the PostGIS extension. + + + Now create a table in which the CSV file will be loaded: + + CREATE TABLE flights( - et bigint, - icao24 varchar(20), - lat float, - lon float, - velocity float, - heading float, - vertrate float, - callsign varchar(10), - onground boolean, - alert boolean, - spi boolean, - squawk integer, - baroaltitude numeric(7,2), - geoaltitude numeric(7,2), - lastposupdate numeric(13,3), - lastcontact numeric(13,3) + et bigint, + icao24 varchar(20), + lat float, + lon float, + velocity float, + heading float, + vertrate float, + callsign varchar(10), + onground boolean, + alert boolean, + spi boolean, + squawk integer, + baroaltitude numeric(7,2), + geoaltitude numeric(7,2), + lastposupdate numeric(13,3), + lastcontact numeric(13,3) ); - - Load the data into the database using the following command. - Replace the <path_to_file> with the - actual path of the CSV file. Do this for all files. As before, if this command - throws a permission error, you can use the \copy command - from the psql shell. - - + + Load the data into the database using the following command. Replace the <path_to_file> with the actual path of the CSV file. Do this for all files. As before, if this command throws a permission error, you can use the \copy command from the psql shell. + + COPY flights(et, icao24, lat, lon, velocity, heading, - vertrate, callsign, onground, alert, spi, squawk, - baroaltitude, geoaltitude, lastposupdate, lastcontact) + vertrate, callsign, onground, alert, spi, squawk, + baroaltitude, geoaltitude, lastposupdate, lastcontact) FROM '<path_to_file>' DELIMITER ',' CSV HEADER; - - All the times in this dataset are in unix timestamp (an integer) - with timezone being UTC. So we need to convert them to PostgreSQL - timestamp type. - - + + All the times in this dataset are in Unix timestamp (an integer) with timezone being UTC. So we need to convert them to PostgreSQL timestamp type. + + ALTER TABLE flights - ADD COLUMN et_ts timestamp, - ADD COLUMN lastposupdate_ts timestamp, - ADD COLUMN lastcontact_ts timestamp; + ADD COLUMN et_ts timestamp, + ADD COLUMN lastposupdate_ts timestamp, + ADD COLUMN lastcontact_ts timestamp; UPDATE flights - SET et_ts = to_timestamp(et), - lastposupdate_ts = to_timestamp(lastposupdate), - lastcontact_ts = to_timestamp(lastcontact); + SET et_ts = to_timestamp(et), + lastposupdate_ts = to_timestamp(lastposupdate), + lastcontact_ts = to_timestamp(lastcontact); - - You can check the size of the database with: - - + + You can check the size of the database with: + + SELECT pg_size_pretty( pg_total_relation_size('flights') ); - -
-
- Data Cleaning - - Delete all icao24 that have all NULL latitudes - - + +
+
+ Data Cleaning + + Delete all icao24 that have all NULL latitudes + + -- icao24_with_null_lat is used to provide a list of rows that will be deleted WITH icao24_with_null_lat AS ( - SELECT icao24, COUNT(lat) - FROM flights - GROUP BY icao24 - HAVING COUNT(lat) = 0 - ) + SELECT icao24, COUNT(lat) + FROM flights + GROUP BY icao24 + HAVING COUNT(lat) = 0 ) DELETE FROM flights WHERE icao24 IN -- this SELECT statement is needed for the IN statement to compare against a list (SELECT icao24 FROM icao24_with_null_lat); - - - Note: This data cleaning is not comprehensive. It was just to - highlight that before creating trajectories, it may be very - important to have a look at the data and do some cleaning as - that will directly impact the quality of mobilityDB trajectories - being created. If there as NULLs in mobilityDB trajectories, - some operation on it can give error. - -
-
- Setting up the Dashboard and Connecting to Data - Source - - - Data for the workshop is loaded into a MobilityDB database hosted - on Azure, with all login information provided in the - - Sign-in - and Connect to Data Source - - section below. - - - The workshop is using the following settings in Grafana to connect - to the postgres server on Azure. More detailed instruction to - set up Grafana can be found in section 2.3 to 2.5 of the Dashboard - and Visualization of Ship Trajectories (AIS) workshop. - - - - - Name: OpenSkyLOCAL - - - - - Host: - 20.79.254.53:5432 - - - - - Database: opensky - - - - - User: - mobilitydb-guest - - - - - Password: - mobilitydb@guest - - - - - TLS/SSL Mode: - disable - - - - - Version: - 12+ - - - - - The data used for this workshop provided by - The OpenSky - Network. This is data from a 24hr period from June 1, 2020 - (dataset - link). The raw data is originally provided in separate CSV - documents for each hour of the day. - - - Open a new browser and go to - http://localhost:3000/ - to work in your instance of Grafana. With a new dashboard we can - start creating the panels below. - -
-
+ + + Note: This data cleaning is not comprehensive. It was just to highlight that before creating trajectories, it may be very important to have a look at the data and do some cleaning as that will directly impact the quality of MobilityDB trajectories being created. If there as NULLs in MobilityDB trajectories, some operation on it can give error. + +
+
+ Setting up the Dashboard and Connecting to Data + Source + + + Data for the workshop is loaded into a MobilityDB database hosted on Azure, with all login information provided in the Sign-in and Connect to Data Source section below. + + + The workshop is using the following settings in Grafana to connect to the postgres server on Azure. More detailed instruction to set up Grafana can be found in section 2.3 to 2.5 of the Dashboard and Visualization of Ship Trajectories (AIS) workshop. + + + + + Name: OpenSkyLOCAL + + + + + Host: + 20.79.254.53:5432 + + + + + Database: opensky + + + + + User: + mobilitydb-guest + + + + + Password: + mobilitydb@guest + + + + + TLS/SSL Mode: + disable + + + + + Version: + 12+ + + + + + The data used for this workshop provided by The OpenSky Network. This is data from a 24 hour period from June 1, 2020 (dataset link). The raw data is originally provided in separate CSV documents for each hour of the day. + + + Open a new browser and go to http://localhost:3000/ to work in your instance of Grafana. With a new dashboard we can start creating the panels below. + +
+
- Part 2 - Working with Discrete Points -
- Visualizing 24hr Flight Pattern of Single Airplane - - We will start by looking at a single airplane. Grafana proves to - be a good way to quickly visualize our dataset and can be useful - to support pre-processing and cleaning. If using a connection to - the Azure database, required tables are already created. - - - A full description of each parameter is included in the - OpenSky - original dataset readme. The table structure in the Azure - dataset after loading and transformations looks like the - following: - -
- First row of table <quote>single_airframe</quote>, with - 24hrs of flight information for airplane - <quote>c827a6</quote> - - - - - - - First row of table - single_airframe, with 24hrs of flight information - for airplane - c827a6 - - - -
-
- Full table <quote>single_airframe_traj</quote> for airplane - <quote>c827a6</quote> - with data in mobilityDB trajectories - format - - - - - - - Full table - single_airframe_traj - for airplane - c827a6 - with data in mobilityDB trajectories - format - - - -
-
- First row of table <quote>flight_traj_sample</quote>, which - includes 200 flight trajectories. - - - - - - - First row of table - flight_traj_sample, which includes 200 flight - trajectories. - - - -
-
- Change Timezone in Grafana - - Make Sure you are visualizing the data in the correct timezone. - The data we had was in UTC. To change the timezone, - - - - - Click on the time-range panel on the top-right of the - window. - -
- Grafana time range panel - - - - - - Grafana time range - panel - - - -
- - - - In the pop-up window, on the bottom there is Change - time settings. Click that to set the desired - timezone. - - - -
-
- Visualize the Coordinates of a Single Airplane - - Let’s visualize the latitude and longitude coordinates of an - airplane’s journey throughout the day. For this one we will not - color the geo-markers, but it is possible to color them using - some criterion. - - - - - Add a new panel - - - - - Select OpenSkyLOCAL as the data source - - - - - In Format as, change Time series to - Table - and choose - Edit SQL - - - - - Here you can add your SQL queries. Let’s replace the existing - query with the following SQL script: - - + Part 2 - Working with Discrete Points +
+ Visualizing 24-hour Flight Pattern of Single Airplane + + We will start by looking at a single airplane. Grafana proves to be a good way to quickly visualize our dataset and can be useful to support pre-processing and cleaning. If using a connection to the Azure database, required tables are already created. + + + A full description of each parameter is included in the OpenSky original dataset readme. The table structure in the Azure dataset after loading and transformations looks like the following: + +
+ First row of table <varname>single_airframe</varname>, with 24 hours of flight information for airplane <varname>c827a6</varname> + + + + + + + First row of table single_airframe, with 24 hours of flight information for airplane c827a6 + + + +
+
+ Full table <varname>single_airframe_traj</varname> for airplane <varname>c827a6</varname> + with data in MobilityDB trajectories format + + + + + + + Full table single_airframe_traj for airplane + c827a6 with data in MobilityDB trajectories format + + + +
+
+ First row of table <varname>flight_traj_sample</varname>, which includes 200 flight trajectories. + + + + + + + First row of table flight_traj_sample, which includes 200 flight trajectories. + + + +
+
+ Change Timezone in Grafana + + Make sure you are visualizing the data in the correct timezone. The data we had was in UTC. To change the timezone, + + + + + Click on the time-range panel on the top-right of the window. + +
+ Grafana time range panel + + + + + + Grafana time range panel + + + +
+ + + + In the pop-up window, on the bottom there is Change time settings. Click that to set the desired timezone. + + + +
+
+ Visualize the Coordinates of a Single Airplane + + Let's visualize the latitude and longitude coordinates of an airplane's journey throughout the day. For this one we will not color the geo-markers, but it is possible to color them using some criterion. + + + + + Add a new panel + + + + + Select OpenSkyLOCAL as the data source + + + + + In Format as, change Time series to Table + and choose Edit SQL + + + + + Here you can add your SQL queries. Let's replace the existing query with the following SQL script: + + -- icao24 is the unique identifier for each airframe (airplane) SELECT et_ts, icao24, lat, lon -- TABLESAMPLE SYSTEM (n) returns only n% of the data from the table. FROM flights TABLESAMPLE SYSTEM (5) WHERE icao24 IN ('738286') AND $__timeFilter(et_ts) - - - - - Change the visualization type to - “Geomap”. - - - - - The options (visualization settings - on the right side of - the screen) should be as follows: - - - Panel Options - - - - - Title →GPS location over time - - - - - Map View - - - - - Initial view: For this one zoom in on the visualization - on the panel as you see fit and then click use - current map settings - button. - - - - - Data Layer - - - - - Layer type: → - markers - - - - - Style size → Fixed Value: 2 - - - - - Color → Green - - - - - - - In this visualization we can see that the airplane is visiting - different countries and almost completing a loop. This indicates - that there are more than 1 trips (flights) completed by this - single airplane. The coordinates are sparse because we are - sampling the results using - TABLESAMPLE SYSTEM (5) - in our query. This is done to speed up the visualization. - -
- Single airframe geopoints vs time - - - - - - Single airframe geopoints vs - time - - - -
-
-
-
- Time-series Graphs for a Single Airplane -
- Velocity vs Time - - Following the similar steps to add a Geomap panel as before, we - include the following SQL script. Note $__timeFilter() is a - Grafana global variable. This global variable will inject time - constraint SQL-conditions from Grafana’s time range panel. - - - - - In Format as, use - Time series - - - - -SELECT - et_ts AS "time", - velocity + + + + + Change the visualization type to Geomap. + + + + + The options (visualization settings - on the right side of the screen) should be as follows: + + + Panel Options + + + + + Title → GPS location over time + + + + + Map View + + + + + Initial view: For this one zoom in on the visualization on the panel as you see fit and then click use current map settings button. + + + + + Data Layer + + + + + Layer type: → markers + + + + + Style size → Fixed Value: 2 + + + + + Color → Green + + + + + + + In this visualization we can see that the airplane is visiting different countries and almost completing a loop. This indicates that there are more than 1 trips (flights) completed by this single airplane. The coordinates are sparse because we are sampling the results using TABLESAMPLE SYSTEM (5) in our query. This is done to speed up the visualization. + +
+ Single airframe geopoints vs time + + + + + + Single airframe geopoints vs time + + + +
+
+
+
+ Time-series Graphs for a Single Airplane +
+ Velocity vs Time + + Following the similar steps to add a Geomap panel as before, we include the following SQL script. Note $__timeFilter() is a Grafana global variable. This global variable will inject time constraint SQL-conditions from Grafana's time range panel. + + + + + In Format as, use Time series + + + + +SELECT et_ts AS "time", velocity FROM flights WHERE icao24 = 'c827a6' AND $__timeFilter(et_ts) - - - - - Change the visualization type to Time Series. - - - - - The options (visualization settings - on the right side of - the screen) should be as follows: - - - Panel Options - - - - - Title → Single AirFrame - Velocity vs Time - - - - - - - In the visualization we can see clearly that on this day, this - airframe took 3 flights. That is why its speed curve has 3 - humps. The zero speed towards the end of each hump is a clear - indicator that plane stopped, thus it must have completed its - flight. - -
- Single airframe velocity vs time - - - - - - Single airframe velocity vs - time - - - -
-
-
- Altitude vs Time - - Follow the similar steps to add a Geomap panel as before, we - include the following SQL script. - - - - - In Format as, we have - Time series - - - - -SELECT - et_ts AS "time", + + + + + Change the visualization type to Time Series. + + + + + The options (visualization settings - on the right side of the screen) should be as follows: + + + Panel Options + + + + + Title → Single AirFrame - Velocity vs Time + + + + + + + In the visualization we can see clearly that on this day, this airframe took 3 flights. That is why its speed curve has 3 humps. The zero speed towards the end of each hump is a clear indicator that plane stopped, thus it must have completed its flight. + +
+ Single airframe velocity vs time + + + + + + Single airframe velocity vs time + + + +
+
+
+ Altitude vs Time + + Follow the similar steps to add a Geomap panel as before, we include the following SQL script. + + + + + In Format as, we have Time series + + + + +SELECT et_ts AS "time", baroaltitude, geoaltitude FROM flights WHERE icao24 = 'c827a6' AND $__timeFilter(et_ts) - - - - - Change the visualization type to Time Series. - - - - - The options (visualization settings - on the right side of - the screen) should be as follows: - - - Panel Options - - - - - Title → Single AirFrame - Altitude vs Time - - - - - - - In the visualization we can again see that on this day, the - airframe took 3 flights, as altitude reaches zero between each - flight. There is some noise in the data, which appear as spikes. - This would be almost impossible to spot in a tabular format, but - on a line graph these data anomalies can be easily identified. - -
- Single airframe altitude vs time - - - - - - Single airframe altitude vs - time - - - -
-
-
- Vertical-Rate vs Time - - Follow the similar steps to add a Geomap panel as before, we - include the following SQL script. - - - - - In Format as, we have - Time series - - - - -SELECT - et_ts AS "time", + + + + + Change the visualization type to Time Series. + + + + + The options (visualization settings - on the right side of the screen) should be as follows: + + + Panel Options + + + + + Title → Single AirFrame - Altitude vs Time + + + + + + + In the visualization we can again see that on this day, the airframe took 3 flights, as altitude reaches zero between each flight. There is some noise in the data, which appear as spikes. This would be almost impossible to spot in a tabular format, but on a line graph these data anomalies can be easily identified. + +
+ Single airframe altitude vs time + + + + + + Single airframe altitude vs time + + + +
+
+
+ Vertical-Rate vs Time + + Follow the similar steps to add a Geomap panel as before, we include the following SQL script. + + + + + In Format as, we have Time series + + + + +SELECT et_ts AS "time", vertrate FROM flights WHERE icao24 = 'c827a6' AND $__timeFilter(et_ts) - - - - - Change the visualization type to Time Series. - - - - - The options (visualization settings - on the right side of - the screen) should be as follows: - - - Panel Options - - - - - Title → Single AirFrame - Verticle-Rate vs Time - - - - - - - The positive values here represents the ascent of the plane. - While at cruising altitude, the plane has almost zero - vertical-rate and during decent this value becomes negative. So - a sequence of positive values, then zero values followed by - negative values would represent a single flight. - -
- Single airframe vertrate vs time - - - - - - Single airframe vertrate vs - time - - - -
-
-
- Callsign vs Time - - The callsign is a unique identifier used for a specific flight - path. For example, ANZ1220 is the callsign of the Air New - Zealand flight 1220 from Queenstown to Auckland in New Zealand. - It is possible for single airplane to make the same flight more - than once in a 24hr period if it goes back and forth. This - information will be used in later queries to partition an - airplanes data into multiple flights. - - - We can find the time at which the callsign of an airplane - changes with the following steps. - - - - - In Format as, we have - Table - - - - -SELECT - min(et_ts) AS "time", callsign + + + + + Change the visualization type to Time Series. + + + + + The options (visualization settings - on the right side of the screen) should be as follows: + + + Panel Options + + + + + Title → Single AirFrame - Verticle-Rate vs Time + + + + + + + The positive values here represents the ascent of the plane. While at cruising altitude, the plane has almost zero vertical-rate and during decent this value becomes negative. So a sequence of positive values, then zero values followed by negative values would represent a single flight. + +
+ Single airframe vertrate vs time + + + + + + Single airframe vertrate vs time + + +
+
+
+ Callsign vs Time + + The callsign is a unique identifier used for a specific flight path. For example, ANZ1220 is the callsign of the Air New Zealand flight 1220 from Queenstown to Auckland in New Zealand. It is possible for single airplane to make the same flight more than once in a 24 hour period if it goes back and forth. This information will be used in later queries to partition an airplanes data into multiple flights. + + + We can find the time at which the callsign of an airplane changes with the following steps. + + + + + In Format as, we have Table + + + + +SELECT min(et_ts) AS "time", callsign FROM flights WHERE icao24 = 'c827a6' GROUP BY callsign - - - - - Change the visualization type to Table. - - - - - The options (visualization settings - on the right side of - the screen) should be as follows: - - - Panel Options - - - - - Title → Single AirFrame - Callsign vs Time - - - - - - - In the visualization we can see that this airplane completed 3 - flights and started the 4th one towards the very end of the day. - We also see there is some NULL data in the callsign column which - is why the first timestamp doesn’t have a corresponding - callsign. - -
- Single airframe callsign vs time - - - - - - Single airframe callsign vs - time - - - -
-
-
-
-
- Part 3 - Working with Continuous Trajectories in MobilityDB - - For the following queries, we will make use of trajectories for - aggregation and creating effective splits in our data based on - parameters that change in time. - -
- Creating MobilityDB Trajectories - - This step is completed once, only on the ingestion of data. It - is shown below to provide an understanding of how to do it. With - temporal datatypes and mobilityDB functionality, we can see the - queries are very intuitive to create. - - - We first create a geometry point. This treats each latitude and - longitude as a point in space. 4326 is the SRID. - - + + + + + Change the visualization type to Table. + + + + + The options (visualization settings - on the right side of the screen) should be as follows: + + + Panel Options + + + + + Title → Single AirFrame - Callsign vs Time + + + + + + + In the visualization we can see that this airplane completed three flights and started the fourth one towards the very end of the day. We also see there is some NULL data in the callsign column which is why the first timestamp doesn't have a corresponding callsign. + +
+ Single airframe callsign vs time + + + + + + Single airframe callsign vs time + + + +
+
+
+
+
+ Part 3 - Working with Continuous Trajectories in MobilityDB + + For the following queries, we will make use of trajectories for aggregation and creating effective splits in our data based on parameters that change in time. + +
+ Creating MobilityDB Trajectories + + This step is completed once, only on the ingestion of data. It is shown below to provide an understanding of how to do it. With temporal data types and MobilityDB functionality, we can see the queries are very intuitive to create. + + + We first create a geometry point. This treats each latitude and longitude as a point in space. 4326 is the SRID. + + ALTER TABLE flights - ADD COLUMN geom geometry(Point, 4326); + ADD COLUMN geom geometry(Point, 4326); UPDATE flights SET geom = ST_SetSRID( ST_MakePoint( lon, lat ), 4326); -
- AirFrame Trajectories - - Now we are ready to construct airframe or airplane - trajectories out of their individual observations. Each - icao24 in our dataset represents a single - airplane. - - - We can create a composite index on icao24 (unique to each - plane) and et_ts (timestamps of observations) to help improve - the performance of trajectory generation. - - +
+ AirFrame Trajectories + + Now we are ready to construct airframe or airplane trajectories out of their individual observations. Each icao24 in our dataset represents a single airplane. + + + We can create a composite index on icao24 (unique to each plane) and et_ts (timestamps of observations) to help improve the performance of trajectory generation. + + CREATE INDEX icao24_time_index - ON flights (icao24, et_ts); + ON flights (icao24, et_ts); - - We create trajectories for a single airframe because: - - - - - this query serves as a simple example of how to use - mobilityDB to create trajectories - - - - - these kind of trajectories can be very important for plane - manufacturer, as they are interested in the airplane’s - analysis. - - - - - we are creating the building blocks for future queries. - Each row would represent a single flight, where flight is - identified by icao24 & callsign. - - - - + + We create trajectories for a single airframe because: + + + + + this query serves as a simple example of how to use MobilityDB to create trajectories + + + + + these kind of trajectories can be very important for plane manufacturer, as they are interested in the airplane's analysis. + + + + + we are creating the building blocks for future queries. Each row would represent a single flight, where flight is identified by icao24 and callsign. + + + + CREATE TABLE airframe_traj(icao24, trip, velocity, heading, vertrate, callsign, squawk, - geoaltitude) AS + geoaltitude) AS SELECT icao24, - tgeompoint_seq(array_agg(tgeompoint_inst(geom, et_ts) ORDER BY et_ts) - FILTER (WHERE geom IS NOT NULL)), - tfloat_seq(array_agg(tfloat_inst(velocity, et_ts) ORDER BY et_ts) - FILTER (WHERE velocity IS NOT NULL)), - tfloat_seq(array_agg(tfloat_inst(heading, et_ts) ORDER BY et_ts) - FILTER (WHERE heading IS NOT NULL)), - tfloat_seq(array_agg(tfloat_inst(vertrate, et_ts) ORDER BY et_ts) - FILTER (WHERE vertrate IS NOT NULL)), - ttext_seq(array_agg(ttext_inst(callsign, et_ts) ORDER BY et_ts) - FILTER (WHERE callsign IS NOT NULL)), - tint_seq(array_agg(tint_inst(squawk, et_ts) ORDER BY et_ts) - FILTER (WHERE squawk IS NOT NULL)), - tfloat_seq(array_agg(tfloat_inst(geoaltitude, et_ts) ORDER BY et_ts) - FILTER (WHERE geoaltitude IS NOT NULL)) + tgeompointSeq(array_agg(tgeompoint(geom, et_ts) ORDER BY et_ts) + FILTER (WHERE geom IS NOT NULL)), + tfloatSeq(array_agg(tfloat(velocity, et_ts) ORDER BY et_ts) + FILTER (WHERE velocity IS NOT NULL)), + tfloatSeq(array_agg(tfloat(heading, et_ts) ORDER BY et_ts) + FILTER (WHERE heading IS NOT NULL)), + tfloatSeq(array_agg(tfloat(vertrate, et_ts) ORDER BY et_ts) + FILTER (WHERE vertrate IS NOT NULL)), + ttextSeq(array_agg(ttext(callsign, et_ts) ORDER BY et_ts) + FILTER (WHERE callsign IS NOT NULL)), + tintSeq(array_agg(tint(squawk, et_ts) ORDER BY et_ts) + FILTER (WHERE squawk IS NOT NULL)), + tfloatSeq(array_agg(tfloat(geoaltitude, et_ts) ORDER BY et_ts) + FILTER (WHERE geoaltitude IS NOT NULL)) FROM flights GROUP BY icao24; - - - Here we create a new table for all the trajectories. We select - all the attributes of interest that change over time. We - can follow the transformation from the inner call to the outer - call: - - - - - tgeompoint_inst: combines each geometry point(lat, long) - with the timestamp where that point existed - - - - - array_agg: aggregates all the instants together into a - single array for each item in the group by. In this case, - it will create an array for each icao24 - - - - - tgeompoint_seq: constructs the array as a sequence which - can be manipulated with mobilityDB functionality. The same - approach is used for each trajectory, with the function - used changing depending on the datatype. - - - -
-
- Flight Trajectories - - Right now we have, in a single row, an airframe’s (where an - airframe is a single physical airplane) entire day’s trip - information. We would like to segment that information per - flight (an airframe flying under a specific callsign). This - query segments the airframe trajectories (in temporal columns) - based on the time period of the callsign. Below we explain the - query and the reason behind segmenting the data this way. - - + + + Here we create a new table for all the trajectories. We select all the attributes of interest that change over time. We can follow the transformation from the inner call to the outer call: + + + + + tgeompoint: combines each geometry point(lat, long) with the timestamp where that point existed + + + + + array_agg: aggregates all the instants together into a single array for each item in the GROUP BY. In this case, it will create an array for each icao24 + + + + + tgeompointSeq: constructs the array as a sequence which can be manipulated with MobilityDB functionality. The same approach is used for each trajectory, with the function used changing depending on the datatype. + + + +
+
+ Flight Trajectories + + Right now we have, in a single row, an airframe's (where an airframe is a single physical airplane) entire day's trip information. We would like to segment that information per flight (an airframe flying under a specific callsign). This query segments the airframe trajectories (in temporal columns) based on the time period of the callsign. Below we explain the query and the reason behind segmenting the data this way. + + -- Each row from airframe will create a new row in flight_traj depending on when the -- callsign changes, regardless of whether a plane repeats the same flight multiple -- times in any period -- Airplane123 (airframe_traj) |-------------------------| -- Flightpath1 (flight_traj) |-----| --- Flightpath2 (flight_traj) |--------| --- Flightpath1 (flight_traj) |-------| --- Flightpath3 (flight_traj) |--| +-- Flightpath2 (flight_traj) |--------| +-- Flightpath1 (flight_traj) |-------| +-- Flightpath3 (flight_traj) |--| - + CREATE TABLE flight_traj(icao24, callsign, flight_period, trip, velocity, heading, - vertrate, squawk, geoaltitude) -AS - -- callsign sequence unpacked into rows to split all other temporal sequences. -WITH airframe_traj_with_unpacked_callsign AS - (SELECT icao24, - trip, - velocity, - heading, - vertrate, - squawk, - geoaltitude, - startValue(unnest(segments(callsign))) AS start_value_callsign, - unnest(segments(callsign))::tstzspan AS callsign_segment_period - FROM airframe_traj) -SELECT icao24 AS icao24, - start_value_callsign AS callsign, - callsign_segment_period AS flight_period, - atTime(trip, callsign_segment_period) AS trip, - atTime(velocity, callsign_segment_period) AS velocity, - atTime(heading, callsign_segment_period) AS heading, - atTime(vertrate, callsign_segment_period) AS vertrate, - atTime(squawk, callsign_segment_period) AS squawk, - atTime(geoaltitude, callsign_segment_period) AS geoaltitude + vertrate, squawk, geoaltitude) AS + -- callsign sequence unpacked into rows to split all other temporal sequences. +WITH airframe_traj_with_unpacked_callsign AS ( + SELECT icao24, trip, velocity, heading, vertrate, squawk, geoaltitude, + startValue(unnest(segments(callsign))) AS start_value_callsign, + unnest(segments(callsign))::tstzspan AS callsignSegmentPeriod + FROM airframe_traj ) +SELECT icao24 AS icao24, start_value_callsign AS callsign, + callsignSegmentPeriod AS flight_period, + atTime(trip, callsignSegmentPeriod) AS trip, + atTime(velocity, callsignSegmentPeriod) AS velocity, + atTime(heading, callsignSegmentPeriod) AS heading, + atTime(vertrate, callsignSegmentPeriod) AS vertrate, + atTime(squawk, callsignSegmentPeriod) AS squawk, + atTime(geoaltitude, callsignSegmentPeriod) AS geoaltitude FROM airframe_traj_with_unpacked_callsign; - - - Note: - We could have tried to - create the above (table ”flight_traj”) per flight trajectories - by simply including callsign in the GROUP BY - statement in the query used to create the previous - airframe_traj table - (GROUP BY icao24, callsign;). - - - The problem with this - solution: This approach would put the trajectory data of 2 - distinct flights where that airplane and flight number are the - same in a single row, which is not correct. - - - MobilityDB functions helped us avoid the use of several - hardcoded conditions that depend on user knowledge of the data. - This approach is very generic and can be applied anytime we want - to split a trajectory by the inflection points in time of some - other trajectory. - -
-
-
- Aggregating Flight Statistics - - We can now use our trajectories to pull flight specific statistics - very easily. - -
- Average Velocity of Each Flight - - - - In Format as, we have - Table - - + + + Note: + We could have tried to create the above (table flight_traj) per flight trajectories by simply including callsign in the GROUP BY statement in the query used to create the previous airframe_traj table (GROUP BY icao24, callsign;). + + + The problem with this solution: This approach would put the trajectory data of two distinct flights where that airplane and flight number are the same in a single row, which is not correct. + + + MobilityDB functions helped us avoid the use of several hardcoded conditions that depend on user knowledge of the data. This approach is very generic and can be applied anytime we want to split a trajectory by the inflection points in time of some other trajectory. + +
+
+
+ Aggregating Flight Statistics + + We can now use our trajectories to pull flight specific statistics very easily. + +
+ Average Velocity of Each Flight + + + + In Format as, we have + Table + + -- Average flight speeds during flight SELECT callsign,twavg(velocity) AS average_velocity FROM flight_traj WHERE twavg(velocity)IS NOT NULL -- drop rows without velocity data AND twavg(velocity) < 1500 -- removes erroneous data ORDER BY twavg(velocity) desc; - - - - - Change the visualization type to Bar gauge. - - - - - The options (visualization settings - on the right side of - the screen) should be as follows - - - Panel Options - - - - - Title → Average Flight Speed - Show → All values - - - - - Bar gauge - - - - - Orientation → Horizontal - - - - - Standard Options - - - - - Unit → meters/second (m/s) - - - - - Min → 200 - - - - - The settings we adjust improve the visualization by cutting - the bar graph values of 0-200, improving the resolution at - higher ranges to see differences. - -
- Average flight speed visualization - - - - - - Average flight speed - visualization - - - -
- - -
-
- Number of Private and Commercial Flights - - We can easily combine results from multiple queries in the same - visualization in Grafana, simplifying the queries themselves. - Here we apply some domain knowledge of sport pilot aircraft - license limits for altitude and speed to provide an estimated - count of each. - - - - - In Format as, we have - Table - - + + + + + Change the visualization type to Bar gauge. + + + + + The options (visualization settings - on the right side of the screen) should be as follows + + + Panel Options + + + + + Title → Average Flight Speed Show → All values + + + + + Bar gauge + + + + + Orientation → Horizontal + + + + + Standard Options + + + + + Unit → meters/second (m/s) + + + + + Min → 200 + + + + + The settings we adjust improve the visualization by cutting the bar graph values of 0-200, improving the resolution at higher ranges to see differences. + +
+ Average flight speed visualization + + + + + + Average flight speed visualization + + + +
+ + +
+
+ Number of Private and Commercial Flights + + We can easily combine results from multiple queries in the same visualization in Grafana, simplifying the queries themselves. Here we apply some domain knowledge of sport pilot aircraft license limits for altitude and speed to provide an estimated count of each. + + + + + In Format as, we have Table + + -- Flights completed by private pilots (estimate) SELECT COUNT(callsign) AS private_flight FROM flight_traj WHERE (maxValue(velocity) IS NOT NULL -- remove flights without velocity - AND maxValue(velocity) <= 65) -- sport aircraft max is 140mph (65m/s) -AND (maxValue(geoaltitude) IS NOT NULL -- remove flights without altitude - AND maxValue(geoaltitude) <= 5500); --18,000ft (5,500m) max for private pilot + AND maxValue(velocity) <= 65) -- sport aircraft max is 140mph (65m/s) + AND (maxValue(geoaltitude) IS NOT NULL -- remove flights without altitude + AND maxValue(geoaltitude) <= 5500); --18,000ft (5,500m) max for private pilot -- Count of commercial flights (estimate) SELECT COUNT(callsign) AS commercial_flight FROM flight_traj WHERE (maxValue(velocity) IS NOT NULL - AND maxValue(velocity) > 65) -AND (maxValue(geoaltitude) IS NOT NULL - AND maxValue(geoaltitude) > 5500); - - - In Grafana, when we are in the query editor we can click on - + Query at the bottom to add multiple queries - that provide different results. - -
- Multiple queries providing results for a single - visualization - - - - - Multiple queries providing results for - a single visualization - -
- - - - Change the visualization type to Stat. - - - To label the data for each result separately, choose - Overrides at the top of the options panel on - the right. Here you can override global panel settings for - specific attributes as shown below. - -
- Override options for panel with multiple - queries - - - - - Override options for panel with - multiple queries - -
- - - - The final statistics visualization will look like this: - -
- Statistic visualization of number of flights by license - type - - - - - Statistic visualization of number of - flights by license type - -
-
-
-
Flights Taking-off in Some Interval of Time (User-Defined) - - Note: This query makes used of a sample set of data that has 200 flights to return results. flight_traj_sample is just a sampled version of flight_traj. As of the writing of this workshop, Grafana does not support display of vectors, and so individual latitude and longitude points are used as a proxy. - - - In order to make the query use Grafana global time range panel replace - the hard-coded timestamps with the ‘[${__from:date}, ${__to:date} )’. - + AND maxValue(velocity) > 65) + AND (maxValue(geoaltitude) IS NOT NULL + AND maxValue(geoaltitude) > 5500); + + + In Grafana, when we are in the query editor we can click on + Query at the bottom to add multiple queries that provide different results. + +
+ Multiple queries providing results for a single visualization + + + + + Multiple queries providing results for a single visualization + +
+ + + + Change the visualization type to Stat. + + + To label the data for each result separately, choose Overrides at the top of the options panel on the right. Here you can override global panel settings for specific attributes as shown below. + +
+ Override options for panel with multiple queries + + + + + Override options for panel with multiple queries + +
+ + + + The final statistics visualization will look like this: + +
+ Statistic visualization of number of flights by license type + + + + + Statistic visualization of number of flights by license type + +
+
+
+
Flights Taking-off in Some Interval of Time (User-Defined) + + Note: This query makes used of a sample set of data that has 200 flights to return results. flight_traj_sample is just a sampled version of flight_traj. As of the writing of this workshop, Grafana does not support display of vectors, and so individual latitude and longitude points are used as a proxy. + + + In order to make the query use Grafana global time range panel replace the hard-coded timestamps with [${__from:date}, ${__to:date}). + - -WITH --- The flight_traj_time_slice CTE is clipping all the temporal columns --- to the user specified time-range. -flight_traj_time_slice (icao24, callsign, time_slice_trip, time_slice_geoaltitude, -time_slice_vertrate) AS - (SELECT icao24, callsign, - atTime(trip, tstzspan '[2020-06-01 02:35:00, 2020-06-01 02:55:00)'), -- I changed the dates to fit my data, you should do the same!! - atTime(geoaltitude, tstzspan '[2020-06-01 02:35:00, 2020-06-01 02:55:00)'), - atTime(vertrate, tstzspan '[2020-06-01 02:35:00, 2020-06-01 02:55:00)') - FROM flight_traj_sample TABLESAMPLE SYSTEM (20)), --- There are 3 things happening in the flight_traj_time_slice_ascent CTE: --- 1. atRange: Clips the temporal data to create ranges where the vertrate --- was between '[1, 20]'. This vertrate means an aircraft was ascending. --- 2. sequenceN: Selects the first sequence from the generated sequences. --- This first sequence is takeoff and eliminates mid-flight ascents. --- 3. atPeriod: Returns the period of the first sequence. -flight_traj_time_slice_ascent(icao24, callsign, ascending_trip, ascending_geoaltitude, -ascending_vertrate) AS - (SELECT icao24, callsign, - atTime(time_slice_trip, sequenceN(atValues(time_slice_vertrate, floatspan '[1,200]'), 1)::tstzspan), - atTime(time_slice_geoaltitude, - sequenceN(atValues(time_slice_vertrate, floatspan '[1,20]'),1)::tstzspan), - atTime(time_slice_vertrate, - sequenceN(atValues(time_slice_vertrate, floatspan '[1,20]'),1)::tstzspan) - FROM flight_traj_time_slice), --- The final_output CTE uses unnest to unpack the temporal data into rows for --- visualization in Grafana. Each row will contain a latitude, longitude and the altitude --- and vertrate at those locations. -final_output AS - (SELECT icao24, callsign, - getValue(unnest(instants(ascending_geoaltitude))) AS geoaltitude, - getValue(unnest(instants(ascending_vertrate))) AS vertrate, - ST_X(getValue(unnest(instants(ascending_trip)))) AS lon, - ST_Y(getValue(unnest(instants(ascending_trip)))) AS lat - FROM flight_traj_time_slice_ascent) + +WITH timePeriod(Period) AS ( + SELECT tstzspan '[2020-06-01 02:35:00, 2020-06-01 02:55:00)' ), +flightTrajTimeSlice (icao24, callsign, time_slice_trip, timeSliceGeoaltitude, + timeSliceVertrate) AS ( + SELECT icao24, callsign, atTime(trip, Period), atTime(geoaltitude, Period), + atTime(vertrate, Period) + FROM flight_traj_sample TABLESAMPLE SYSTEM (20), timePeriod ), +flightTrajTimeSliceAscent(icao24, callsign, ascendingTrip, ascendingGeoaltitude, + ascendingVertrate) AS ( + SELECT icao24, callsign, + atTime(time_slice_trip, + sequenceN(atValues(timeSliceVertrate, floatspan '[1,200]'), 1)::tstzspan), + atTime(timeSliceGeoaltitude, + sequenceN(atValues(timeSliceVertrate, floatspan '[1,20]'),1)::tstzspan), + atTime(timeSliceVertrate, + sequenceN(atValues(timeSliceVertrate, floatspan '[1,20]'),1)::tstzspan) + FROM flightTrajTimeSlice ), +finalOutput AS ( + SELECT icao24, callsign, + getValue(unnest(instants(ascendingGeoaltitude))) AS geoaltitude, + getValue(unnest(instants(ascendingVertrate))) AS vertrate, + ST_X(getValue(unnest(instants(ascendingTrip)))) AS lon, + ST_Y(getValue(unnest(instants(ascendingTrip)))) AS lat + FROM flightTrajTimeSliceAscent ) SELECT * -FROM final_output -WHERE vertrate IS NOT NULL -AND geoaltitude IS NOT NULL; - - - Tips for QGIS visualization: - QGIS uses geometry points for visualization, so for that in the - third CTE you can use trajectory function on ascending_trip and - unnest the result. - - - We will modify make the follow adjustments for the - visualization. - - - - - Change the visualization type to Geomap. - - - - - The options (visualization settings - on the right side of - the screen) should be as follows: - - - Panel Options - - - - - Title → Flight Ascent in Time Window - - - - - Data Layer: - - - - - Layer type: Markers - - - - - Location: Coords - - - - - Latitude field: lat - - - - - Longitude field: lon - - - - - Styles - - - - - Size: geoaltitude - - - - - Min: 1 - - - - - Max: 5 - - - - - Color: vertrate - - - - - Fill opacity: 0.5 - - - - - - - Standard Options: - - - - - Unit: meters/second (m/s) - - - - - Color scheme: Green-Yellow-Red (by value) - - - - - - - We will also add a manual override (top right of panel options, beside "All") to limit the minimum value of vertrate. This will make all values below the minimum the same color, making larger values more obvious. This can be used to quickly pinpoint locations where a large rate of ascent existed. - - - Overrides - - - - - Add field override > Fields with name > vertrate - - - - - Min: 5 - - - - - Max: 20 - - - - - - +FROM finalOutput +WHERE vertrate IS NOT NULL AND geoaltitude IS NOT NULL; + + + We explain next the above query. Table timePeriod specifies a user-specified time period. Table flightTrajTimeSlice clips all the temporal columns to the given time period. Then, in the flightTrajTimeSliceAscent, atSpan clips the temporal data to create ranges where the vertrate was between '[1, 20]'. This vertrate means an aircraft was ascending. Then, sequenceN selects the first sequence from the generated sequences. This first sequence is takeoff and eliminates mid-flight ascents. Finally atTime returns the period of the first sequence. The final table finalOutput uses unnest to unpack the temporal data into rows for visualization in Grafana. Each row will contain a latitude, longitude, altitude, and vertrate at those locations. + + + Tips for QGIS visualization: QGIS uses geometry points for visualization, so for that in the third CTE you can use trajectory function on ascendingTrip and unnest the result. + + + We will modify make the follow adjustments for the visualization. + + + + + Change the visualization type to Geomap. + + + + + The options (visualization settings - on the right side of the screen) should be as follows: + + + Panel Options + + + + + Title → Flight Ascent in Time Window + + + + + Data Layer: + + + + + Layer type: Markers + + + + + Location: Coords + + + + + Latitude field: lat + + + + + Longitude field: lon + + + + + Styles + + + + + Size: geoaltitude + + + + + Min: 1 + + + + + Max: 5 + + + + + Color: vertrate + + + + + Fill opacity: 0.5 + + + + + + + Standard Options: + + + + + Unit: meters/second (m/s) + + + + + Color scheme: Green-Yellow-Red (by value) + + + + + + + We will also add a manual override (top right of panel options, beside "All") to limit the minimum value of vertrate. This will make all values below the minimum the same color, making larger values more obvious. This can be used to quickly pinpoint locations where a large rate of ascent existed. + + + Overrides + + + + + Add field override > Fields with name > vertrate + + + + + Min: 5 + + + + + Max: 20 + + + + + + - - - Here is a zoomed in version of how each individual flight ascent will look, as well as a view of multiple flights at the same time. The marker size is increasing with altitude, and the color is showing more aggressive vertical ascent rates. We can see towards the end of the visualized ascent period, there is a short increased vertical ascent rate. - -
- Zoomed in view of flight ascent - - - - - - Zoomed in view of flight - ascent - - - -
- - The final visualization will look like the below. - -
- Final visualization with multiple flight ascents - - - - - - Final visualization with multiple flight - ascents - - - -
-
-
-
Complete Flight Data Business Intelligence Dashboard - - The dashboard, with all the visualizations at the same time, will look like the screenshot below. Here - we can continue to extend the dashboards functionality by adding more dynamic variables to have the individual flight - data on the left generated with a user query or selection based on the overview take-off - information on the right. This is what really empowers decision makers and subject-matter experts (SMEs) to - quickly move through data and hone-in on important aspects that may have otherwise been over-looked. - -
- Flight data business intelligence dashboard - - - - - - Flight data business intelligence dashboard - - - -
-
+ + + Here is a zoomed in version of how each individual flight ascent will look, as well as a view of multiple flights at the same time. The marker size is increasing with altitude, and the color is showing more aggressive vertical ascent rates. We can see towards the end of the visualized ascent period, there is a short increased vertical ascent rate. + +
+ Zoomed in view of flight ascent + + + + + + Zoomed in view of flight + ascent + + + +
+ + The final visualization will look like the below. + +
+ Final visualization with multiple flight ascents + + + + + + Final visualization with multiple flight ascents + + + +
+
+ +
Complete Flight Data Business Intelligence Dashboard + + The dashboard, with all the visualizations at the same time, will look like the screenshot below. Here we can continue to extend the dashboards functionality by adding more dynamic variables to have the individual flight data on the left generated with a user query or selection based on the overview take-off information on the right. This is what really empowers decision makers and subject-matter experts (SMEs) to quickly move through data and hone-in on important aspects that may have otherwise been over-looked. + +
+ Flight data business intelligence dashboard + + + + + + Flight data business intelligence dashboard + + + +
+
diff --git a/docs/GPX.xml b/docs/GPX.xml index 0c1df6a..31455b1 100644 --- a/docs/GPX.xml +++ b/docs/GPX.xml @@ -121,7 +121,7 @@ CREATE TABLE trips_mdb ( ); INSERT INTO trips_mdb(date, trip) -SELECT date, tgeompoint_seq(array_agg(tgeompoint_inst( +SELECT date, tgeompointSeq(array_agg(tgeompoint( ST_SetSRID(ST_Point(lon, lat), 4326), time) ORDER BY time)) FROM trips_input GROUP BY date; diff --git a/docs/location_history.xml b/docs/location_history.xml index 8350559..6c6b35d 100644 --- a/docs/location_history.xml +++ b/docs/location_history.xml @@ -85,7 +85,7 @@ CREATE TABLE locations_mdb ( ); INSERT INTO locations_mdb(date, trip) -SELECT date, tgeompoint_seq(array_agg(tgeompoint_inst( +SELECT date, tgeompointSeq(array_agg(tgeompoint( ST_SetSRID(ST_Point(longitudeE7/1e7, latitudeE7/1e7),4326), to_timestamp(timestampMs / 1000.0)::timestamptz) ORDER BY timestampMs)) FROM location_history