DB importing image, replication, CI

hiddewie · Mar 30, 2024 · 312e30c · 312e30c
1 parent 2eb28f6
commit 312e30c
Show file tree

Hide file tree

Showing 3 changed files with 52 additions and 24 deletions.
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -47,30 +47,46 @@ jobs:
         run: |
           echo "key=$(date +%Y-%W)" >> $GITHUB_OUTPUT
 
-      - name: Cache downloaded OpenStreetMap data
-        id: cache-data
-        uses: actions/cache@v4
-        with:
-          key: 'data-europe-${{ steps.cache-key.outputs.key }}'
-          path: data/filtered/europe.osm.pbf
-
-      - name: Download data
-        if: ${{ steps.cache-data.outputs.cache-hit != 'true' }}
+#      - name: Cache downloaded OpenStreetMap data
+#        id: cache-data
+#        uses: actions/cache@v4
+#        with:
+#          key: 'data-europe-${{ steps.cache-key.outputs.key }}'
+#          path: data/filtered/europe.osm.pbf
+#
+#      - name: Download data
+#        if: ${{ steps.cache-data.outputs.cache-hit != 'true' }}
+#        run: |
+#          curl --progress-bar http://ftp.snt.utwente.nl/pub/misc/openstreetmap/europe-latest.osm.pbf --output data/europe.osm.pbf
+
+      - name: Pull import database
         run: |
-          curl --progress-bar http://ftp.snt.utwente.nl/pub/misc/openstreetmap/europe-latest.osm.pbf --output data/europe.osm.pbf
+          docker compose up --no-build --pull always --detach db
+
 
-      - name: Import data
+#      - name: Import data
+      - name: Ensure data is up to date
         env:
           # Available cores: 4
           OSM2PGSQL_NUMPROC: 4
           # Available memory: 16GB
           OSM2PGSQL_CACHE: 14000
-          OSM2PGSQL_DATAFILE: europe.osm.pbf
+#          OSM2PGSQL_DATAFILE: europe.osm.pbf
         run: |
           docker compose run --build import
 
+      - name: Commit and push new database image
+        run: |
+          docker compose down
+          DB_CONTAINER_ID="$(docker compose ps --all --format json | jq -r 'select(.Service == "db") | .ID')"
+          DB_IMAGE="$(docker compose ps --all --format json | jq -r 'select(.Service == "db") | .Image')"
+          docker compose commit "$DB_CONTAINER_ID" "$DB_IMAGE"
+          docker image ls "$DB_IMAGE"
+          docker push "$DB_IMAGE"
+
       - name: Generate tiles
         run: |
+          docker compose up --no-build --pull never --detach db
           docker compose up --build martin-cp
 
       - name: Deploy

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,6 +1,6 @@
 services:
   db:
-    image: db:v1
+    image: ghcr.io/hiddewie/openrailwaymap-import-db:latest
     build:
       context: .
       dockerfile: db.Dockerfile
@@ -9,8 +9,8 @@ services:
       -c shared_preload_libraries='pg_stat_statements'
     ports:
       - '5432:5432'
-    volumes:
-      - ./postgres-data:/var/lib/postgresql/postgres-data
+#    volumes:
+#      - ./postgres-data:/var/lib/postgresql/postgres-data
     shm_size: 1g
     environment:
       - POSTGRES_HOST_AUTH_METHOD=trust
@@ -19,7 +19,6 @@ services:
       - PGDATA=/var/lib/postgresql/postgres-data
 
   import:
-    image: import:v1
     build:
       context: .
       dockerfile: import.Dockerfile

diff --git a/import/docker-startup.sh b/import/docker-startup.sh
@@ -44,25 +44,38 @@ echo "Filtering data from $OSM2PGSQL_INPUT_FILE to $OSM2PGSQL_FILTERED_FILE"
     r/route=light_rail \
     r/route=subway
 
-echo "Importing data"
-# Importing data to a database
-osm2pgsql \
-  --create \
-  --database gis \
-  --drop \
+#echo "Importing data"
+## Importing data to a database
+#osm2pgsql \
+#  --create \
+#  --database gis \
+#  --slim \
+#  --middle-database-format new \
+#  --output flex \
+#  --style openrailwaymap.lua \
+#  --cache "${OSM2PGSQL_CACHE:-256}" \
+#  --number-processes "${OSM2PGSQL_NUMPROC:-4}" \
+#  "$OSM2PGSQL_FILTERED_FILE"
+#
+#osm2pgsql-replication init --database gis
+
+echo "Updating data"
+osm2pgsql-replication update --database gis -- \
   --slim \
   --output flex \
   --style openrailwaymap.lua \
   --cache "${OSM2PGSQL_CACHE:-256}" \
-  --number-processes "${OSM2PGSQL_NUMPROC:-4}" \
-  "$OSM2PGSQL_FILTERED_FILE"
+  --number-processes "${OSM2PGSQL_NUMPROC:-4}"
 
 echo "Post processing imported data"
 psql -d gis -f sql/functions.sql
 psql -d gis -f sql/get_station_importance.sql
 psql -d gis -f sql/signals_with_azimuth.sql
 psql -d gis -f sql/tile_views.sql
 
+echo "Vacuuming database"
+psql -d gis -c "VACUUM FULL;"
+
 echo "Import summary"
 psql -d gis -c "select table_name as table, pg_size_pretty(pg_total_relation_size(quote_ident(table_name))) as size from information_schema.tables where table_schema = 'public' order by table_name;"
 psql -d gis -c "select pg_size_pretty(sum(pg_total_relation_size(quote_ident(table_name)))) as total_size from information_schema.tables where table_schema = 'public';"