fix: upgrade dependencies

flask upgrade required to change to a single instance of SQAlchemy, docker-compose has health-check dependency.

docker-compose.yml

@@ -17,7 +17,8 @@ services:
   ports:
    - "5000:5000"
   depends_on:
-    - db
+    db:
+      condition: service_healthy
   mem_limit: 28g
   networks:
    - poi_network
@@ -38,6 +39,11 @@ services:
    - ALLOW_IP_RANGE=0.0.0.0/0
   ports:
    - 5432:5432
+  healthcheck:
+      test: ["CMD-SHELL", "pg_isready", "-d", "${POSTGRES_DB}"]
+      interval: 10s
+      timeout: 10s
+      retries: 10
   restart: on-failure
   networks:
    - poi_network
@@ -56,7 +62,8 @@ services:
   - ./ops_settings_docker.yml:/deploy/app/openpoiservice/server/ops_settings.yml
   - ./categories_docker.yml:/deploy/app/openpoiservice/server/categories/categories.yml
   depends_on:
-    - db
+    db:
+      condition: service_healthy
   mem_limit: 28g
   networks:
    - poi_network
@@ -74,7 +81,8 @@ services:
   - ./ops_settings_docker.yml:/deploy/app/openpoiservice/server/ops_settings.yml
   - ./categories_docker.yml:/deploy/app/openpoiservice/server/categories/categories.yml
   depends_on:
-    - db
+    db:
+      condition: service_healthy
   mem_limit: 28g
   networks:
    - poi_network
@@ -92,7 +100,8 @@ services:
    - ./ops_settings_docker.yml:/deploy/app/openpoiservice/server/ops_settings.yml
    - ./categories_docker.yml:/deploy/app/openpoiservice/server/categories/categories.yml
   depends_on:
-   - db
+    db:
+      condition: service_healthy
   mem_limit: 8g
   networks:
    - poi_network

manage.py

@@ -89,7 +89,7 @@ def import_data():
 
     logger.info(f"Starting to import OSM data ({len(osm_files)} files in batch)")
     logger.debug(f"Files in import batch: {osm_files}")
-    parser.run_import(osm_files, import_log)
+    parser.run_import(osm_files, import_log, db)
 
     with open(logfile, "w") as f:
         json.dump(import_log, f, indent=4, sort_keys=True)

openpoiservice/server/db_import/parser.py

@@ -71,17 +71,13 @@ def parse_file(osm_file, osm_file_index=0, update_mode=False):
 
 
 @timeit
-def run_import(osm_files_to_import, import_log):
+def run_import(osm_files_to_import, import_log, db_con):
     try:
         update_mode = False
-        # run query on separate database connection, will conflict otherwise since parse_import runs in separate process
-        separate_db_con = SQLAlchemy()
-        prev_poi_count = separate_db_con.session.query(POIs.osm_type, POIs.osm_id).count()
+        prev_poi_count = db_con.session.query(POIs.osm_type, POIs.osm_id).count()
         if prev_poi_count > 0:
             update_mode = True
             logger.info("Data import running in UPDATE MODE")
-        separate_db_con.session.remove()
-        separate_db_con.engine.dispose()
     except sqlalchemy.exc.ProgrammingError:
         logger.error("Database has not been initialized! Existing.")
         return
@@ -92,35 +88,29 @@ def run_import(osm_files_to_import, import_log):
             continue
 
         if update_mode:
-            separate_db_con = SQLAlchemy()
-            prev_poi_count_file = separate_db_con.session.query(POIs.osm_type, POIs.osm_id).filter_by(src_index=osm_file_index).count()
+            prev_poi_count_file = db_con.session.query(POIs.osm_type, POIs.osm_id).filter_by(src_index=osm_file_index).count()
             logger.info(f"Setting flags on {prev_poi_count_file} POIs.")
-            separate_db_con.session.query(POIs).filter_by(src_index=osm_file_index).update({POIs.delflag: True})
-            separate_db_con.session.commit()
-            separate_db_con.session.remove()
-            separate_db_con.engine.dispose()
+            db_con.session.query(POIs).filter_by(src_index=osm_file_index).update({POIs.delflag: True})
+            db_con.session.commit()
 
         if parse_file(osm_file, osm_file_index, update_mode=update_mode) == 0:
             import_log[osm_file] = os.path.getmtime(osm_file)
         else:
             import_log[osm_file] = 0  # import failed, this file has to be inserted in the next update
 
     if update_mode:
-        delete_marked_entries()
+        delete_marked_entries(db_con)
 
     logger.info(f"Import complete.")
 
 
 @timeit
-def delete_marked_entries():
+def delete_marked_entries(db_con):
     logger.info(f"Updates complete, now performing delete operations...")
-    separate_db_con = SQLAlchemy()
-    to_delete = separate_db_con.session.query(POIs.osm_type, POIs.osm_id).filter_by(delflag=True).count()
+    to_delete = db_con.session.query(POIs.osm_type, POIs.osm_id).filter_by(delflag=True).count()
     if to_delete > 0:
         logger.info(f"{to_delete} POIs in the database have been removed from the OSM data, deleting...")
-        separate_db_con.session.query(POIs).filter_by(delflag=True).delete()
-        separate_db_con.session.commit()
+        db_con.session.query(POIs).filter_by(delflag=True).delete()
+        db_con.session.commit()
     else:
         logger.info(f"No POIs marked for deletion, nothing to do.")
-    separate_db_con.session.remove()
-    separate_db_con.engine.dispose()

openpoiservice/server/utils/decorators.py

@@ -100,7 +100,7 @@ def wrapper(*args, **kwargs):
 
 def timeit(method):
     def timed(*args, **kw):
-        if len(args) == 0:
+        if len(args) == 1:
             operation = "Delete operations"
         else:
             operation = "File import" if isinstance(args[0], str) else f"Batch of {len(args[0])} files"

requirements.txt

@@ -1,58 +1,58 @@
-attrs==20.3.0
-Automat==20.2.0
-blinker==1.4
-certifi==2022.12.7
-cffi==1.14.4
-click==7.1.2
-constantly==15.1.0
-coverage==5.3.1
+attrs==23.2.0
+Automat==22.10.0
+blinker==1.7.0
+certifi==2024.2.2
+cffi==1.16.0
+click==8.1.7
+constantly==23.10.4
+coverage==7.4.1
 cryptography>=3.0.0
-cssselect==1.1.0
-flasgger==0.9.5
-Flask==1.1.2
-Flask-Cors==3.0.10
-Flask-DebugToolbar==0.11.0
-Flask-SQLAlchemy==2.4.4
+cssselect==1.2.0
+flasgger==0.9.7.1
+Flask==3.0.2
+Flask-Cors==4.0.0
+Flask-DebugToolbar==0.14.1
+Flask-SQLAlchemy==3.1.1
 Flask-Testing==0.8.1
-GeoAlchemy2==0.8.4
-geojson==2.5.0
-gevent==20.12.1
-greenlet==0.4.17
-gunicorn==20.0.4
-hyperlink==20.0.1
-idna==3.1
-incremental==17.5.0
+GeoAlchemy2==0.14.4
+geojson==3.1.0
+gevent>=20.12.1
+greenlet==3.0.3
+gunicorn==21.2.0
+hyperlink==21.0.0
+idna==3.6
+incremental==22.10.0
 itemadapter>=0.1.0
 itemloaders>=1.0.0
-itsdangerous==1.1.0
-Jinja2==2.11.3
-jmespath==0.10.0
-jsonschema==3.2.0
-lxml==4.9.1
-MarkupSafe==1.1.1
-mistune==2.0.3
-parsel==1.6.0
-Protego==0.1.16
-psycopg2-binary==2.8.6
-pyasn1==0.4.8
-pyasn1-modules==0.2.8
-pycparser==2.20
-PyDispatcher==2.0.5
-PyHamcrest==2.0.2
-pyOpenSSL==20.0.1
-pyproj==3.1.0
-pyrsistent==0.17.3
+itsdangerous==2.1.2
+Jinja2==3.1.3
+jmespath==1.0.1
+jsonschema==4.21.1
+lxml==5.1.0
+MarkupSafe==2.1.5
+mistune==3.0.2
+parsel==1.8.1
+Protego==0.3.0
+psycopg2-binary==2.9.9
+pyasn1==0.5.1
+pyasn1-modules==0.3.0
+pycparser==2.21
+PyDispatcher==2.0.7
+PyHamcrest==2.1.0
+pyOpenSSL==24.0.0
+pyproj==3.6.1
+pyrsistent==0.20.0
 PyYAML
-queuelib==1.5.0
+queuelib==1.6.2
 Scrapy>=2.3.0
-service-identity==18.1.0
-Shapely==1.7.1
-six==1.15.0
-SQLAlchemy==1.3.22
-Twisted==21.2.0
-voluptuous==0.12.1
-w3lib==1.22.0
-Werkzeug==1.0.1
-zope.event==4.5.0
-zope.interface==5.2.0
+service-identity==24.1.0
+Shapely==2.0.3
+six==1.16.0
+SQLAlchemy==2.0.27
+Twisted==23.10.0
+voluptuous==0.14.2
+w3lib==2.1.2
+Werkzeug==3.0.1
+zope.event==5.0
+zope.interface==6.2
 git+https://github.com/timmccauley/imposm-parser.git@python3