update python and Postgres

.github/workflows/python-tests.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     services:
       postgres:
-        image: postgres:16-alpine
+        image: postgres:17-alpine
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
@@ -23,10 +23,10 @@ jobs:
           - 5432:5432
     steps:
     - uses: actions/checkout@v4
-    - name: Python 3.12 Setup
+    - name: Python 3.13 Setup
       uses: actions/setup-python@v5
       with:
-        python-version: 3.12
+        python-version: 3.13
     - name: Install dependencies
       run: |
         sudo apt-get update

README.md

@@ -61,7 +61,7 @@ You'll need to replace `police-data-trust-api-1` with the name of the container
 docker container ls
 CONTAINER ID   IMAGE                   COMMAND                  CREATED              STATUS              PORTS                    NAMES
 c0cf********   police-data-trust-api   "/bin/sh -c '/wait &…"   About a minute ago   Up About a minute   0.0.0.0:5001->5001/tcp   police-data-trust-api-1
-5e6f********   postgres:16             "docker-entrypoint.s…"   3 days ago           Up About a minute   0.0.0.0:5432->5432/tcp   police-data-trust-db-1
+5e6f********   postgres:17             "docker-entrypoint.s…"   3 days ago           Up About a minute   0.0.0.0:5432->5432/tcp   police-data-trust-db-1
 dacd********   police-data-trust-web   "docker-entrypoint.s…"   3 days ago           Up About a minute   0.0.0.0:3000->3000/tcp   police-data-trust-web-1
 ```
 

backend/database/core.py

@@ -10,12 +10,10 @@
 import click
 import pandas as pd
 import psycopg
-import psycopg2.errors
 from flask import abort, current_app
 from flask.cli import AppGroup, with_appcontext
 from flask_sqlalchemy import SQLAlchemy
-from psycopg2 import connect
-from psycopg2.extensions import connection
+from psycopg import connect, connection
 from sqlalchemy.exc import ResourceClosedError
 from werkzeug.utils import secure_filename
 
@@ -123,7 +121,7 @@ def create_database(
 
     try:
         cursor.execute(f"CREATE DATABASE {database};")
-    except (psycopg2.errors.lookup("42P04"), psycopg.errors.DuplicateDatabase):
+    except psycopg.errors.DuplicateDatabase:
         click.echo(f"Database {database!r} already exists.")
     else:
         click.echo(f"Created database {database!r}.")
@@ -184,7 +182,7 @@ def delete_database(conn: connection, test_db: bool):
 
     try:
         cursor.execute(f"DROP DATABASE {database};")
-    except psycopg2.errors.lookup("3D000"):
+    except psycopg.errors.InvalidCatalogName:
         click.echo(f"Database {database!r} does not exist.")
     else:
         click.echo(f"Database {database!r} was deleted.")

backend/scraper/notebooks/cpdp.ipynb

@@ -72,7 +72,7 @@
     "import numpy as np\n",
     "import pandas as pd\n",
     "import sqlalchemy\n",
-    "import psycopg2\n",
+    "import psycopg\n",
     "from IPython.display import display, HTML\n",
     "from collections import namedtuple\n",
     "from backend.database import db, Incident, Officer, Accusation, Victim\n",
@@ -350,7 +350,7 @@
     "    try:\n",
     "        insert_bulk(dicts, OrmClass)\n",
     "    except sqlalchemy.exc.IntegrityError as e:\n",
-    "        if isinstance(e.orig, psycopg2.errors.UniqueViolation):\n",
+    "        if isinstance(e.orig, psycopg.errors.UniqueViolation):\n",
     "            print(f\"Already created {OrmClass.__name__} records\")\n",
     "        else:\n",
     "            raise e\n",

backend/scraper/notebooks/mpv.ipynb

@@ -89,7 +89,6 @@
     "import numpy as np\n",
     "import pandas as pd\n",
     "from flask_sqlalchemy import SQLAlchemy\n",
-    "import psycopg2\n",
     "from itertools import zip_longest\n",
     "from typing import List\n",
     "import requests\n",

backend/tests/conftest.py

@@ -1,5 +1,4 @@
 import psycopg.errors
-import psycopg2.errors
 import pytest
 from backend.api import create_app
 from backend.auth import user_manager
@@ -34,13 +33,13 @@ def database():
         host=cfg.POSTGRES_HOST,
         port=cfg.PGPORT,
         dbname=cfg.POSTGRES_DB,
-        version=16.3,
+        version=16.4,
         password=cfg.POSTGRES_PASSWORD,
     )
 
     try:
         janitor.init()
-    except (psycopg2.errors.lookup("42P04"), psycopg.errors.DuplicateDatabase):
+    except psycopg.errors.DuplicateDatabase:
         pass
 
     yield

docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   db:
-    image: postgres:16 #AWS RDS latest version
+    image: postgres:17 #AWS RDS latest version
     env_file:
       - ".env"
     volumes:

docs/src/setup.md

@@ -12,7 +12,7 @@ Install all of the following programs onto your computer:
 
 **Required:**
 
-- [Python 3.12](https://www.python.org/downloads/)
+- [Python 3.13](https://www.python.org/downloads/)
 - [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) (first time setup guide [here](https://git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup))
 - [Postgres](https://www.postgresql.org/) _(see installation instructions below)_
 - _(OSX only)_ [Homebrew](https://brew.sh/)

requirements/Dockerfile

@@ -1,8 +1,8 @@
-# psycopg2 requires postgres development files in order to compile the
+# psycopg requires postgres development files in order to compile the
 # requirements, so this image starts with the same image as the database
 # containers and installs the same version of python as the api containers
 
-FROM postgres:16 as base
+FROM postgres:17 as base
 
 RUN apt-get update && apt-get install -y \
     make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \
@@ -15,7 +15,7 @@ SHELL ["bash", "-lc"]
 RUN curl https://pyenv.run | bash && \
     echo 'export PATH="$HOME/.pyenv/shims:$HOME/.pyenv/bin:$PATH"' >> ~/.bashrc
 
-ENV PYTHON_VERSION=3.12.5
+ENV PYTHON_VERSION=3.13.0
 RUN pyenv install ${PYTHON_VERSION} && pyenv global ${PYTHON_VERSION}
 RUN pip install -U pip-tools
 

requirements/_core.in

@@ -1,12 +1,12 @@
-bcrypt==3.2.2
+bcrypt
 black
 boto3
 celery
 flake8
 flask
 flask-sqlalchemy
 gunicorn
-pandas==2.2.2
+pandas==2.2.3
 pip-tools
 pydantic
 pydantic-sqlalchemy
@@ -19,7 +19,7 @@ pytest-mock
 python-dotenv
 flask-user<0.7
 wtforms
-SQLAlchemy==1.4.51
+SQLAlchemy==1.4.54
 flask-wtf
 flask-babelex
 PyYAML

requirements/dev_unix.in

@@ -1,3 +1,2 @@
 -r _core.in
-psycopg2-binary
 psycopg-binary

requirements/dev_unix.txt

@@ -32,7 +32,7 @@ babel==2.14.0
     # via
     #   flask-babelex
     #   jupyterlab-server
-bcrypt==3.2.2
+bcrypt==4.2.0
     # via
     #   -r requirements/_core.in
     #   flask-user
@@ -318,7 +318,7 @@ packaging==24.0
     #   pytest-flask-sqlalchemy
     #   qtconsole
     #   qtpy
-pandas==2.2.2
+pandas==2.2.3
     # via -r requirements/_core.in
 pandocfilters==1.5.1
     # via nbconvert
@@ -353,11 +353,9 @@ psutil==5.9.8
     # via
     #   ipykernel
     #   mirakuru
-psycopg==3.1.18
+psycopg==3.2.3
     # via pytest-postgresql
-psycopg-binary==3.1.18
-    # via -r requirements/dev_unix.in
-psycopg2-binary==2.9.9
+psycopg-binary==3.2.3
     # via -r requirements/dev_unix.in
 ptyprocess==0.7.0
     # via
@@ -484,7 +482,7 @@ speaklater==1.3
     # via flask-babelex
 spectree==1.2.9
     # via -r requirements/_core.in
-sqlalchemy==1.4.51
+sqlalchemy==1.4.54
     # via
     #   -r requirements/_core.in
     #   alembic

requirements/dev_windows.in

@@ -1,3 +1,2 @@
 -r _core.in
-psycopg2
 psycopg

requirements/dev_windows.txt

@@ -32,7 +32,7 @@ babel==2.14.0
     # via
     #   flask-babelex
     #   jupyterlab-server
-bcrypt==3.2.2
+bcrypt==4.2.0
     # via
     #   -r requirements/_core.in
     #   flask-user
@@ -318,7 +318,7 @@ packaging==24.0
     #   pytest-flask-sqlalchemy
     #   qtconsole
     #   qtpy
-pandas==2.2.2
+pandas==2.2.3
     # via -r requirements/_core.in
 pandocfilters==1.5.1
     # via nbconvert
@@ -353,12 +353,10 @@ psutil==5.9.8
     # via
     #   ipykernel
     #   mirakuru
-psycopg==3.1.18
+psycopg==3.2.3
     # via
     #   -r requirements/dev_windows.in
     #   pytest-postgresql
-psycopg2==2.9.9
-    # via -r requirements/dev_windows.in
 ptyprocess==0.7.0
     # via
     #   pexpect
@@ -484,7 +482,7 @@ speaklater==1.3
     # via flask-babelex
 spectree==1.2.9
     # via -r requirements/_core.in
-sqlalchemy==1.4.51
+sqlalchemy==1.4.54
     # via
     #   -r requirements/_core.in
     #   alembic

requirements/prod.in

@@ -1,3 +1,2 @@
 -r _core.in
-psycopg2-binary==2.9.9
 psycopg-binary==3.1.18

requirements/prod.txt

@@ -32,7 +32,7 @@ babel==2.14.0
     # via
     #   flask-babelex
     #   jupyterlab-server
-bcrypt==3.2.2
+bcrypt==4.2.0
     # via
     #   -r requirements/_core.in
     #   flask-user
@@ -318,7 +318,7 @@ packaging==24.0
     #   pytest-flask-sqlalchemy
     #   qtconsole
     #   qtpy
-pandas==2.2.2
+pandas==2.2.3
     # via -r requirements/_core.in
 pandocfilters==1.5.1
     # via nbconvert
@@ -353,11 +353,9 @@ psutil==5.9.8
     # via
     #   ipykernel
     #   mirakuru
-psycopg==3.1.18
+psycopg==3.2.3
     # via pytest-postgresql
-psycopg-binary==3.1.18
-    # via -r requirements/prod.in
-psycopg2-binary==2.9.9
+psycopg-binary==3.2.3
     # via -r requirements/prod.in
 ptyprocess==0.7.0
     # via
@@ -484,7 +482,7 @@ speaklater==1.3
     # via flask-babelex
 spectree==1.2.9
     # via -r requirements/_core.in
-sqlalchemy==1.4.51
+sqlalchemy==1.4.54
     # via
     #   -r requirements/_core.in
     #   alembic

runtime.txt

@@ -1 +1 @@
-python-3.12.5
+python-3.13.0