update db syntax to sqlalchemy 2.x (#2)

* add relational one-to-many, use sqlalchemy 2 syntax

* create correct db

* temp changes

* can submit text

* with metadata

* with second model

* relational one-to-many working

* add some basic tests

* run CI

* fix typo

* add missing dep; include key in env

* fix space in key and pass to env

.github/workflows/CI.yml

@@ -0,0 +1,39 @@
+# workflow for testing
+name: CI
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  workflow_dispatch:
+
+jobs:
+  test-webserver:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.11]
+        os: [ubuntu-latest]
+        # os: [ubuntu-latest, macos-latest]
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install -r requirements.txt
+    - name: run base tests
+      env:
+        FLASK_SECRET_KEY: ${{ secrets.FLASK_SECRET_KEY }}
+      run: |
+        cd src
+        python -m pytest -svv --cov=. --cov-report=xml
+    - name: Upload coverage reports to Codecov
+      uses: codecov/codecov-action@v4
+      env:
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

requirements.txt

@@ -1,2 +1,5 @@
 flask
-flask-sqlalchemy
+flask-sqlalchemy
+sqlalchemy
+pytest
+pytest-cov

src/tests/conftest.py

@@ -0,0 +1,23 @@
+import pytest
+from website import create_app
+
+
+@pytest.fixture()
+def app():
+    app = create_app()
+    app.config.update(
+        {
+            "TESTING": True,
+        }
+    )
+    yield app
+
+
+@pytest.fixture()
+def client(app):
+    return app.test_client()
+
+
+@pytest.fixture()
+def runner(app):
+    return app.test_cli_runner()

src/tests/test_donate.py

@@ -0,0 +1,11 @@
+def test_donate(client):
+    response = client.get("/donation")
+    assert response.status_code == 200
+    assert b"Please provide text input" not in response.data
+    response = client.post("/donation", data={"text": "some text"})
+    # test the redirect after submission
+    assert response.status_code == 302
+    response = client.post("/donation", data={"text": ""})
+    # assert the error response
+    assert response.status_code == 200
+    assert b"Please provide text input" in response.data

src/website/__init__.py

@@ -1,14 +1,22 @@
 from flask import Flask
 from flask_sqlalchemy import SQLAlchemy
+from sqlalchemy.orm import DeclarativeBase
 from os import path
 
-db = SQLAlchemy()
-DB_NAME = "email-donations.db"
+DB_NAME = "email_donations.db"
+
+
+class Base(DeclarativeBase):
+    pass
+
+
+db = SQLAlchemy(model_class=Base)
 
 
 def create_app():
     app = Flask(__name__)
-    app.config["SECRET_KEY"] = "wrgeerngh npitgn rion"
+    app.config.from_prefixed_env()
+    # reads the key from FLASK_SECRET_KEY env var
     app.config["SQLALCHEMY_DATABASE_URI"] = f"sqlite:///{DB_NAME}"
     db.init_app(app)
 
@@ -20,7 +28,7 @@ def create_app():
     app.register_blueprint(donate, url_prefix="/")
     app.register_blueprint(about, url_prefix="/")
 
-    from .models import RawData, ProcessedData  # noqa
+    from .models import RawData
 
     with app.app_context():
         db.create_all()

src/website/models.py

@@ -1,49 +1,57 @@
-from . import db
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+from sqlalchemy import Integer, String, DateTime, ForeignKey
 
 # import hashlib
 from sqlalchemy.sql import func
+import datetime
+from typing import List
+from . import db
 
 
 # the raw data model
 class RawData(db.Model):
     # the submission id
-    id = db.Column(db.Integer, primary_key=True)
+    donor_id: Mapped[int] = mapped_column(primary_key=True)
     # should this be the donated data as zip?
-    donation = db.Column(db.String(10000), nullable=False)
+    donation: Mapped[str] = mapped_column(String, nullable=True)
     # the hash checksum of the donation zip file, for example SHA-256
     # could also be SHA-3
     # Compute SHA-256 hash
     # sha256_hash = hashlib.sha256(data).hexdigest()
-    checksum = db.Column(db.String(100), nullable=False)
+    checksum: Mapped[str] = mapped_column(String, nullable=True)
     # Now the metadata
     # the date of the donation
-    date = db.Column(db.DateTime(timezone=True), default=func.now(), nullable=False)
+    date: Mapped[datetime.datetime] = mapped_column(
+        DateTime(timezone=True), server_default=func.now(), nullable=False
+    )
     # the email of the donor
-    email = db.Column(db.String(100), nullable=True)
+    email: Mapped[str] = mapped_column(String, nullable=True)
     # the age group of the donor in categories
-    age = db.Column(db.Integer, nullable=True)
+    age: Mapped[int] = mapped_column(Integer, nullable=True)
     # the region of the donor in categories
-    region = db.Column(db.String(100), nullable=True)
+    region: Mapped[int] = mapped_column(String, nullable=True)
     # the gender of the donor in categories
-    gender = db.Column(db.Integer, nullable=True)
+    gender: Mapped[int] = mapped_column(Integer, nullable=True)
     # if the emails are in the mother tongue of the donor
-    mother_tongue = db.Column(db.Integer, nullable=True)
+    mother_tongue: Mapped[int] = mapped_column(Integer, nullable=True)
     # the type of emails: formal, informal, etc. as categories
-    email_type = db.Column(db.Integer, nullable=True)
+    email_type: Mapped[int] = mapped_column(Integer, nullable=True)
     # set up the relationship with the processed data
-    # emails = db.relationship("ProcessedData")
+    children: Mapped[List["ProcessedData"]] = relationship()
 
 
 class ProcessedData(db.Model):
     # the submission id
-    id = db.Column(db.Integer, primary_key=True)
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
     # the raw email text
-    raw_email = db.Column(db.String(100000), nullable=False)
+    raw_email: Mapped[str] = mapped_column(String, nullable=False)
     # the processed pseudonymized email text
-    processed_email = db.Column(db.String(100000), nullable=False)
+    processed_email: Mapped[str] = mapped_column(String, nullable=False)
     # the date of the processing
-    date = db.Column(db.DateTime(timezone=True), default=func.now(), nullable=False)
+    date: Mapped[datetime.datetime] = mapped_column(
+        DateTime(timezone=True), default=func.now(), nullable=False
+    )
     # the language of the email
-    language = db.Column(db.String(10), nullable=False)
+    language: Mapped[str] = mapped_column(String, nullable=False)
     # the original donation id, one to many relationship
-    # donation_id = db.Column(db.Integer, db.ForeignKey("rawdata.id"), nullable=False)
+    donation_id: Mapped[int] = mapped_column(ForeignKey("raw_data.donor_id"))