From 92380127e44cffb1b438221e1e57197cef979ab1 Mon Sep 17 00:00:00 2001 From: Inga Ulusoy Date: Thu, 18 Jul 2024 09:18:04 +0200 Subject: [PATCH] update db syntax to sqlalchemy 2.x (#2) * add relational one-to-many, use sqlalchemy 2 syntax * create correct db * temp changes * can submit text * with metadata * with second model * relational one-to-many working * add some basic tests * run CI * fix typo * add missing dep; include key in env * fix space in key and pass to env --- .github/workflows/CI.yml | 39 +++++++++++++++++++++++++++++++++++ requirements.txt | 5 ++++- src/tests/conftest.py | 23 +++++++++++++++++++++ src/tests/test_donate.py | 11 ++++++++++ src/website/__init__.py | 16 +++++++++++---- src/website/models.py | 44 ++++++++++++++++++++++++---------------- 6 files changed, 115 insertions(+), 23 deletions(-) create mode 100644 .github/workflows/CI.yml create mode 100644 src/tests/conftest.py create mode 100644 src/tests/test_donate.py diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml new file mode 100644 index 0000000..2557c2c --- /dev/null +++ b/.github/workflows/CI.yml @@ -0,0 +1,39 @@ +# workflow for testing +name: CI +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + workflow_dispatch: + +jobs: + test-webserver: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: [3.11] + os: [ubuntu-latest] + # os: [ubuntu-latest, macos-latest] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + - name: run base tests + env: + FLASK_SECRET_KEY: ${{ secrets.FLASK_SECRET_KEY }} + run: | + cd src + python -m pytest -svv --cov=. --cov-report=xml + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v4 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index fd02b61..4594725 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,5 @@ flask -flask-sqlalchemy \ No newline at end of file +flask-sqlalchemy +sqlalchemy +pytest +pytest-cov \ No newline at end of file diff --git a/src/tests/conftest.py b/src/tests/conftest.py new file mode 100644 index 0000000..2f49263 --- /dev/null +++ b/src/tests/conftest.py @@ -0,0 +1,23 @@ +import pytest +from website import create_app + + +@pytest.fixture() +def app(): + app = create_app() + app.config.update( + { + "TESTING": True, + } + ) + yield app + + +@pytest.fixture() +def client(app): + return app.test_client() + + +@pytest.fixture() +def runner(app): + return app.test_cli_runner() diff --git a/src/tests/test_donate.py b/src/tests/test_donate.py new file mode 100644 index 0000000..6ab297f --- /dev/null +++ b/src/tests/test_donate.py @@ -0,0 +1,11 @@ +def test_donate(client): + response = client.get("/donation") + assert response.status_code == 200 + assert b"Please provide text input" not in response.data + response = client.post("/donation", data={"text": "some text"}) + # test the redirect after submission + assert response.status_code == 302 + response = client.post("/donation", data={"text": ""}) + # assert the error response + assert response.status_code == 200 + assert b"Please provide text input" in response.data diff --git a/src/website/__init__.py b/src/website/__init__.py index e7a10af..db37539 100644 --- a/src/website/__init__.py +++ b/src/website/__init__.py @@ -1,14 +1,22 @@ from flask import Flask from flask_sqlalchemy import SQLAlchemy +from sqlalchemy.orm import DeclarativeBase from os import path -db = SQLAlchemy() -DB_NAME = "email-donations.db" +DB_NAME = "email_donations.db" + + +class Base(DeclarativeBase): + pass + + +db = SQLAlchemy(model_class=Base) def create_app(): app = Flask(__name__) - app.config["SECRET_KEY"] = "wrgeerngh npitgn rion" + app.config.from_prefixed_env() + # reads the key from FLASK_SECRET_KEY env var app.config["SQLALCHEMY_DATABASE_URI"] = f"sqlite:///{DB_NAME}" db.init_app(app) @@ -20,7 +28,7 @@ def create_app(): app.register_blueprint(donate, url_prefix="/") app.register_blueprint(about, url_prefix="/") - from .models import RawData, ProcessedData # noqa + from .models import RawData with app.app_context(): db.create_all() diff --git a/src/website/models.py b/src/website/models.py index e5a69c6..6af6f8b 100644 --- a/src/website/models.py +++ b/src/website/models.py @@ -1,49 +1,57 @@ -from . import db +from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy import Integer, String, DateTime, ForeignKey # import hashlib from sqlalchemy.sql import func +import datetime +from typing import List +from . import db # the raw data model class RawData(db.Model): # the submission id - id = db.Column(db.Integer, primary_key=True) + donor_id: Mapped[int] = mapped_column(primary_key=True) # should this be the donated data as zip? - donation = db.Column(db.String(10000), nullable=False) + donation: Mapped[str] = mapped_column(String, nullable=True) # the hash checksum of the donation zip file, for example SHA-256 # could also be SHA-3 # Compute SHA-256 hash # sha256_hash = hashlib.sha256(data).hexdigest() - checksum = db.Column(db.String(100), nullable=False) + checksum: Mapped[str] = mapped_column(String, nullable=True) # Now the metadata # the date of the donation - date = db.Column(db.DateTime(timezone=True), default=func.now(), nullable=False) + date: Mapped[datetime.datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) # the email of the donor - email = db.Column(db.String(100), nullable=True) + email: Mapped[str] = mapped_column(String, nullable=True) # the age group of the donor in categories - age = db.Column(db.Integer, nullable=True) + age: Mapped[int] = mapped_column(Integer, nullable=True) # the region of the donor in categories - region = db.Column(db.String(100), nullable=True) + region: Mapped[int] = mapped_column(String, nullable=True) # the gender of the donor in categories - gender = db.Column(db.Integer, nullable=True) + gender: Mapped[int] = mapped_column(Integer, nullable=True) # if the emails are in the mother tongue of the donor - mother_tongue = db.Column(db.Integer, nullable=True) + mother_tongue: Mapped[int] = mapped_column(Integer, nullable=True) # the type of emails: formal, informal, etc. as categories - email_type = db.Column(db.Integer, nullable=True) + email_type: Mapped[int] = mapped_column(Integer, nullable=True) # set up the relationship with the processed data - # emails = db.relationship("ProcessedData") + children: Mapped[List["ProcessedData"]] = relationship() class ProcessedData(db.Model): # the submission id - id = db.Column(db.Integer, primary_key=True) + id: Mapped[int] = mapped_column(Integer, primary_key=True) # the raw email text - raw_email = db.Column(db.String(100000), nullable=False) + raw_email: Mapped[str] = mapped_column(String, nullable=False) # the processed pseudonymized email text - processed_email = db.Column(db.String(100000), nullable=False) + processed_email: Mapped[str] = mapped_column(String, nullable=False) # the date of the processing - date = db.Column(db.DateTime(timezone=True), default=func.now(), nullable=False) + date: Mapped[datetime.datetime] = mapped_column( + DateTime(timezone=True), default=func.now(), nullable=False + ) # the language of the email - language = db.Column(db.String(10), nullable=False) + language: Mapped[str] = mapped_column(String, nullable=False) # the original donation id, one to many relationship - # donation_id = db.Column(db.Integer, db.ForeignKey("rawdata.id"), nullable=False) + donation_id: Mapped[int] = mapped_column(ForeignKey("raw_data.donor_id"))