diff --git a/.env.dist b/.env.dist index b2ca4bf..72a6efe 100644 --- a/.env.dist +++ b/.env.dist @@ -11,6 +11,9 @@ MORK_WARNING_PERIOD=P3Y30D MORK_DELETION_PERIOD=P3Y MORK_DELETE_MAX_RETRIES=3 +# Edx forum configuration +MORK_EDX_FORUM_PLACEHOLDER_USER_ID=1234 + # Mork database MORK_DB_ENGINE=postgresql+psycopg2 MORK_DB_HOST=postgresql @@ -21,15 +24,24 @@ MORK_DB_PORT=5432 MORK_DB_DEBUG=False MORK_TEST_DB_NAME=test-mork-db -# Edx database -MORK_EDX_DB_ENGINE=mysql+pymysql -MORK_EDX_DB_HOST=mysql -MORK_EDX_DB_NAME=edxapp -MORK_EDX_DB_USER=edxapp -MORK_EDX_DB_PASSWORD=password -MORK_EDX_DB_PORT=3306 -MORK_EDX_DB_DEBUG=False -MORK_EDX_QUERY_BATCH_SIZE=1000 +# Edx MySQL database +MORK_EDX_MYSQL_DB_ENGINE=mysql+pymysql +MORK_EDX_MYSQL_DB_HOST=mysql +MORK_EDX_MYSQL_DB_NAME=edxapp +MORK_EDX_MYSQL_DB_USER=edxapp +MORK_EDX_MYSQL_DB_PASSWORD=password +MORK_EDX_MYSQL_DB_PORT=3306 +MORK_EDX_MYSQL_DB_DEBUG=False +MORK_EDX_MYSQL_QUERY_BATCH_SIZE=1000 + +# Edx MongoDB database +MORK_EDX_MONGO_DB_ENGINE=mongodb +MORK_EDX_MONGO_DB_HOST=mongo +MORK_EDX_MONGO_DB_NAME=cs_comments_service +MORK_EDX_MONGO_DB_USER= +MORK_EDX_MONGO_DB_PASSWORD= +MORK_EDX_MONGO_DB_PORT=27017 +MORK_EDX_MONGO_DB_DEBUG=False # Redis configuration REDIS_HOST=localhost diff --git a/Makefile b/Makefile index 3dbac73..ba9a6fe 100644 --- a/Makefile +++ b/Makefile @@ -9,8 +9,12 @@ COMPOSE_RUN_API = $(COMPOSE_RUN) api COMPOSE_RUN_MAIL = $(COMPOSE_RUN) mail-generator # -- MySQL -EDX_DB_HOST = mysql -EDX_DB_PORT = 3306 +EDX_MYSQL_DB_HOST = mysql +EDX_MYSQL_DB_PORT = 3306 + +# -- MongoDB +EDX_MONGO_DB_HOST = mongo +EDX_MONGO_DB_PORT = 27017 # -- Postgresql DB_HOST = postgresql @@ -52,7 +56,7 @@ bootstrap: \ build \ run \ migrate \ - seed-edx-database \ + seed-edx-databases \ mails-install \ mails-build .PHONY: bootstrap @@ -116,12 +120,14 @@ stop: ## stop all servers @$(COMPOSE) stop .PHONY: stop -seed-edx-database: ## seed the edx database with test data +seed-edx-databases: ## seed the edx MySQL and MongoDB databases with test data @echo "Waiting for mysql to be up and running…" - @$(COMPOSE_RUN) dockerize -wait tcp://$(EDX_DB_HOST):$(EDX_DB_PORT) -timeout 60s + @$(COMPOSE_RUN) dockerize -wait tcp://$(EDX_MYSQL_DB_HOST):$(EDX_MYSQL_DB_PORT) -timeout 60s + @echo "Waiting for mongodb to be up and running…" + @$(COMPOSE_RUN) dockerize -wait tcp://$(EDX_MONGO_DB_HOST):$(EDX_MONGO_DB_PORT) -timeout 60s @echo "Seeding the edx database…" @$(COMPOSE) exec -T celery python /opt/src/seed_edx_database.py -.PHONY: seed-edx-database +.PHONY: seed-edx-databases # -- Provisioning create-test-db: ## create test database diff --git a/bin/seed_edx_database.py b/bin/seed_edx_database.py index 21dee5a..cf4f4dc 100755 --- a/bin/seed_edx_database.py +++ b/bin/seed_edx_database.py @@ -5,14 +5,17 @@ from sqlalchemy import create_engine from sqlalchemy.orm import Session +from mongoengine import connect, disconnect + +from mork.edx.mongo.factories import CommentFactory, CommentThreadFactory from mork.conf import settings -from mork.edx.factories.auth import EdxAuthUserFactory -from mork.edx.models.base import Base +from mork.edx.mysql.factories.auth import EdxAuthUserFactory +from mork.edx.mysql.models.base import Base -async def seed_edx_database(): +async def seed_edx_mysql_database(): """Seed the MySQL edx database with mocked data.""" - engine = create_engine(settings.EDX_DB_URL) + engine = create_engine(settings.EDX_MYSQL_DB_URL) session = Session(engine) EdxAuthUserFactory._meta.sqlalchemy_session = session # noqa: SLF001 EdxAuthUserFactory._meta.sqlalchemy_session_persistence = "commit" # noqa: SLF001 @@ -21,5 +24,20 @@ async def seed_edx_database(): EdxAuthUserFactory.create_batch(1000) +async def seed_edx_mongodb_database(): + """Seed the MongoDB edx database with mocked data.""" + connect(host=settings.EDX_MONGO_DB_HOST, db=settings.EDX_MONGO_DB_NAME) + + CommentFactory.create_batch(1000) + CommentThreadFactory.create_batch(1000) + + disconnect(alias="mongodb") + + +async def main(): + tasks = [seed_edx_mysql_database(), seed_edx_mongodb_database()] + await asyncio.gather(*tasks) + + if __name__ == "__main__": - asyncio.run(seed_edx_database()) + asyncio.run(main()) diff --git a/docker-compose.yml b/docker-compose.yml index e0239c3..688ee1a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -49,6 +49,7 @@ services: - api - redis - mailcatcher + - mongo - mysql - postgresql @@ -76,3 +77,10 @@ services: env_file: - .env command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci + + mongo: + image: mongo:3.0.15 + ports: + - "27017:27017" + env_file: + - .env diff --git a/renovate.json b/renovate.json index 6381aa2..1724f06 100644 --- a/renovate.json +++ b/renovate.json @@ -6,6 +6,7 @@ "commitMessageAction": "upgrade", "commitBodyTable": true, "ignoreDeps": [ + "pymongo" ], "dependencyDashboard": true } diff --git a/src/app/mork/celery/tasks/deletion.py b/src/app/mork/celery/tasks/deletion.py index 8a8d641..c8236a4 100644 --- a/src/app/mork/celery/tasks/deletion.py +++ b/src/app/mork/celery/tasks/deletion.py @@ -9,8 +9,8 @@ from mork.celery.celery_app import app from mork.conf import settings from mork.database import MorkDB -from mork.edx import crud -from mork.edx.database import OpenEdxDB +from mork.edx.mysql import crud +from mork.edx.mysql.database import OpenEdxMySQLDB from mork.exceptions import UserDeleteError from mork.models import EmailStatus @@ -20,16 +20,16 @@ @app.task def delete_inactive_users(): """Celery task to delete inactive users accounts.""" - db = OpenEdxDB() + db = OpenEdxMySQLDB() threshold_date = datetime.now() - settings.DELETION_PERIOD total = crud.get_inactive_users_count(db.session, threshold_date) - for batch_offset in range(0, total, settings.EDX_QUERY_BATCH_SIZE): + for batch_offset in range(0, total, settings.EDX_MYSQL_QUERY_BATCH_SIZE): inactive_users = crud.get_inactive_users( db.session, threshold_date, offset=batch_offset, - limit=settings.EDX_QUERY_BATCH_SIZE, + limit=settings.EDX_MYSQL_QUERY_BATCH_SIZE, ) delete_group = group([delete_user.s(user.email) for user in inactive_users]) delete_group.delay() @@ -53,7 +53,7 @@ def delete_user(self, email: str): def delete_user_from_db(email): """Delete user from edX database.""" - db = OpenEdxDB() + db = OpenEdxMySQLDB() # Delete user from edX database crud.delete_user(db.session, email=email) diff --git a/src/app/mork/celery/tasks/emailing.py b/src/app/mork/celery/tasks/emailing.py index c968c56..e2d0c0e 100644 --- a/src/app/mork/celery/tasks/emailing.py +++ b/src/app/mork/celery/tasks/emailing.py @@ -9,8 +9,8 @@ from mork.celery.celery_app import app from mork.conf import settings from mork.database import MorkDB -from mork.edx import crud -from mork.edx.database import OpenEdxDB +from mork.edx.mysql import crud +from mork.edx.mysql.database import OpenEdxMySQLDB from mork.exceptions import EmailAlreadySent, EmailSendError from mork.mail import send_email from mork.models import EmailStatus @@ -21,17 +21,17 @@ @app.task def warn_inactive_users(): """Celery task to warn inactive users by email.""" - db = OpenEdxDB() + db = OpenEdxMySQLDB() threshold_date = datetime.now() - settings.WARNING_PERIOD total = crud.get_inactive_users_count(db.session, threshold_date) - for batch_offset in range(0, total, settings.EDX_QUERY_BATCH_SIZE): + for batch_offset in range(0, total, settings.EDX_MYSQL_QUERY_BATCH_SIZE): inactive_users = crud.get_inactive_users( db.session, threshold_date, offset=batch_offset, - limit=settings.EDX_QUERY_BATCH_SIZE, + limit=settings.EDX_MYSQL_QUERY_BATCH_SIZE, ) send_email_group = group( [warn_user.s(user.email, user.username) for user in inactive_users] diff --git a/src/app/mork/conf.py b/src/app/mork/conf.py index b859900..d1c15bf 100644 --- a/src/app/mork/conf.py +++ b/src/app/mork/conf.py @@ -34,6 +34,9 @@ class Settings(BaseSettings): DELETION_PERIOD: timedelta = "P3Y" DELETE_MAX_RETRIES: int = 3 + # Edx forum configuration + EDX_FORUM_PLACEHOLDER_USER_ID: int = 1234 + # API Root path # (used at least by everything that is alembic-configuration-related) ROOT_PATH: Path = Path(__file__).parent @@ -54,15 +57,24 @@ class Settings(BaseSettings): DB_DEBUG: bool = False TEST_DB_NAME: str = "test-mork-db" - # EDX database - EDX_DB_ENGINE: str = "mysql+pymysql" - EDX_DB_HOST: str = "mysql" - EDX_DB_NAME: str = "edxapp" - EDX_DB_USER: str = "edxapp" - EDX_DB_PASSWORD: str = "password" - EDX_DB_PORT: int = 3306 - EDX_DB_DEBUG: bool = False - EDX_QUERY_BATCH_SIZE: int = 1000 + # EDX MySQL database + EDX_MYSQL_DB_ENGINE: str = "mysql+pymysql" + EDX_MYSQL_DB_HOST: str = "mysql" + EDX_MYSQL_DB_NAME: str = "edxapp" + EDX_MYSQL_DB_USER: str = "edxapp" + EDX_MYSQL_DB_PASSWORD: str = "password" + EDX_MYSQL_DB_PORT: int = 3306 + EDX_MYSQL_DB_DEBUG: bool = False + EDX_MYSQL_QUERY_BATCH_SIZE: int = 1000 + + # EDX MongoDB database + EDX_MONGO_DB_ENGINE: str = "mongodb" + EDX_MONGO_DB_HOST: str = "mongo" + EDX_MONGO_DB_NAME: str = "cs_comments_service" + EDX_MONGO_DB_USER: str = "cs_comments_service" + EDX_MONGO_DB_PASSWORD: str = "password" + EDX_MONGO_DB_PORT: int = 27017 + EDX_MONGO_DB_DEBUG: bool = False # Redis configuration REDIS_HOST: str = "localhost" @@ -116,12 +128,21 @@ def TEST_DB_URL(self) -> str: ) @property - def EDX_DB_URL(self) -> str: + def EDX_MYSQL_DB_URL(self) -> str: + """Get the edx MySQL database URL as required by SQLAlchemy.""" + return ( + f"{self.EDX_MYSQL_DB_ENGINE}://" + f"{self.EDX_MYSQL_DB_USER}:{self.EDX_MYSQL_DB_PASSWORD}@" + f"{self.EDX_MYSQL_DB_HOST}/{self.EDX_MYSQL_DB_NAME}" + ) + + @property + def EDX_MONGO_DB_URL(self) -> str: """Get the edx database URL as required by SQLAlchemy.""" return ( - f"{self.EDX_DB_ENGINE}://" - f"{self.EDX_DB_USER}:{self.EDX_DB_PASSWORD}@" - f"{self.EDX_DB_HOST}/{self.EDX_DB_NAME}" + f"{self.EDX_MONGO_DB_ENGINE}://" + f"{self.EDX_MONGO_DB_USER}:{self.EDX_MONGO_DB_PASSWORD}@" + f"{self.EDX_MONGO_DB_HOST}/{self.EDX_MONGO_DB_NAME}" ) @property diff --git a/src/app/mork/edx/__init__.py b/src/app/mork/edx/__init__.py new file mode 100644 index 0000000..6e03199 --- /dev/null +++ b/src/app/mork/edx/__init__.py @@ -0,0 +1 @@ +# noqa: D104 diff --git a/src/app/mork/edx/factories/__init__.py b/src/app/mork/edx/factories/__init__.py deleted file mode 100644 index 5d3684c..0000000 --- a/src/app/mork/edx/factories/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Mork edx factories.""" diff --git a/src/app/mork/edx/models/__init__.py b/src/app/mork/edx/models/__init__.py deleted file mode 100644 index e5a1027..0000000 --- a/src/app/mork/edx/models/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Mork edx models.""" diff --git a/src/app/mork/edx/mongo/__init__.py b/src/app/mork/edx/mongo/__init__.py new file mode 100644 index 0000000..6e03199 --- /dev/null +++ b/src/app/mork/edx/mongo/__init__.py @@ -0,0 +1 @@ +# noqa: D104 diff --git a/src/app/mork/edx/mongo/crud.py b/src/app/mork/edx/mongo/crud.py new file mode 100644 index 0000000..10467d2 --- /dev/null +++ b/src/app/mork/edx/mongo/crud.py @@ -0,0 +1,40 @@ +"""Module for MongoDB CRUD functions.""" + +from logging import getLogger + +from mongoengine.queryset.visitor import Q + +from mork.conf import settings +from mork.edx.mongo.models import Comment, CommentThread + +logger = getLogger(__name__) + + +def anonymize_comments(username: str) -> None: + """Anonymize user comments and threads. + + Parameters: + username (str): The username of the user to delete comments from. + """ + comments = Comment.objects(Q(type="Comment") & Q(author_username=username)).all() + + comments.update( + author_username="[deleted]", + body="[deleted]", + author_id=settings.EDX_FORUM_PLACEHOLDER_USER_ID, + anonymous=True, + ) + + comment_threads = CommentThread.objects( + Q(type="CommentThread") & Q(author_username=username) + ).all() + + comment_threads.update( + author_username="[deleted]", + title="[deleted]", + body="[deleted]", + author_id=settings.EDX_FORUM_PLACEHOLDER_USER_ID, + anonymous=True, + ) + + logger.info(f"Anonymised user {username} comments and threads") diff --git a/src/app/mork/edx/mongo/database.py b/src/app/mork/edx/mongo/database.py new file mode 100644 index 0000000..6f98ae5 --- /dev/null +++ b/src/app/mork/edx/mongo/database.py @@ -0,0 +1,18 @@ +"""Mork edx MongoDB database connection.""" + +from pymongo import MongoClient + +# from pymongo.collection import Collection +from mork.conf import settings + + +class OpenEdxMongoDB: + """Class to connect to the Open edX MongoDB database.""" + + session = None + + def __init__(self): + """Instantiate the MongoDB client.""" + self.client = MongoClient(settings.EDX_MONGO_DB_URL) + self.database = self.client[settings.EDX_MONGO_DB_NAME] + self.collection = self.database["contents"] diff --git a/src/app/mork/edx/mongo/factories.py b/src/app/mork/edx/mongo/factories.py new file mode 100644 index 0000000..058c993 --- /dev/null +++ b/src/app/mork/edx/mongo/factories.py @@ -0,0 +1,52 @@ +"""Factory classes for MongoDB models.""" + +import factory + +from mork.edx.mongo.models import Comment, CommentThread + + +class CommentBaseFactory(factory.mongoengine.MongoEngineFactory): + """Base factory for MongoDB comment objects.""" + + votes = {} + visible = factory.Faker("pybool") + abuse_flaggers = [] + historical_abuse_flaggers = [] + thread_type = factory.Faker("pystr") + context = factory.Faker("pystr") + comment_count = factory.Faker("pyint") + at_position_list = [] + title = factory.Faker("pystr") + body = factory.Faker("pystr") + course_id = factory.Faker("pystr") + commentable_id = factory.Faker("pystr") + anonymous = factory.Faker("pybool") + anonymous_to_peers = factory.Faker("pybool") + closed = factory.Faker("pybool") + author_id = factory.Faker("pyint") + author_username = factory.Faker("pystr") + updated_at = factory.Faker("date_time") + created_at = factory.Faker("date_time") + last_activity_at = factory.Faker("date_time") + + +class CommentFactory(CommentBaseFactory): + """Factory for the `Comment` document type.""" + + class Meta: + """Factory configuration.""" + + model = Comment + + type = "Comment" + + +class CommentThreadFactory(CommentBaseFactory): + """Factory for the `CommentThread` document type.""" + + class Meta: + """Factory configuration.""" + + model = CommentThread + + type = "CommentThread" diff --git a/src/app/mork/edx/mongo/models.py b/src/app/mork/edx/mongo/models.py new file mode 100644 index 0000000..b2a34bc --- /dev/null +++ b/src/app/mork/edx/mongo/models.py @@ -0,0 +1,42 @@ +"""Mork edx MongoDB models.""" + +import mongoengine + + +class CommentBase(mongoengine.Document): + """Base model for a forum comment document.""" + + _id = mongoengine.ObjectIdField() + votes = mongoengine.DictField() + visible = mongoengine.BooleanField() + abuse_flaggers = mongoengine.ListField() + historical_abuse_flaggers = mongoengine.ListField() + thread_type = mongoengine.StringField() + context = mongoengine.StringField() + comment_count = mongoengine.IntField() + at_position_list = mongoengine.ListField() + title = mongoengine.StringField() + body = mongoengine.StringField() + course_id = mongoengine.StringField() + commentable_id = mongoengine.StringField() + anonymous = mongoengine.BooleanField() + anonymous_to_peers = mongoengine.BooleanField() + closed = mongoengine.BooleanField() + author_id = mongoengine.IntField() + author_username = mongoengine.StringField() + updated_at = mongoengine.DateField() + created_at = mongoengine.DateField() + last_activity_at = mongoengine.DateField() + meta = {"abstract": True, "collection": "contents"} + + +class CommentThread(CommentBase): + """Model for the `CommentThread` document type.""" + + type = mongoengine.StringField(default="CommentThread", db_field="_type") + + +class Comment(CommentBase): + """Model for the `Comment` document type.""" + + type = mongoengine.StringField(default="Comment", db_field="_type") diff --git a/src/app/mork/edx/mysql/__init__.py b/src/app/mork/edx/mysql/__init__.py new file mode 100644 index 0000000..6e03199 --- /dev/null +++ b/src/app/mork/edx/mysql/__init__.py @@ -0,0 +1 @@ +# noqa: D104 diff --git a/src/app/mork/edx/crud.py b/src/app/mork/edx/mysql/crud.py similarity index 89% rename from src/app/mork/edx/crud.py rename to src/app/mork/edx/mysql/crud.py index 494a644..3dcea87 100644 --- a/src/app/mork/edx/crud.py +++ b/src/app/mork/edx/mysql/crud.py @@ -8,19 +8,19 @@ from sqlalchemy.orm import Session, load_only from sqlalchemy.sql.functions import count -from mork.edx.models.auth import AuthtokenToken, AuthUser -from mork.edx.models.certificates import ( +from mork.edx.mysql.models.auth import AuthtokenToken, AuthUser +from mork.edx.mysql.models.certificates import ( CertificatesCertificatehtmlviewconfiguration, ) -from mork.edx.models.contentstore import ContentstoreVideouploadconfig -from mork.edx.models.course import ( +from mork.edx.mysql.models.contentstore import ContentstoreVideouploadconfig +from mork.edx.mysql.models.course import ( CourseActionStateCoursererunstate, CourseCreatorsCoursecreator, ) -from mork.edx.models.dark import DarkLangDarklangconfig -from mork.edx.models.student import StudentCourseenrollmentallowed -from mork.edx.models.util import UtilRatelimitconfiguration -from mork.edx.models.verify import VerifyStudentHistoricalverificationdeadline +from mork.edx.mysql.models.dark import DarkLangDarklangconfig +from mork.edx.mysql.models.student import StudentCourseenrollmentallowed +from mork.edx.mysql.models.util import UtilRatelimitconfiguration +from mork.edx.mysql.models.verify import VerifyStudentHistoricalverificationdeadline from mork.exceptions import UserDeleteError, UserProtectedDeleteError logger = getLogger(__name__) diff --git a/src/app/mork/edx/database.py b/src/app/mork/edx/mysql/database.py similarity index 61% rename from src/app/mork/edx/database.py rename to src/app/mork/edx/mysql/database.py index 54365c5..82f5505 100644 --- a/src/app/mork/edx/database.py +++ b/src/app/mork/edx/mysql/database.py @@ -1,4 +1,4 @@ -"""Mork edx database connection.""" +"""Mork edx MySQL database connection.""" import logging @@ -10,18 +10,20 @@ logger = logging.getLogger(__name__) -class OpenEdxDB: - """Class to connect to the Open edX database.""" +class OpenEdxMySQLDB: + """Class to connect to the Open edX MySQL database.""" session = None def __init__(self, engine=None, session=None): - """Initialize SqlAlchemy engine and session.""" + """Instantiate SQLAlchemy engine and session.""" if engine is not None: self.engine = engine else: self.engine = create_engine( - settings.EDX_DB_URL, echo=settings.EDX_DB_DEBUG, pool_pre_ping=True + settings.EDX_MYSQL_DB_URL, + echo=settings.EDX_MYSQL_DB_DEBUG, + pool_pre_ping=True, ) if session is not None: self.session = session diff --git a/src/app/mork/edx/mysql/factories/__init__.py b/src/app/mork/edx/mysql/factories/__init__.py new file mode 100644 index 0000000..f4af411 --- /dev/null +++ b/src/app/mork/edx/mysql/factories/__init__.py @@ -0,0 +1 @@ +"""Mork edx MySQL factories.""" diff --git a/src/app/mork/edx/factories/auth.py b/src/app/mork/edx/mysql/factories/auth.py similarity index 99% rename from src/app/mork/edx/factories/auth.py rename to src/app/mork/edx/mysql/factories/auth.py index fc7fb53..1604bb5 100644 --- a/src/app/mork/edx/factories/auth.py +++ b/src/app/mork/edx/mysql/factories/auth.py @@ -2,7 +2,7 @@ import factory -from mork.edx.models.auth import ( +from mork.edx.mysql.models.auth import ( AuthRegistration, AuthtokenToken, AuthUser, diff --git a/src/app/mork/edx/factories/base.py b/src/app/mork/edx/mysql/factories/base.py similarity index 86% rename from src/app/mork/edx/factories/base.py rename to src/app/mork/edx/mysql/factories/base.py index d8ba6ba..0916f16 100644 --- a/src/app/mork/edx/factories/base.py +++ b/src/app/mork/edx/mysql/factories/base.py @@ -4,7 +4,7 @@ from sqlalchemy import create_engine from sqlalchemy.orm import Session -from mork.edx.models.base import Base +from mork.edx.mysql.models.base import Base faker = Faker() engine = create_engine("sqlite+pysqlite:///:memory:", echo=False, pool_pre_ping=True) diff --git a/src/app/mork/edx/factories/bulk.py b/src/app/mork/edx/mysql/factories/bulk.py similarity index 94% rename from src/app/mork/edx/factories/bulk.py rename to src/app/mork/edx/mysql/factories/bulk.py index a3f0ac4..fc5c85f 100644 --- a/src/app/mork/edx/factories/bulk.py +++ b/src/app/mork/edx/mysql/factories/bulk.py @@ -2,7 +2,7 @@ import factory -from mork.edx.models.bulk import BulkEmailCourseemail, BulkEmailOptout +from mork.edx.mysql.models.bulk import BulkEmailCourseemail, BulkEmailOptout from .base import faker, session diff --git a/src/app/mork/edx/factories/certificates.py b/src/app/mork/edx/mysql/factories/certificates.py similarity index 97% rename from src/app/mork/edx/factories/certificates.py rename to src/app/mork/edx/mysql/factories/certificates.py index 773efe8..5c85715 100644 --- a/src/app/mork/edx/factories/certificates.py +++ b/src/app/mork/edx/mysql/factories/certificates.py @@ -2,7 +2,7 @@ import factory -from mork.edx.models.certificates import ( +from mork.edx.mysql.models.certificates import ( CertificatesCertificatehtmlviewconfiguration, CertificatesGeneratedcertificate, ) diff --git a/src/app/mork/edx/factories/contentstore.py b/src/app/mork/edx/mysql/factories/contentstore.py similarity index 89% rename from src/app/mork/edx/factories/contentstore.py rename to src/app/mork/edx/mysql/factories/contentstore.py index f5f9197..9955bbc 100644 --- a/src/app/mork/edx/factories/contentstore.py +++ b/src/app/mork/edx/mysql/factories/contentstore.py @@ -2,7 +2,7 @@ import factory -from mork.edx.models.contentstore import ContentstoreVideouploadconfig +from mork.edx.mysql.models.contentstore import ContentstoreVideouploadconfig from .base import session diff --git a/src/app/mork/edx/factories/course.py b/src/app/mork/edx/mysql/factories/course.py similarity index 98% rename from src/app/mork/edx/factories/course.py rename to src/app/mork/edx/mysql/factories/course.py index 7e1b0af..e6abb98 100644 --- a/src/app/mork/edx/factories/course.py +++ b/src/app/mork/edx/mysql/factories/course.py @@ -2,7 +2,7 @@ import factory -from mork.edx.models.course import ( +from mork.edx.mysql.models.course import ( CourseActionStateCoursererunstate, CourseCreatorsCoursecreator, CourseGroupsCohortmembership, diff --git a/src/app/mork/edx/factories/courseware.py b/src/app/mork/edx/mysql/factories/courseware.py similarity index 98% rename from src/app/mork/edx/factories/courseware.py rename to src/app/mork/edx/mysql/factories/courseware.py index 771ba9c..b853f4a 100644 --- a/src/app/mork/edx/factories/courseware.py +++ b/src/app/mork/edx/mysql/factories/courseware.py @@ -2,7 +2,7 @@ import factory -from mork.edx.models.courseware import ( +from mork.edx.mysql.models.courseware import ( CoursewareOfflinecomputedgrade, CoursewareStudentmodule, CoursewareStudentmodulehistory, diff --git a/src/app/mork/edx/factories/dark.py b/src/app/mork/edx/mysql/factories/dark.py similarity index 90% rename from src/app/mork/edx/factories/dark.py rename to src/app/mork/edx/mysql/factories/dark.py index 6661600..1a47820 100644 --- a/src/app/mork/edx/factories/dark.py +++ b/src/app/mork/edx/mysql/factories/dark.py @@ -2,7 +2,7 @@ import factory -from mork.edx.models.dark import DarkLangDarklangconfig +from mork.edx.mysql.models.dark import DarkLangDarklangconfig from .base import session diff --git a/src/app/mork/edx/factories/django.py b/src/app/mork/edx/mysql/factories/django.py similarity index 88% rename from src/app/mork/edx/factories/django.py rename to src/app/mork/edx/mysql/factories/django.py index 6ca940e..a8f1b27 100644 --- a/src/app/mork/edx/factories/django.py +++ b/src/app/mork/edx/mysql/factories/django.py @@ -2,7 +2,7 @@ import factory -from mork.edx.models.django import DjangoCommentClientRoleUsers +from mork.edx.mysql.models.django import DjangoCommentClientRoleUsers from .base import session diff --git a/src/app/mork/edx/factories/instructor.py b/src/app/mork/edx/mysql/factories/instructor.py similarity index 92% rename from src/app/mork/edx/factories/instructor.py rename to src/app/mork/edx/mysql/factories/instructor.py index 6c63d66..f3914e1 100644 --- a/src/app/mork/edx/factories/instructor.py +++ b/src/app/mork/edx/mysql/factories/instructor.py @@ -2,7 +2,7 @@ import factory -from mork.edx.models.instructor import InstructorTaskInstructortask +from mork.edx.mysql.models.instructor import InstructorTaskInstructortask from .base import faker, session diff --git a/src/app/mork/edx/factories/notify.py b/src/app/mork/edx/mysql/factories/notify.py similarity index 89% rename from src/app/mork/edx/factories/notify.py rename to src/app/mork/edx/mysql/factories/notify.py index 5e23eed..488e290 100644 --- a/src/app/mork/edx/factories/notify.py +++ b/src/app/mork/edx/mysql/factories/notify.py @@ -2,7 +2,7 @@ import factory -from mork.edx.models.notify import NotifySetting +from mork.edx.mysql.models.notify import NotifySetting from .base import session diff --git a/src/app/mork/edx/factories/payment.py b/src/app/mork/edx/mysql/factories/payment.py similarity index 89% rename from src/app/mork/edx/factories/payment.py rename to src/app/mork/edx/mysql/factories/payment.py index cc46580..7193edb 100644 --- a/src/app/mork/edx/factories/payment.py +++ b/src/app/mork/edx/mysql/factories/payment.py @@ -2,7 +2,7 @@ import factory -from mork.edx.models.payment import PaymentUseracceptance +from mork.edx.mysql.models.payment import PaymentUseracceptance from .base import session diff --git a/src/app/mork/edx/factories/proctoru.py b/src/app/mork/edx/mysql/factories/proctoru.py similarity index 95% rename from src/app/mork/edx/factories/proctoru.py rename to src/app/mork/edx/mysql/factories/proctoru.py index 088c354..6df311f 100644 --- a/src/app/mork/edx/factories/proctoru.py +++ b/src/app/mork/edx/mysql/factories/proctoru.py @@ -2,7 +2,7 @@ import factory -from mork.edx.models.proctoru import ProctoruProctoruexam, ProctoruProctoruuser +from mork.edx.mysql.models.proctoru import ProctoruProctoruexam, ProctoruProctoruuser from .base import session diff --git a/src/app/mork/edx/factories/student.py b/src/app/mork/edx/mysql/factories/student.py similarity index 99% rename from src/app/mork/edx/factories/student.py rename to src/app/mork/edx/mysql/factories/student.py index c376478..23695c9 100644 --- a/src/app/mork/edx/factories/student.py +++ b/src/app/mork/edx/mysql/factories/student.py @@ -2,7 +2,7 @@ import factory -from mork.edx.models.student import ( +from mork.edx.mysql.models.student import ( StudentAnonymoususerid, StudentCourseaccessrole, StudentCourseenrollment, diff --git a/src/app/mork/edx/factories/user.py b/src/app/mork/edx/mysql/factories/user.py similarity index 91% rename from src/app/mork/edx/factories/user.py rename to src/app/mork/edx/mysql/factories/user.py index cca5477..07ec677 100644 --- a/src/app/mork/edx/factories/user.py +++ b/src/app/mork/edx/mysql/factories/user.py @@ -2,7 +2,7 @@ import factory -from mork.edx.models.user import UserApiUserpreference +from mork.edx.mysql.models.user import UserApiUserpreference from .base import session diff --git a/src/app/mork/edx/factories/util.py b/src/app/mork/edx/mysql/factories/util.py similarity index 89% rename from src/app/mork/edx/factories/util.py rename to src/app/mork/edx/mysql/factories/util.py index 1f852d9..b86475d 100644 --- a/src/app/mork/edx/factories/util.py +++ b/src/app/mork/edx/mysql/factories/util.py @@ -2,7 +2,7 @@ import factory -from mork.edx.models.util import UtilRatelimitconfiguration +from mork.edx.mysql.models.util import UtilRatelimitconfiguration from .base import session diff --git a/src/app/mork/edx/factories/verify.py b/src/app/mork/edx/mysql/factories/verify.py similarity index 98% rename from src/app/mork/edx/factories/verify.py rename to src/app/mork/edx/mysql/factories/verify.py index 4f56987..e01c789 100644 --- a/src/app/mork/edx/factories/verify.py +++ b/src/app/mork/edx/mysql/factories/verify.py @@ -2,7 +2,7 @@ import factory -from mork.edx.models.verify import ( +from mork.edx.mysql.models.verify import ( VerifyStudentHistoricalverificationdeadline, VerifyStudentSoftwaresecurephotoverification, ) diff --git a/src/app/mork/edx/mysql/models/__init__.py b/src/app/mork/edx/mysql/models/__init__.py new file mode 100644 index 0000000..4e06166 --- /dev/null +++ b/src/app/mork/edx/mysql/models/__init__.py @@ -0,0 +1 @@ +"""Mork edx MySQL models.""" diff --git a/src/app/mork/edx/models/auth.py b/src/app/mork/edx/mysql/models/auth.py similarity index 100% rename from src/app/mork/edx/models/auth.py rename to src/app/mork/edx/mysql/models/auth.py diff --git a/src/app/mork/edx/models/base.py b/src/app/mork/edx/mysql/models/base.py similarity index 100% rename from src/app/mork/edx/models/base.py rename to src/app/mork/edx/mysql/models/base.py diff --git a/src/app/mork/edx/models/bulk.py b/src/app/mork/edx/mysql/models/bulk.py similarity index 100% rename from src/app/mork/edx/models/bulk.py rename to src/app/mork/edx/mysql/models/bulk.py diff --git a/src/app/mork/edx/models/certificates.py b/src/app/mork/edx/mysql/models/certificates.py similarity index 100% rename from src/app/mork/edx/models/certificates.py rename to src/app/mork/edx/mysql/models/certificates.py diff --git a/src/app/mork/edx/models/contentstore.py b/src/app/mork/edx/mysql/models/contentstore.py similarity index 100% rename from src/app/mork/edx/models/contentstore.py rename to src/app/mork/edx/mysql/models/contentstore.py diff --git a/src/app/mork/edx/models/course.py b/src/app/mork/edx/mysql/models/course.py similarity index 100% rename from src/app/mork/edx/models/course.py rename to src/app/mork/edx/mysql/models/course.py diff --git a/src/app/mork/edx/models/courseware.py b/src/app/mork/edx/mysql/models/courseware.py similarity index 100% rename from src/app/mork/edx/models/courseware.py rename to src/app/mork/edx/mysql/models/courseware.py diff --git a/src/app/mork/edx/models/dark.py b/src/app/mork/edx/mysql/models/dark.py similarity index 100% rename from src/app/mork/edx/models/dark.py rename to src/app/mork/edx/mysql/models/dark.py diff --git a/src/app/mork/edx/models/django.py b/src/app/mork/edx/mysql/models/django.py similarity index 100% rename from src/app/mork/edx/models/django.py rename to src/app/mork/edx/mysql/models/django.py diff --git a/src/app/mork/edx/models/instructor.py b/src/app/mork/edx/mysql/models/instructor.py similarity index 100% rename from src/app/mork/edx/models/instructor.py rename to src/app/mork/edx/mysql/models/instructor.py diff --git a/src/app/mork/edx/models/notify.py b/src/app/mork/edx/mysql/models/notify.py similarity index 100% rename from src/app/mork/edx/models/notify.py rename to src/app/mork/edx/mysql/models/notify.py diff --git a/src/app/mork/edx/models/payment.py b/src/app/mork/edx/mysql/models/payment.py similarity index 100% rename from src/app/mork/edx/models/payment.py rename to src/app/mork/edx/mysql/models/payment.py diff --git a/src/app/mork/edx/models/proctoru.py b/src/app/mork/edx/mysql/models/proctoru.py similarity index 100% rename from src/app/mork/edx/models/proctoru.py rename to src/app/mork/edx/mysql/models/proctoru.py diff --git a/src/app/mork/edx/models/student.py b/src/app/mork/edx/mysql/models/student.py similarity index 100% rename from src/app/mork/edx/models/student.py rename to src/app/mork/edx/mysql/models/student.py diff --git a/src/app/mork/edx/models/user.py b/src/app/mork/edx/mysql/models/user.py similarity index 100% rename from src/app/mork/edx/models/user.py rename to src/app/mork/edx/mysql/models/user.py diff --git a/src/app/mork/edx/models/util.py b/src/app/mork/edx/mysql/models/util.py similarity index 100% rename from src/app/mork/edx/models/util.py rename to src/app/mork/edx/mysql/models/util.py diff --git a/src/app/mork/edx/models/verify.py b/src/app/mork/edx/mysql/models/verify.py similarity index 100% rename from src/app/mork/edx/models/verify.py rename to src/app/mork/edx/mysql/models/verify.py diff --git a/src/app/mork/tests/celery/test_deletion_task.py b/src/app/mork/tests/celery/test_deletion_task.py index 671dd08..cfee253 100644 --- a/src/app/mork/tests/celery/test_deletion_task.py +++ b/src/app/mork/tests/celery/test_deletion_task.py @@ -14,14 +14,14 @@ delete_user, delete_user_from_db, ) -from mork.edx import crud -from mork.edx.factories.auth import EdxAuthUserFactory +from mork.edx.mysql import crud +from mork.edx.mysql.factories.auth import EdxAuthUserFactory from mork.exceptions import UserDeleteError from mork.factories import EmailStatusFactory from mork.models import EmailStatus -def test_delete_inactive_users(edx_db, monkeypatch): +def test_delete_inactive_users(edx_mysql_db, monkeypatch): """Test the `delete_inactive_users` function.""" # 2 users that did not log in for 3 years EdxAuthUserFactory.create( @@ -42,7 +42,9 @@ def test_delete_inactive_users(edx_db, monkeypatch): email="janedah2@example.com", ) - monkeypatch.setattr("mork.celery.tasks.deletion.OpenEdxDB", lambda *args: edx_db) + monkeypatch.setattr( + "mork.celery.tasks.deletion.OpenEdxMySQLDB", lambda *args: edx_mysql_db + ) mock_group = Mock() monkeypatch.setattr("mork.celery.tasks.deletion.group", mock_group) @@ -59,7 +61,7 @@ def test_delete_inactive_users(edx_db, monkeypatch): ) -def test_delete_inactive_users_with_batch_size(edx_db, monkeypatch): +def test_delete_inactive_users_with_batch_size(edx_mysql_db, monkeypatch): """Test the `warn_inactive_users` function.""" # 2 users that did not log in for 3 years EdxAuthUserFactory.create( @@ -71,7 +73,9 @@ def test_delete_inactive_users_with_batch_size(edx_db, monkeypatch): email="johndoe2@example.com", ) - monkeypatch.setattr("mork.celery.tasks.deletion.OpenEdxDB", lambda *args: edx_db) + monkeypatch.setattr( + "mork.celery.tasks.deletion.OpenEdxMySQLDB", lambda *args: edx_mysql_db + ) mock_group = Mock() monkeypatch.setattr("mork.celery.tasks.deletion.group", mock_group) @@ -79,7 +83,9 @@ def test_delete_inactive_users_with_batch_size(edx_db, monkeypatch): monkeypatch.setattr("mork.celery.tasks.deletion.delete_user", mock_delete_user) # Set batch size to 1 - monkeypatch.setattr("mork.celery.tasks.deletion.settings.EDX_QUERY_BATCH_SIZE", 1) + monkeypatch.setattr( + "mork.celery.tasks.deletion.settings.EDX_MYSQL_QUERY_BATCH_SIZE", 1 + ) delete_inactive_users() @@ -132,45 +138,49 @@ def mock_delete_user_from_db(*args): delete_user("johndoe@example.com") -def test_delete_user_from_db(edx_db, monkeypatch): +def test_delete_user_from_db(edx_mysql_db, monkeypatch): """Test the `delete_user_from_db` function.""" - EdxAuthUserFactory._meta.sqlalchemy_session = edx_db.session + EdxAuthUserFactory._meta.sqlalchemy_session = edx_mysql_db.session EdxAuthUserFactory.create(email="johndoe1@example.com") EdxAuthUserFactory.create(email="johndoe2@example.com") - monkeypatch.setattr("mork.celery.tasks.deletion.OpenEdxDB", lambda *args: edx_db) + monkeypatch.setattr( + "mork.celery.tasks.deletion.OpenEdxMySQLDB", lambda *args: edx_mysql_db + ) assert crud.get_user( - edx_db.session, + edx_mysql_db.session, email="johndoe1@example.com", ) assert crud.get_user( - edx_db.session, + edx_mysql_db.session, email="johndoe2@example.com", ) delete_user_from_db(email="johndoe1@example.com") assert not crud.get_user( - edx_db.session, + edx_mysql_db.session, email="johndoe1@example.com", ) assert crud.get_user( - edx_db.session, + edx_mysql_db.session, email="johndoe2@example.com", ) -def test_delete_user_from_db_with_failure(edx_db, monkeypatch): +def test_delete_user_from_db_with_failure(edx_mysql_db, monkeypatch): """Test the `delete_user_from_db` function with a commit failure.""" - EdxAuthUserFactory._meta.sqlalchemy_session = edx_db.session + EdxAuthUserFactory._meta.sqlalchemy_session = edx_mysql_db.session EdxAuthUserFactory.create(email="johndoe1@example.com") def mock_session_commit(): raise SQLAlchemyError("An error occurred") - edx_db.session.commit = mock_session_commit - monkeypatch.setattr("mork.celery.tasks.deletion.OpenEdxDB", lambda *args: edx_db) + edx_mysql_db.session.commit = mock_session_commit + monkeypatch.setattr( + "mork.celery.tasks.deletion.OpenEdxMySQLDB", lambda *args: edx_mysql_db + ) with pytest.raises(UserDeleteError, match="Failed to delete user."): delete_user_from_db(email="johndoe1@example.com") diff --git a/src/app/mork/tests/celery/test_emailing_tasks.py b/src/app/mork/tests/celery/test_emailing_tasks.py index 4871919..f926de4 100644 --- a/src/app/mork/tests/celery/test_emailing_tasks.py +++ b/src/app/mork/tests/celery/test_emailing_tasks.py @@ -11,12 +11,12 @@ warn_inactive_users, warn_user, ) -from mork.edx.factories.auth import EdxAuthUserFactory +from mork.edx.mysql.factories.auth import EdxAuthUserFactory from mork.exceptions import EmailAlreadySent, EmailSendError from mork.factories import EmailStatusFactory -def test_warn_inactive_users(edx_db, monkeypatch): +def test_warn_inactive_users(edx_mysql_db, monkeypatch): """Test the `warn_inactive_users` function.""" # 2 users that did not log in for 3 years EdxAuthUserFactory.create( @@ -41,7 +41,9 @@ def test_warn_inactive_users(edx_db, monkeypatch): email="janedah2@example.com", ) - monkeypatch.setattr("mork.celery.tasks.emailing.OpenEdxDB", lambda *args: edx_db) + monkeypatch.setattr( + "mork.celery.tasks.emailing.OpenEdxMySQLDB", lambda *args: edx_mysql_db + ) mock_group = Mock() monkeypatch.setattr("mork.celery.tasks.emailing.group", mock_group) @@ -58,7 +60,7 @@ def test_warn_inactive_users(edx_db, monkeypatch): ) -def test_warn_inactive_users_with_batch_size(edx_db, monkeypatch): +def test_warn_inactive_users_with_batch_size(edx_mysql_db, monkeypatch): """Test the `warn_inactive_users` function.""" # 2 users that did not log in for 3 years EdxAuthUserFactory.create( @@ -72,7 +74,9 @@ def test_warn_inactive_users_with_batch_size(edx_db, monkeypatch): email="johndoe2@example.com", ) - monkeypatch.setattr("mork.celery.tasks.emailing.OpenEdxDB", lambda *args: edx_db) + monkeypatch.setattr( + "mork.celery.tasks.emailing.OpenEdxMySQLDB", lambda *args: edx_mysql_db + ) mock_group = Mock() monkeypatch.setattr("mork.celery.tasks.emailing.group", mock_group) @@ -80,7 +84,9 @@ def test_warn_inactive_users_with_batch_size(edx_db, monkeypatch): monkeypatch.setattr("mork.celery.tasks.emailing.warn_user", mock_warn_user) # Set batch size to 1 - monkeypatch.setattr("mork.celery.tasks.emailing.settings.EDX_QUERY_BATCH_SIZE", 1) + monkeypatch.setattr( + "mork.celery.tasks.emailing.settings.EDX_MYSQL_QUERY_BATCH_SIZE", 1 + ) warn_inactive_users() diff --git a/src/app/mork/tests/conftest.py b/src/app/mork/tests/conftest.py index 587a0ae..ab5da79 100644 --- a/src/app/mork/tests/conftest.py +++ b/src/app/mork/tests/conftest.py @@ -7,6 +7,6 @@ from .fixtures.app import http_client from .fixtures.asynchronous import anyio_backend from .fixtures.auth import auth_headers -from .fixtures.db import db_engine, db_session, edx_db +from .fixtures.db import db_engine, db_session, edx_mongo_db, edx_mysql_db TEST_STATIC_PATH = Path(__file__).parent / "static" diff --git a/src/app/mork/tests/edx/mongo/__init__.py b/src/app/mork/tests/edx/mongo/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/app/mork/tests/edx/mongo/test_crud.py b/src/app/mork/tests/edx/mongo/test_crud.py new file mode 100644 index 0000000..ffabf90 --- /dev/null +++ b/src/app/mork/tests/edx/mongo/test_crud.py @@ -0,0 +1,21 @@ +"""Tests of the MongoDB related CRUD functions.""" + +from mork.edx.mongo import crud +from mork.edx.mongo.factories import CommentFactory, CommentThreadFactory +from mork.edx.mongo.models import Comment, CommentThread + + +def test_edx_mongo_crud_anonymize_comments(edx_mongo_db): + """Test the `anonymize_comments` method.""" + username = "JohnDoe" + + CommentFactory.create_batch(3, author_username=username) + CommentThreadFactory.create_batch(3, author_username=username) + + assert Comment.objects(author_username=username).count() > 0 + assert CommentThread.objects(author_username=username).count() > 0 + + crud.anonymize_comments(username) + + assert Comment.objects(author_username=username).count() == 0 + assert CommentThread.objects(author_username=username).count() == 0 diff --git a/src/app/mork/tests/edx/mysql/__init__.py b/src/app/mork/tests/edx/mysql/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/app/mork/tests/edx/test_crud.py b/src/app/mork/tests/edx/mysql/test_crud.py similarity index 71% rename from src/app/mork/tests/edx/test_crud.py rename to src/app/mork/tests/edx/mysql/test_crud.py index ec8b9e9..df27a02 100644 --- a/src/app/mork/tests/edx/test_crud.py +++ b/src/app/mork/tests/edx/mysql/test_crud.py @@ -5,30 +5,32 @@ import pytest from faker import Faker -from mork.edx import crud -from mork.edx.factories.auth import EdxAuthtokenTokenFactory, EdxAuthUserFactory -from mork.edx.factories.certificates import ( +from mork.edx.mysql import crud +from mork.edx.mysql.factories.auth import EdxAuthtokenTokenFactory, EdxAuthUserFactory +from mork.edx.mysql.factories.certificates import ( EdxCertificatesCertificatehtmlviewconfigurationFactory, ) -from mork.edx.factories.contentstore import EdxContentstoreVideouploadconfigFactory -from mork.edx.factories.course import ( +from mork.edx.mysql.factories.contentstore import ( + EdxContentstoreVideouploadconfigFactory, +) +from mork.edx.mysql.factories.course import ( EdxCourseActionStateCoursererunstateFactory, EdxCourseCreatorsCoursecreatorFactory, ) -from mork.edx.factories.dark import EdxDarkLangDarklangconfigFactory -from mork.edx.factories.student import ( +from mork.edx.mysql.factories.dark import EdxDarkLangDarklangconfigFactory +from mork.edx.mysql.factories.student import ( EdxStudentCourseenrollmentallowedFactory, ) -from mork.edx.factories.util import EdxUtilRatelimitconfigurationFactory -from mork.edx.factories.verify import ( +from mork.edx.mysql.factories.util import EdxUtilRatelimitconfigurationFactory +from mork.edx.mysql.factories.verify import ( EdxVerifyStudentHistoricalverificationdeadlineFactory, ) -from mork.edx.models.auth import AuthUser -from mork.edx.models.base import Base +from mork.edx.mysql.models.auth import AuthUser +from mork.edx.mysql.models.base import Base from mork.exceptions import UserDeleteError, UserProtectedDeleteError -def test_edx_crud_get_inactive_users_count(edx_db): +def test_edx_crud_get_inactive_users_count(edx_mysql_db): """Test the `get_inactive_users_count` method.""" # 3 users that did not log in for 3 years EdxAuthUserFactory.create_batch( @@ -42,22 +44,22 @@ def test_edx_crud_get_inactive_users_count(edx_db): threshold_date = datetime.now() - timedelta(days=365 * 3) # Get count of users inactive for more than 3 years - users_count = crud.get_inactive_users_count(edx_db.session, threshold_date) + users_count = crud.get_inactive_users_count(edx_mysql_db.session, threshold_date) assert users_count == 3 -def test_edx_crud_get_inactive_users_count_empty(edx_db): +def test_edx_crud_get_inactive_users_count_empty(edx_mysql_db): """Test the `get_inactive_users_count` method with no inactive users.""" threshold_date = datetime.now() - timedelta(days=365 * 3) # Get count of users inactive for more than 3 years - users_count = crud.get_inactive_users_count(edx_db.session, threshold_date) + users_count = crud.get_inactive_users_count(edx_mysql_db.session, threshold_date) assert users_count == 0 -def test_edx_crud_get_inactive_users(edx_db): +def test_edx_crud_get_inactive_users(edx_mysql_db): """Test the `get_inactive_users` method.""" # 3 users that did not log in for 3 years @@ -72,23 +74,27 @@ def test_edx_crud_get_inactive_users(edx_db): threshold_date = datetime.now() - timedelta(days=365 * 3) # Get all users inactive for more than 3 years - users = crud.get_inactive_users(edx_db.session, threshold_date, offset=0, limit=9) + users = crud.get_inactive_users( + edx_mysql_db.session, threshold_date, offset=0, limit=9 + ) assert len(users) == 3 assert users == inactive_users -def test_edx_crud_get_inactive_users_empty(edx_db): +def test_edx_crud_get_inactive_users_empty(edx_mysql_db): """Test the `get_inactive_users` method with no inactive users.""" threshold_date = datetime.now() - timedelta(days=365 * 3) - users = crud.get_inactive_users(edx_db.session, threshold_date, offset=0, limit=9) + users = crud.get_inactive_users( + edx_mysql_db.session, threshold_date, offset=0, limit=9 + ) assert users == [] -def test_edx_crud_get_inactive_users_slice(edx_db): +def test_edx_crud_get_inactive_users_slice(edx_mysql_db): """Test the `get_inactive_users` method with a slice.""" # 3 users that did not log in for 3 years inactive_users = EdxAuthUserFactory.create_batch( @@ -102,13 +108,15 @@ def test_edx_crud_get_inactive_users_slice(edx_db): threshold_date = datetime.now() - timedelta(days=365 * 3) # Get two users inactive for more than 3 years - users = crud.get_inactive_users(edx_db.session, threshold_date, offset=0, limit=2) + users = crud.get_inactive_users( + edx_mysql_db.session, threshold_date, offset=0, limit=2 + ) assert len(users) == 2 assert users == inactive_users[:2] -def test_edx_crud_get_inactive_users_slice_empty(edx_db): +def test_edx_crud_get_inactive_users_slice_empty(edx_mysql_db): """Test the `get_inactive_users` method with an empty slice .""" # 3 users that did not log in for 3 years EdxAuthUserFactory.create_batch( @@ -121,36 +129,38 @@ def test_edx_crud_get_inactive_users_slice_empty(edx_db): threshold_date = datetime.now() - timedelta(days=365 * 3) - users = crud.get_inactive_users(edx_db.session, threshold_date, offset=4, limit=9) + users = crud.get_inactive_users( + edx_mysql_db.session, threshold_date, offset=4, limit=9 + ) assert users == [] -def test_edx_crud_get_user_missing(edx_db): +def test_edx_crud_get_user_missing(edx_mysql_db): """Test the `get_user` method with missing user in the database.""" - user = crud.get_user(session=edx_db.session, email="john_doe@example.com") + user = crud.get_user(session=edx_mysql_db.session, email="john_doe@example.com") assert user is None -def test_edx_crud_get_user(edx_db): +def test_edx_crud_get_user(edx_mysql_db): """Test the `get_user` method.""" email = "john_doe@example.com" EdxAuthUserFactory.create_batch(1, email=email) - user = crud.get_user(session=edx_db.session, email=email) + user = crud.get_user(session=edx_mysql_db.session, email=email) assert user.email == email -def test_edx_crud_delete_user_missing(edx_db): +def test_edx_crud_delete_user_missing(edx_mysql_db): """Test the `delete_user` method with missing user in the database.""" with pytest.raises(UserDeleteError, match="User to delete does not exist"): - crud.delete_user(edx_db.session, email="john_doe@example.com") + crud.delete_user(edx_mysql_db.session, email="john_doe@example.com") -def test_edx_crud_delete_user(edx_db): +def test_edx_crud_delete_user(edx_mysql_db): """Test the `delete_user` method.""" email = "john_doe@example.com" EdxAuthUserFactory.create_batch(1, email=email) @@ -194,20 +204,20 @@ def test_edx_crud_delete_user(edx_db): for table_name in related_tables: table = Base.metadata.tables[table_name] - assert edx_db.session.query(table).count() > 0 + assert edx_mysql_db.session.query(table).count() > 0 - crud.delete_user(edx_db.session, email="john_doe@example.com") + crud.delete_user(edx_mysql_db.session, email="john_doe@example.com") # Ensure the parent table is empty - assert edx_db.session.query(AuthUser).count() == 0 + assert edx_mysql_db.session.query(AuthUser).count() == 0 # Ensure the deletion has cascaded properly on children table for table_name in related_tables: table = Base.metadata.tables[table_name] - assert edx_db.session.query(table).count() == 0 + assert edx_mysql_db.session.query(table).count() == 0 -def test_edx_crud_delete_user_protected_table(edx_db): +def test_edx_crud_delete_user_protected_table(edx_mysql_db): """Test the `delete_user` method on a user with entries in a protected tables.""" EdxAuthUserFactory.create_batch( 1, @@ -229,24 +239,24 @@ def test_edx_crud_delete_user_protected_table(edx_db): for table_name in protected_tables: table = Base.metadata.tables[table_name] - assert edx_db.session.query(table).count() > 0 + assert edx_mysql_db.session.query(table).count() > 0 with pytest.raises( UserProtectedDeleteError, match="User is linked to a protected table and cannot be deleted", ): - crud.delete_user(edx_db.session, email="john_doe@example.com") + crud.delete_user(edx_mysql_db.session, email="john_doe@example.com") # Ensure the parent table is empty - assert edx_db.session.query(AuthUser).count() > 0 + assert edx_mysql_db.session.query(AuthUser).count() > 0 # Ensure the deletion has not cascaded on any protected children table for table_name in protected_tables: table = Base.metadata.tables[table_name] - assert edx_db.session.query(table).count() > 0 + assert edx_mysql_db.session.query(table).count() > 0 -def test_edx_crud_has_protected_children(edx_db): +def test_edx_crud_has_protected_children(edx_mysql_db): """Test the `_has_protected_children` method.""" id = 123 EdxAuthtokenTokenFactory.create(user_id=id) @@ -260,8 +270,8 @@ def test_edx_crud_has_protected_children(edx_db): EdxVerifyStudentHistoricalverificationdeadlineFactory.create(history_user_id=id) user_id = 456 - assert not crud._has_protected_children(edx_db.session, user_id) + assert not crud._has_protected_children(edx_mysql_db.session, user_id) EdxCourseActionStateCoursererunstateFactory.create(updated_user_id=user_id) - assert crud._has_protected_children(edx_db.session, user_id) + assert crud._has_protected_children(edx_mysql_db.session, user_id) diff --git a/src/app/mork/tests/edx/test_models.py b/src/app/mork/tests/edx/mysql/test_models.py similarity index 86% rename from src/app/mork/tests/edx/test_models.py rename to src/app/mork/tests/edx/mysql/test_models.py index ff8d6eb..a49bec3 100644 --- a/src/app/mork/tests/edx/test_models.py +++ b/src/app/mork/tests/edx/mysql/test_models.py @@ -1,12 +1,14 @@ """Tests of the edx models.""" -from mork.edx.factories.auth import EdxAuthUserFactory, EdxAuthUserprofileFactory -from mork.edx.factories.certificates import EdxCertificatesGeneratedCertificateFactory -from mork.edx.factories.student import EdxStudentCourseenrollmentFactory -from mork.edx.factories.user import EdxUserApiUserpreferenceFactory +from mork.edx.mysql.factories.auth import EdxAuthUserFactory, EdxAuthUserprofileFactory +from mork.edx.mysql.factories.certificates import ( + EdxCertificatesGeneratedCertificateFactory, +) +from mork.edx.mysql.factories.student import EdxStudentCourseenrollmentFactory +from mork.edx.mysql.factories.user import EdxUserApiUserpreferenceFactory -def test_edx_models_auth_user_safe_dict(edx_db): +def test_edx_models_auth_user_safe_dict(edx_mysql_db): """Test the `safe_dict` method for the AuthUser model.""" edx_auth_user = EdxAuthUserFactory() @@ -25,7 +27,7 @@ def test_edx_models_auth_user_safe_dict(edx_db): } -def test_edx_models_auth_user_profile_safe_dict(edx_db): +def test_edx_models_auth_user_profile_safe_dict(edx_mysql_db): """Test the `safe_dict` method for the AuthUserprofile model.""" edx_auth_user_profile = EdxAuthUserprofileFactory() @@ -50,7 +52,7 @@ def test_edx_models_auth_user_profile_safe_dict(edx_db): } -def test_edx_models_user_api_user_preference_safe_dict(edx_db): +def test_edx_models_user_api_user_preference_safe_dict(edx_mysql_db): """Test the `safe_dict` method for the UserApiUserPreference model.""" edx_user_api_user_preference = EdxUserApiUserpreferenceFactory() @@ -62,7 +64,7 @@ def test_edx_models_user_api_user_preference_safe_dict(edx_db): } -def test_edx_models_student_course_enrollment_safe_dict(edx_db): +def test_edx_models_student_course_enrollment_safe_dict(edx_mysql_db): """Test the `safe_dict` method for the StudentCourseEnrollment model.""" edx_student_course_enrollment = EdxStudentCourseenrollmentFactory() @@ -76,7 +78,7 @@ def test_edx_models_student_course_enrollment_safe_dict(edx_db): } -def test_edx_models_certificates_generated_certificate_safe_dict(edx_db): +def test_edx_models_certificates_generated_certificate_safe_dict(edx_mysql_db): """Test the `safe_dict` method for the StudentCourseEnrollment model.""" edx_certificates_generated_certificate = ( EdxCertificatesGeneratedCertificateFactory() diff --git a/src/app/mork/tests/fixtures/db.py b/src/app/mork/tests/fixtures/db.py index 7c5d980..fe0acca 100644 --- a/src/app/mork/tests/fixtures/db.py +++ b/src/app/mork/tests/fixtures/db.py @@ -1,26 +1,41 @@ """Edx database test fixtures.""" +import mongomock import pytest +from mongoengine import connect, disconnect from sqlalchemy import create_engine from sqlalchemy.orm import Session from mork.conf import settings -from mork.edx.database import OpenEdxDB -from mork.edx.factories.base import engine, session -from mork.edx.models.base import Base as EdxBase +from mork.edx.mysql.database import OpenEdxMySQLDB +from mork.edx.mysql.factories.base import engine, session +from mork.edx.mysql.models.base import Base as EdxBase from mork.models import Base @pytest.fixture -def edx_db(): - """""" - db = OpenEdxDB(engine, session) +def edx_mysql_db(): + """Test MySQL database fixture.""" + db = OpenEdxMySQLDB(engine, session) EdxBase.metadata.create_all(engine) yield db db.session.rollback() EdxBase.metadata.drop_all(engine) +@pytest.fixture +def edx_mongo_db(): + """Test MongDB database fixture.""" + connect( + "mongoenginetest", + host=settings.EDX_MONGO_DB_HOST, + db=settings.EDX_MONGO_DB_NAME, + mongo_client_class=mongomock.MongoClient, + ) + yield + disconnect() + + @pytest.fixture(scope="session") def db_engine(): """Test database engine fixture.""" diff --git a/src/app/pyproject.toml b/src/app/pyproject.toml index 7003944..144590d 100644 --- a/src/app/pyproject.toml +++ b/src/app/pyproject.toml @@ -29,10 +29,13 @@ dependencies = [ "fastapi[standard]==0.115.2", "Jinja2==3.1.4", "jinja2-simple-tags==0.6.1", + "mongoengine==0.29.1", + "mongomock==4.2.0.post1", "psycopg2-binary==2.9.9", "pydantic==2.9.2", "pydantic_settings==2.5.2", "python-datauri==2.2.0", + "pymongo==3.13.0", # pin as it is the last version compatible with MongoDB 3.0 "pymysql==1.1.1", "redis==5.1.1", "sentry-sdk==2.16.0",