Fix ICS Calendar Timezone and Parsing Issues

.github/workflows/shared-build-deploy.yaml

@@ -9,7 +9,12 @@ on: push
 jobs:
   backend-check:
     name: "Backend Check"
-    uses: pennlabs/shared-actions/.github/workflows/[email protected]
+<<<<<<< HEAD
+    uses: pennlabs/shared-actions/.github/workflows/django.yaml@8785a7d7b9158d8d5705a0202f5695db2c0beb97
+
+=======
+    uses: pennlabs/shared-actions/.github/workflows/django-check.yaml@2453452f049da3144fd225e2782d4e8816e41b62
+>>>>>>> Update shared-build-deploy.yaml
     with:
       projectName: PennCourses
       path: backend

backend/Dockerfile

@@ -1,4 +1,4 @@
-FROM pennlabs/django-base:f0f05216db7c23c1dbb5b95c3bc9e8a2603bf2fd
+FROM pennlabs/django-base:9c4f31bf1af44219d0f9019271a0033a222291c2-3.10.1
 
 LABEL maintainer="Penn Labs"
 

backend/PennCourses/asgi.py

@@ -0,0 +1,8 @@
+import os
+
+from django.core.asgi import get_asgi_application
+
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "PennCourses.settings.production")
+
+application = get_asgi_application()

backend/Pipfile

@@ -4,7 +4,6 @@ verify_ssl = true
 name = "pypi"
 
 [dev-packages]
-codecov = "*"
 coverage = "*"
 unittest-xml-reporting = "*"
 flake8 = "*"
@@ -22,6 +21,8 @@ django = ">=3.2"
 psycopg2 = "*"
 requests = "*"
 twilio = "*"
+channels = "<3"
+channels-redis = "*"
 phonenumbers = "*"
 celery = "<5"
 redis = ">=3.4.1"
@@ -60,6 +61,9 @@ scikit-learn = "*"
 pandas = "*"
 python-dateutil = "*"
 docutils = "*"
+ics = "*"
+psycopg2-binary = "*"
+drf-nested-routers = "*"
 
 [requires]
 python_full_version = "3.10"

backend/README.md

@@ -4,9 +4,12 @@ Welcome to the Penn Courses Backend (PCX)!
 
 ### Prerequisites
 
--   Python 3.10 ([`pyenv`](https://github.com/pyenv/pyenv) is recommended)
--   [`pipenv`](https://pipenv.pypa.io/en/latest/)
--   [`docker` and `docker-compose`](https://docs.docker.com/get-docker/)
+- Python 3.10 ([`pyenv`](https://github.com/pyenv/pyenv) is recommended)
+- [`pipenv`](https://pipenv.pypa.io/en/latest/)
+- [`docker` and `docker-compose`](https://docs.docker.com/get-docker/)
+- Python 3.10 ([`pyenv`](https://github.com/pyenv/pyenv) is recommended)
+- [`pipenv`](https://pipenv.pypa.io/en/latest/)
+- [`docker` and `docker-compose`](https://docs.docker.com/get-docker/)
 
 ### Environment Variables
 
@@ -17,69 +20,78 @@ pasted in this public README). If you are not in Penn Labs, see the "Loading Cou
 NOTE: when using `pipenv`, environment variables are only refreshed when you exit your shell and rerun `pipenv shell` (this is a common source of confusing behavior, so it's good to know about).
 
 ### Setting Up the Backend
+
 1. `cd backend`
 2. Compiling postgres (`psycopg2`)
-    - **Mac**
-        > :warning: NOTE: If you are having trouble installing packages (e.g. on Apple silicon), you can use the workaround described in the next section ([Trouble Installing Packages (Apple Silicon)](#trouble-installing-packages-apple-silicon)) to run the server in a docker container. 
-
-        > :warning: NOTE: If your computer runs on Apple silicon and you use Rosetta to run Python as an x86 program, use `arch -x86_64 brew <rest of command>` for all `brew` commands.
-        1. `brew install postgresql`
-        2. `brew install openssl`
-        3. `brew unlink openssl && brew link openssl --force`
-        4. Follow the instructions printed by the previous command to add openssl to your PATH and export flags for compilers, e.g.:
-            - `echo 'export PATH="/usr/local/opt/openssl@3/bin:$PATH"' >> ~/.zshrc`
-            - `export LDFLAGS="-L/usr/local/opt/openssl@3/lib"`
-            - `export CPPFLAGS="-I/usr/local/opt/openssl@3/include"`
-    - **Windows/Linux**
-        1. `apt-get install gcc python3-dev libpq-dev`
+
+   - **Mac**
+
+     > :warning: NOTE: If you are having trouble installing packages (e.g. on Apple silicon), you can use the workaround described in the next section ([Trouble Installing Packages (Apple Silicon)](#trouble-installing-packages-apple-silicon)) to run the server in a docker container.
+
+     > :warning: NOTE: If your computer runs on Apple silicon and you use Rosetta to run Python as an x86 program, use `arch -x86_64 brew <rest of command>` for all `brew` commands.
+
+     1. `brew install postgresql`
+     2. `brew install openssl`
+     3. `brew unlink openssl && brew link openssl --force`
+     4. Follow the instructions printed by the previous command to add openssl to your PATH and export flags for compilers, e.g.:
+        - `echo 'export PATH="/usr/local/opt/openssl@3/bin:$PATH"' >> ~/.zshrc`
+        - `export LDFLAGS="-L/usr/local/opt/openssl@3/lib"`
+        - `export CPPFLAGS="-I/usr/local/opt/openssl@3/include"`
+
+   - Windows (WSL) or Linux:
+     - `apt-get install gcc python3-dev libpq-dev`
 
 3. Running Docker
-    1. Open a new terminal window (also in the `backend` directory) and run `docker-compose up` 
-        > :warning: Depending on your system configuration, you may have to start `docker` manually. If this is the case (ie, if you cannot get `docker-compose up` to work due to a docker connection error) try this:
-        >    - (linux) `[sudo] systemctl start docker`
-        >    - (WSL) `[sudo] service docker start`
+   1. Open a new terminal window (also in the `backend` directory) and run `docker-compose up`
+      > :warning: Depending on your system configuration, you may have to start `docker` manually. If this is the case (ie, if you cannot get `docker-compose up` to work due to a docker connection error) try this:
+      >
+      > - (linux) `[sudo] systemctl start docker`
+      > - (WSL) `[sudo] service docker start`
 
 > ⚠️ **NOTE** If you are having trouble installing packages or with installing pipenv (e.g., on Apple silicon), see [Trouble Installing Packages (Apple Silicon)](#trouble-installing-packages-apple-silicon). Return to finish these instructions from step 4.
 
 4. Setting up your Penn Courses development environment
-    1. `pipenv install --dev`
-    2. `pipenv shell`
-    3. `python manage.py migrate`
+
+   1. `pipenv install --dev`
+   2. `pipenv shell`
+   3. `python manage.py migrate`
 
 5. Loading test data (if you are a member of Penn Labs). If you are not a member of Penn Labs, you can skip this section and load in course data from the registrar, as explained below.
 
-    - To get going quickly with a local database loaded with lots of test data,
-      you can download this [pcx_test.sql](https://files.slack.com/files-pri/T4EM1119V-F04FPSTNF46/download/pcx_test_12_2022.sql)
-      SQL dump file.
-    - Clear the existing contents of your local database with `psql template1 -c 'drop database postgres;' -h localhost -U penn-courses` (the password is `postgres`)
-    - Create a new database with `psql template1 -c 'create database postgres with owner "penn-courses";' -h localhost -U penn-courses` (same password).
-    - Finally, run `psql -h localhost -d postgres -U penn-courses -f pcx_test.sql` (replacing `pcx_test.sql` with the full path to that file on your computer) to load
-      the contents of the test database (this might take a while).
-    - For accessing the Django admin site, the admin username is `admin` and the password is `admin` if you use this test db.
-    - Run `python manage.py migrate` to apply migrations
+   - To get going quickly with a local database loaded with lots of test data,
+     you can download this [pcx_test.sql](https://files.slack.com/files-pri/T4EM1119V-F04FPSTNF46/download/pcx_test_12_2022.sql)
+     SQL dump file.
+   - Clear the existing contents of your local database with `psql template1 -c 'drop database postgres;' -h localhost -U penn-courses` (the password is `postgres`)
+   - Create a new database with `psql template1 -c 'create database postgres with owner "penn-courses";' -h localhost -U penn-courses` (same password).
+   - Finally, run `psql -h localhost -d postgres -U penn-courses -f pcx_test.sql` (replacing `pcx_test.sql` with the full path to that file on your computer) to load
+     the contents of the test database (this might take a while).
+   - For accessing the Django admin site, the admin username is `admin` and the password is `admin` if you use this test db.
+   - Run `python manage.py migrate` to apply migrations
 
 6. Running the backend
 
-    - Run the backend in development mode with the command `python manage.py runserver`. This will start the server at port `8000`.
-    - Once the server is running, you can access the admin console at `localhost:8000/admin`, browse auto-generated API documentation from the code on your branch at `localhost:8000/api/documentation`, or use any of the other routes supported by this backend (comprehensively described by the API documentation), usually of the form `localhost:8000/api/...`
-    - Note: if you don't need documentation specific to your branch, it is usually more convenient to browse the API docs at [penncoursereview.com/api/documentation](https://penncoursereview.com/api/documentation)
-    - With the backend server running, you can also run the frontend for any of our PCX products by following the instructions in the `frontend` README.
-    - Note: If you have not loaded the test data from the previous step (Step 4), ensure that you have created a local user named "Penn-Courses" with the password "postgres" in your PostgreSQL. To add the user, navigate to your pgAdmin, and follow the path of Object -> Create -> Login/Group Role and create the appropriate user.
+   - Run the backend in development mode with the command `python manage.py runserver`. This will start the server at port `8000`.
+   - Once the server is running, you can access the admin console at `localhost:8000/admin`, browse auto-generated API documentation from the code on your branch at `localhost:8000/api/documentation`, or use any of the other routes supported by this backend (comprehensively described by the API documentation), usually of the form `localhost:8000/api/...`
+   - Note: if you don't need documentation specific to your branch, it is usually more convenient to browse the API docs at [penncoursereview.com/api/documentation](https://penncoursereview.com/api/documentation)
+   - With the backend server running, you can also run the frontend for any of our PCX products by following the instructions in the `frontend` README.
+   - Note: If you have not loaded the test data from the previous step (Step 4), ensure that you have created a local user named "Penn-Courses" with the password "postgres" in your PostgreSQL. To add the user, navigate to your pgAdmin, and follow the path of Object -> Create -> Login/Group Role and create the appropriate user.
+   - Note: If you ever encounter a `pg_hba.conf` entry error, open the `~/var/lib/postgresql/data/pg_hba.conf` file in your docker container and append the line `host  all  all 0.0.0.0/0 md5` into the file.
 
 7. Running tests
-    - Run `python manage.py test` to run our test suite.
-    - To run a specific test, you can use the format `python manage.py test tests.review.test_api.OneReviewTestCase.test_course` (also note that in this example, you can use any prefix of that path to run a larger set of tests).
+8. Running tests
+   - Run `python manage.py test` to run our test suite.
+   - To run a specific test, you can use the format `python manage.py test tests.review.test_api.OneReviewTestCase.test_course` (also note that in this example, you can use any prefix of that path to run a larger set of tests).
 
 ### Trouble Installing Packages (Apple Silicon)
 
 1. Always run `docker-compose --profile=dev up` instead of just `docker-compose up`.
-    - To alias this command, run `echo "alias courses-compose='cd "$PWD"; docker-compose up'" >> ~/.zshrc; source ~/.zshrc` (replacing `~/.zshrc` with `~/.bashrc` or whatever configuration file your shell uses, if you don't use zsh).
-    - This will spin up a container from which you can run the server (with all required packages preinstalled). 
-3. In a separate terminal (from any directory), run `docker exec -it backend-development-1 /bin/bash` to open a shell in the container (if this says "no such container", run `docker container ls` and use the name of whatever container most closely matches the `backend_development` image). Just like exiting a Pipenv shell, you can exit the container by pressing `Ctrl+D` (which sends the "end of transmission" / EOF character). 
-    - You might want to add an alias for this command so it is easier to run (e.g. `echo 'alias courses-backend="docker exec -it backend_development_1 /bin/bash"' >> ~/.zshrc && source ~/.zshrc`). Then you can just run `courses-backend` from any directory to connect to the Docker container from which you will run the server (assuming `courses-compose` is already running in another terminal).
-4. Once you have a shell open in the container, you can continue running the rest of the commands in the [Setting Up the Backend](#setting-up-the-backend) section of this README (except you can skip `pipenv install --dev` since that has already been done for you). 
-    1. Remember to run `pipenv shell` (to open a [Pipenv] shell inside of a [docker container] shell inside of your computer's shell!). Note that the `/backend` directory inside the container is automatically synced with the `backend` directory on your host machine (from which you ran `docker-compose --profile=dev up`). 
-5. There's just one last complication. Due to some annoying details of Docker networking, you have to expose the server on IP address `0.0.0.0` inside the container, rather than `127.0.0.1` or `localhost` as is default (otherwise the server won't be accessible from outside of the container). To do this, instead of running `python manage.py runserver`, run `python manage.py runserver 0.0.0.0:8000`. In `Dockerfile.dev`, we automatically alias the command `runserver` to the latter, so in the container shell (in `/backend`, as is default) you can simply run the command `runserver` (no `python manage.py` necessary).
+   - To alias this command, run `echo "alias courses-compose='cd "$PWD"; docker-compose up'" >> ~/.zshrc; source ~/.zshrc` (replacing `~/.zshrc` with `~/.bashrc` or whatever configuration file your shell uses, if you don't use zsh).
+   - This will spin up a container from which you can run the server (with all required packages preinstalled).
+2. In a separate terminal (from any directory), run `docker exec -it backend-development-1 /bin/bash` to open a shell in the container (if this says "no such container", run `docker container ls` and use the name of whatever container most closely matches the `backend_development` image). Just like exiting a Pipenv shell, you can exit the container by pressing `Ctrl+D` (which sends the "end of transmission" / EOF character).
+   - You might want to add an alias for this command so it is easier to run (e.g. `echo 'alias courses-backend="docker exec -it backend_development_1 /bin/bash"' >> ~/.zshrc && source ~/.zshrc`). Then you can just run `courses-backend` from any directory to connect to the Docker container from which you will run the server (assuming `courses-compose` is already running in another terminal).
+3. Once you have a shell open in the container, you can continue running the rest of the commands in this README (except you can skip `pipenv install --dev` since that has already been done for you).
+   1. Remember to run `pipenv shell` (to open a [Pipenv] shell inside of a [docker container] shell inside of your computer's shell!). Note that the `/backend` directory inside the container is automatically synced with the `backend` directory on your host machine (from which you ran `docker-compose --profile=dev up`).
+4. There's just one last complication. Due to some annoying details of Docker networking, you have to expose the server on IP address `0.0.0.0` inside the container, rather than `127.0.0.1` or `localhost` as is default (otherwise the server won't be accessible from outside of the container). To do this, instead of running `python manage.py runserver`, run `python manage.py runserver 0.0.0.0:8000`. In `Dockerfile.dev`, we automatically alias the command `runserver` to the latter, so in the container shell (in `/backend`, as is default) you can simply run the command `runserver` (no `python manage.py` necessary).
 
 ## Loading Course Data on Demand
 

backend/alert/management/commands/recomputestats.py

@@ -53,9 +53,9 @@ def recompute_has_reviews():
     with connection.cursor() as cursor:
         cursor.execute(
             """
-        UPDATE "courses_section" U0
+        UPDATE "courses_section" AS U0
         SET "has_reviews" = CASE WHEN
-            EXISTS (SELECT id FROM "review_review" U1
+            EXISTS (SELECT id FROM "review_review" AS U1
                 WHERE U0."id" = U1."section_id")
             THEN true ELSE false
         END
@@ -67,9 +67,9 @@ def recompute_has_status_updates():
     with connection.cursor() as cursor:
         cursor.execute(
             """
-        UPDATE "courses_section" U0
+        UPDATE "courses_section" AS U0
         SET "has_status_updates" = CASE WHEN
-            EXISTS (SELECT id FROM "courses_statusupdate" U1
+            EXISTS (SELECT id FROM "courses_statusupdate" AS U1
                 WHERE U0."id" = U1."section_id")
             THEN true ELSE false
         END

backend/courses/course_similarity/heuristics.py

@@ -1,7 +1,5 @@
 import re
 
-from jellyfish import levenshtein_distance
-
 
 def title_rejection_heuristics(title_a, title_b):
     """
@@ -87,10 +85,3 @@ def description_rejection_heuristics(desc_a, desc_b):
             return True
 
     return False
-
-
-def lev_divided_by_avg_length(a, b):
-    """
-    Compute levenshtein distance between 2 strings and then divide by avg length.
-    """
-    return 2 * levenshtein_distance(a, b) / (len(a) + len(b))

backend/courses/management/commands/merge_topics.py

@@ -8,7 +8,6 @@
 
 from courses.course_similarity.heuristics import (
     description_rejection_heuristics,
-    lev_divided_by_avg_length,
     title_rejection_heuristics,
 )
 from courses.management.commands.load_crosswalk import load_crosswalk
@@ -54,16 +53,10 @@ def same_course(course_a, course_b):
 
 def similar_courses(course_a, course_b):
     title_a, title_b = course_a.title.strip().lower(), course_b.title.strip().lower()
-    if (
-        not title_rejection_heuristics(title_a, title_b)
-        and lev_divided_by_avg_length(title_a, title_b) < 0.2
-    ):
+    if not title_rejection_heuristics(title_a, title_b):
         return True
     desc_a, desc_b = course_a.description.strip().lower(), course_b.description.strip().lower()
-    if (
-        not description_rejection_heuristics(desc_a, desc_b)
-        and lev_divided_by_avg_length(desc_a, desc_b) < 0.2
-    ):
+    if not description_rejection_heuristics(desc_a, desc_b):
         return True
     return False
 

backend/courses/migrations/0054_userprofile_uuid_secret_and_more.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.1.5 on 2023-01-15 16:39
+
+import uuid
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("courses", "0053_alter_ngssrestriction_code"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="ngssrestriction",
+            name="inclusive",
+            field=models.BooleanField(
+                help_text='\nWhether this is an include or exclude restriction. Corresponds to the `incl_excl_ind`\nresponse field. `True` if include (ie, `incl_excl_ind` is "I") and `False`\nif exclude ("E").\n'
+            ),
+        ),
+    ]