Merge pull request #147 from dwreeves/dev_postgres

Db migration and CLI bugfix

docs/docs/cloud/heroku_deployment.md

@@ -196,3 +196,49 @@ git remote -v
 ```
 git push staging master
 ```
+
+## Updating the database schema
+
+We do not use Alembic or any other tools to handle database migrations. Our database is relatively simple and does not store information that cannot be trivially rewritten, so we can migrate to a new database schema by simply creating the new database locally and completely overwriting the database in production.
+
+Here are the steps to do that:
+
+1. Make your changes to the database schema.
+
+```shell
+flask create-db --overwrite
+flask init-db
+```
+
+2. Reset the database. You'll be asked if you're sure you want to reset, and go through that prompt.
+
+```shell
+heroku pg:reset -a crwa-flagging
+```
+
+3. Get a list of add-ons for the Heroku app. You'll need it for the final step.
+
+```shell
+heroku addons -a crwa-flagging
+```
+
+???+ success
+    You should see something like this:
+
+    ```
+    Add-on                                        Plan       Price  State  
+    ────────────────────────────────────────────  ─────────  ─────  ───────
+    heroku-postgresql (postgresql-ukulele-12345)  hobby-dev  free   created
+     └─ as DATABASE
+
+    scheduler (scheduler-banjo-67890)             standard   free   created
+     └─ as SCHEDULER
+
+    The table above shows add-ons and the attachments to the current app (crwa-flagging) or other apps.
+    ```
+
+4. Take the add-on name for the DATABASE in parentheses (in the above case, `postgresql-ukulele-12345`) as the target for the Postgres push command:
+
+```shell
+heroku pg:push flagging postgresql-ukulele-12345 -a crwa-flagging
+```

flagging_site/app.py

@@ -162,15 +162,21 @@ def register_commands(app: Flask):
     """
 
     @app.cli.command('create-db')
-    def create_db_command():
+    @click.option('--overwrite/--no-overwrite',
+                  default=False,
+                  is_flag=True,
+                  show_default=True,
+                  help='If true, overwrite the database if it exists.')
+    def create_db_command(overwrite: bool = False):
         """Create database (after verifying that it isn't already there)."""
         from .data.database import create_db
-        create_db()
+        create_db(overwrite=overwrite)
 
     @app.cli.command('init-db')
     @click.option('--pop/--no-pop',
                   default=True,
                   is_flag=True,
+                  show_default=True,
                   help='If true, then do a db update when initializing the db.')
     @click.pass_context
     def init_db_command(ctx: click.Context, pop: bool):
@@ -179,35 +185,28 @@ def init_db_command(ctx: click.Context, pop: bool):
         init_db()
         click.echo('Initialized the database.')
         if pop:
-            # Update the database if `pop` is True
             ctx.invoke(update_db_command)
 
     @app.cli.command('update-db')
     def update_db_command():
         """Update the database with the latest live data."""
         from .data.database import update_database
-        try:
-            update_database()
-            click.echo('Updated the database.')
-            updated = True
-        except Exception as e:
-            click.echo("Note: while updating database, something didn't "
-                       f'work: {e}')
-            updated = False
-        return updated
+        update_database()
+        click.echo('Updated the database successfully.')
 
     @app.cli.command('update-website')
     @click.pass_context
     def update_website_command(ctx: click.Context):
         """Updates the database, then Tweets a message."""
-        from .data.live_website_options import is_boating_season
+        from .data.live_website_options import LiveWebsiteOptions
+
+        # Update the database
+        ctx.invoke(update_db_command)
 
-        updated = ctx.invoke(update_db_command)
         # If the model updated and it's boating season, send a tweet.
         # Otherwise, do nothing.
         if (
-                updated
-                and is_boating_season()
+                LiveWebsiteOptions.is_boating_season()
                 and current_app.config['SEND_TWEETS']
         ):
             from .twitter import tweet_current_status

flagging_site/config.py

@@ -187,7 +187,7 @@ def __init__(self):
         # to be consistent with the `DATABASE_URL` below:
         postgres_url_schema = re.compile('''
             ^
-            postgres://
+            postgres(?:ql)?://
             ([^\s:@/]+) # Username
             :([^\s:@/]+) # Password
             @([^\s:@/]+) # Host
@@ -227,7 +227,7 @@ class DevelopmentConfig(Config):
     the vault hasn't been loaded but doesn't prevent the website from loading
     just because the vault is not open.
     """
-    SQLALCHEMY_ECHO: bool = True
+    SQLALCHEMY_ECHO: bool = strtobool(os.getenv('SQLALCHEMY_ECHO') or 'true')
     VAULT_OPTIONAL: bool = True
     DEBUG: bool = True
     TESTING: bool = True

flagging_site/data/database.py

@@ -68,7 +68,7 @@ def execute_sql_from_file(file_name: str) -> Optional[pd.DataFrame]:
         return execute_sql(f.read().decode('utf8'))
 
 
-def create_db() -> bool:
+def create_db(overwrite: bool = False) -> bool:
     """If the database defined by `POSTGRES_DBNAME` doesn't exist, create it
     and return True, otherwise do nothing and return False. By default, the
     config variable `POSTGRES_DBNAME` is set to "flagging".
@@ -86,6 +86,14 @@ def create_db() -> bool:
     database = current_app.config['POSTGRES_DBNAME']
     cursor = conn.cursor()
 
+    if overwrite:
+        cursor.execute("SELECT bool_or(datname = 'flagging') FROM pg_database;")
+        exists = cursor.fetchall()[0][0]
+        if exists:
+            cursor.execute('COMMIT;')
+            cursor.execute(f'DROP DATABASE {database};')
+            click.echo(f'Dropped database {database!r}.')
+
     try:
         cursor.execute('COMMIT;')
         cursor.execute(f'CREATE DATABASE {database};')

flagging_site/templates/admin/logout.html

@@ -1,7 +1,6 @@
 {% extends "admin/base.html" %}
 {% block body %}
-    <p>Logged out.</p>
-    <p>Redirecting...</p>
+    <p>Logged out. Redirecting...</p>
 {% endblock %}
 {% block tail %}
     {{ super() }}
@@ -10,7 +9,7 @@
             $.ajax({
                async: false,
                type: "GET",
-               url: "/admin",
+               url: "{{ url_for('admin.index') }}",
                username: "logout"
             })
             .done(function(){