feat!: upgrade mysql charset and collation to utf8mb4

changelog.d/20240520_150131_danyal.faheem_upgrade_mysql_utf8mb4.md

@@ -0,0 +1,2 @@
+- [Feature] Upgrade default charset and collation of mysql to utf8mb4 and utf8mb4_unicode_ci respectively (by @Danyal-Faheem)
+    - Add do command to upgrade the charset and collation of tables in mysql.

docs/local.rst

@@ -141,6 +141,21 @@ The default Open edX theme is rather bland, so Tutor makes it easy to switch to
 
 Out of the box, only the default "open-edx" theme is available. We also developed `Indigo, a beautiful, customizable theme <https://github.com/overhangio/indigo>`__ which is easy to install with Tutor.
 
+Changing the mysql charset and collation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Your database's charset and collation might not support specific characters or emojis. To upgrade your charset and collation of all the tables in your database to utf8mb4 and utf8mb4_unicode_ci respectively, run::
+
+    tutor local do change-charset-collation --all-tables --charset=utf8mb4 --collation=utf8mb4_unicode_ci
+
+Alternatively, if you only want to upgrade certain tables or exclude certain tables, you can use the `status` option. To upgrade the `courseware_studentmodule` and `courseware_studentmodulehistory` tables, run::
+
+    tutor local do change-charset-collation --status=include courseware_studentmodule courseware_studentmodulehistory
+
+Tutor performs pattern matching from the start of the table name so you can just enter the name of the app to include/exclude all the tables under that app. To upgrade all the tables in the database except the ones under the student and wiki apps, run::
+
+    tutor local do change-charset-collation --status=exclude student wiki
+
 Running arbitrary ``manage.py`` commands
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

tests/commands/test_jobs.py

@@ -90,3 +90,89 @@ def test_set_theme(self) -> None:
             self.assertIn("lms-job", dc_args)
             self.assertIn("assign_theme('beautiful', 'domain1')", dc_args[-1])
             self.assertIn("assign_theme('beautiful', 'domain2')", dc_args[-1])
+
+    def test_change_charset_collation_all_tables(self) -> None:
+        with temporary_root() as root:
+            self.invoke_in_root(root, ["config", "save"])
+            with patch("tutor.utils.docker_compose") as mock_docker_compose:
+                result = self.invoke_in_root(
+                    root,
+                    [
+                        "local",
+                        "do",
+                        "change-charset-collation",
+                        "--charset",
+                        "utf8mb3",
+                        "--collation",
+                        "utf8mb3_general_ci",
+                        "--all-tables",
+                    ],
+                )
+                dc_args, _dc_kwargs = mock_docker_compose.call_args
+
+            self.assertIsNone(result.exception)
+            self.assertEqual(0, result.exit_code)
+            self.assertIn("lms-job", dc_args)
+            self.assertIn("utf8mb3", dc_args[-1])
+            self.assertIn("utf8mb3_general_ci", dc_args[-1])
+            self.assertNotIn("regexp", dc_args[-1])
+
+    def test_change_charset_collation_include_tables(self) -> None:
+        with temporary_root() as root:
+            self.invoke_in_root(root, ["config", "save"])
+            with patch("tutor.utils.docker_compose") as mock_docker_compose:
+                result = self.invoke_in_root(
+                    root,
+                    [
+                        "local",
+                        "do",
+                        "change-charset-collation",
+                        "--status=include",
+                        "courseware_studentmodule",
+                        "xblock",
+                    ],
+                )
+                dc_args, _dc_kwargs = mock_docker_compose.call_args
+
+            self.assertIsNone(result.exception)
+            self.assertEqual(0, result.exit_code)
+            self.assertIn("lms-job", dc_args)
+            self.assertIn("utf8mb4", dc_args[-1])
+            self.assertIn("utf8mb4_unicode_ci", dc_args[-1])
+            self.assertIn("regexp", dc_args[-1])
+            self.assertIn("courseware_studentmodule", dc_args[-1])
+            self.assertIn("^xblock", dc_args[-1])
+
+    def test_change_charset_collation_exclude_tables(self) -> None:
+        with temporary_root() as root:
+            self.invoke_in_root(root, ["config", "save"])
+            with patch("tutor.utils.docker_compose") as mock_docker_compose:
+                result = self.invoke_in_root(
+                    root,
+                    [
+                        "local",
+                        "do",
+                        "change-charset-collation",
+                        "--status=exclude",
+                        "course",
+                        "wiki",
+                    ],
+                )
+                dc_args, _dc_kwargs = mock_docker_compose.call_args
+
+            self.assertIsNone(result.exception)
+            self.assertEqual(0, result.exit_code)
+            self.assertIn("lms-job", dc_args)
+            self.assertIn("utf8mb4", dc_args[-1])
+            self.assertIn("utf8mb4_unicode_ci", dc_args[-1])
+            self.assertIn("regexp", dc_args[-1])
+            self.assertIn("NOT", dc_args[-1])
+            self.assertIn("^course", dc_args[-1])
+            self.assertIn("^wiki", dc_args[-1])
+
+
+
+
+
+
+

tutor/commands/jobs.py

@@ -12,6 +12,7 @@
 
 from tutor import config as tutor_config
 from tutor import env, fmt, hooks
+from tutor.commands.upgrade.common import get_mysql_change_charset_query
 from tutor.hooks import priorities
 
 
@@ -308,12 +309,72 @@ def sqlshell(args: list[str]) -> t.Iterable[tuple[str, str]]:
     Extra arguments will be passed to the `mysql` command verbatim. For instance, to
     show tables from the "openedx" database, run `do sqlshell openedx -e 'show tables'`.
     """
-    command = "mysql --user={{ MYSQL_ROOT_USERNAME }} --password={{ MYSQL_ROOT_PASSWORD }} --host={{ MYSQL_HOST }} --port={{ MYSQL_PORT }} --default-character-set=utf8mb3"
+    command = "mysql --user={{ MYSQL_ROOT_USERNAME }} --password={{ MYSQL_ROOT_PASSWORD }} --host={{ MYSQL_HOST }} --port={{ MYSQL_PORT }} --default-character-set=utf8mb4"
     if args:
         command += " " + shlex.join(args)  # pylint: disable=protected-access
     yield ("lms", command)
 
 
+
+@click.command(
+    short_help="Upgrade mysql to a specific charset and collation",
+    help=(
+        "Upgrade mysql to a specific charset and collation. You can either upgrade all tables, specify only certain tables to upgrade or specify certain tables to exclude from the upgrade process"
+    ),
+    context_settings={"ignore_unknown_options": True},
+)
+@click.option("--all-tables", is_flag=True, help="change all tables in the openedx database")
+@click.option("-s", "--status", type=click.Choice(['include', 'exclude'], case_sensitive=False), help="Whether to include or exclude certain tables/apps")
+@click.option("--charset", is_flag=False, default="utf8mb4", show_default=True, required=True, type=str, help="The charset to change the tables to")
+@click.option("--collation", is_flag=False, default="utf8mb4_unicode_ci", show_default=True, required=True, type=str, help="The collation to change the tables to")
+@click.argument("tables", nargs=-1)
+@click.pass_context
+def change_charset_collation(
+    context: click.Context,
+    all_tables: bool,
+    status: str,
+    charset: str,
+    collation: str,
+    tables: list[str],
+) -> t.Iterable[tuple[str, str]]:
+    """
+    Do command to upgrade the charset and collation of tables in MySQL
+    
+    Can specify whether to upgrade all tables, or include certain tables/apps or to exclude certain tables/apps
+    """
+    # Make sure user hasn't specified both the options or neither of the options
+    if (status and all_tables) or (not status and not all_tables):
+        fmt.echo_info("Please choose one of the options from: --all-tables or --status")
+        return
+    context = click.get_current_context().obj
+    config = tutor_config.load(context.root)
+
+    if not config["RUN_MYSQL"]:
+        fmt.echo_info(
+            f"You are not running MySQL (RUN_MYSQL=false). It is your "
+            f"responsibility to update your MySQL instance to {charset} charset and {collation} collation."
+        )
+        return
+
+    query_to_append = ""
+    if status:
+        # Make sure user has provided table/apps names as arguments
+        if len(tables) < 1:
+            fmt.echo_info(f"Please provide the names of the tables/apps to {status} for the update process")
+            return
+        include = "NOT" if status == "exclude" else ""
+        table_names = f"^{tables[0]}"
+        for i in range(1, len(tables)):
+            table_names += f"|^{tables[i]}"
+        # We use regexp for pattern matching the names from the start of the tablename
+        query_to_append = f"AND table_name {include} regexp '{table_names}'"
+    click.echo(fmt.title(f"Updating charset and collation of tables in MySQL to {charset} and {collation} respectively."))
+    query = get_mysql_change_charset_query(config["OPENEDX_MYSQL_DATABASE"], charset, collation, query_to_append)
+    mysql_command = "mysql --user={{ MYSQL_ROOT_USERNAME }} --password={{ MYSQL_ROOT_PASSWORD }} --host={{ MYSQL_HOST }} --port={{ MYSQL_PORT }} --database={{ OPENEDX_MYSQL_DATABASE }} --skip-column-names --silent " + shlex.join(["-e", query])
+    yield ("lms", mysql_command)
+    click.echo(fmt.info(f"MySQL charset and collation successfully upgraded"))
+
+
 def add_job_commands(do_command_group: click.Group) -> None:
     """
     This is meant to be called with the `local/dev/k8s do` group commands, to add the
@@ -389,6 +450,7 @@ def do_callback(service_commands: t.Iterable[tuple[str, str]]) -> None:
 
 hooks.Filters.CLI_DO_COMMANDS.add_items(
     [
+        change_charset_collation,
         createuser,
         importdemocourse,
         importdemolibraries,

tutor/commands/upgrade/common.py

@@ -66,3 +66,55 @@ def get_mongo_upgrade_parameters(
     mv '/openedx/data/ora2/SET-ME-PLEASE (ex. bucket-name)' /openedx/data/ora2/openedxuploads
 fi
 """
+
+def get_mysql_change_charset_query(
+    database: str,
+    charset: str,
+    collation: str,
+    query_to_append: str,
+) -> None:
+    """
+    Helper function to generate the mysql query to upgrade the charset and collation of tables
+    
+    Utilized in the `tutor local do change-charset-collation` command
+    """
+    return f"""
+            DROP PROCEDURE IF EXISTS UpdateTables;
+            DELIMITER $$
+
+            CREATE PROCEDURE UpdateTables()
+            BEGIN
+
+                DECLARE done INT DEFAULT FALSE;
+                DECLARE _table_name VARCHAR(255);
+                DECLARE cur CURSOR FOR
+                        SELECT table_name FROM information_schema.tables
+                        WHERE table_schema = '{database}' AND table_type = 'BASE TABLE' {query_to_append};
+                DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = TRUE;
+                Select "The following tables will have their charset and collation updated:";
+                OPEN cur;
+                tables_loop: LOOP
+                    FETCH cur INTO _table_name;
+
+                    IF done THEN
+                    LEAVE tables_loop;
+                    END IF;
+
+                    SET FOREIGN_KEY_CHECKS = 0;
+                    Select _table_name;
+                    SET @statement = CONCAT('ALTER TABLE ', _table_name, ' CONVERT TO CHARACTER SET {charset} COLLATE {collation};'); 
+                    PREPARE query FROM @statement; 
+                    EXECUTE query; 
+                    DEALLOCATE PREPARE query;   
+                    SET FOREIGN_KEY_CHECKS = 1;
+                        
+                END LOOP;
+                CLOSE cur;
+
+            END$$
+
+            DELIMITER ;
+            use {database};
+            ALTER DATABASE {database} CHARACTER SET {charset} COLLATE {collation};
+            CALL UpdateTables();
+            """

tutor/templates/apps/openedx/config/partials/auth.yml

@@ -18,7 +18,7 @@ DATABASES:
     OPTIONS:
       init_command: "SET sql_mode='STRICT_TRANS_TABLES'"
       {%- if RUN_MYSQL %}
-      charset: "utf8mb3"
+      charset: "utf8mb4"
       {%- endif %}
 EMAIL_HOST_USER: "{{ SMTP_USERNAME }}"
 EMAIL_HOST_PASSWORD: "{{ SMTP_PASSWORD }}"

tutor/templates/k8s/deployments.yml

@@ -394,8 +394,8 @@ spec:
           image: {{ DOCKER_IMAGE_MYSQL }}
           args:
             - "mysqld"
-            - "--character-set-server=utf8mb3"
-            - "--collation-server=utf8mb3_general_ci"
+            - "--character-set-server=utf8mb4"
+            - "--collation-server=utf8mb4_unicode_ci"
             - "--binlog-expire-logs-seconds=259200"
           env:
             - name: MYSQL_ROOT_PASSWORD

tutor/templates/local/docker-compose.yml

@@ -42,8 +42,8 @@ services:
     image: {{ DOCKER_IMAGE_MYSQL }}
     command: >
       mysqld
-      --character-set-server=utf8mb3
-      --collation-server=utf8mb3_general_ci
+      --character-set-server=utf8mb4
+      --collation-server=utf8mb4_unicode_ci
       --binlog-expire-logs-seconds=259200
     restart: unless-stopped
     user: "999:999"