From d9c2f840452011407c75ed8f34d0d6d7824c96d9 Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Fri, 14 Jun 2024 14:15:27 +1000 Subject: [PATCH 1/6] Ensure process ADDED events in case kopf collapses ADDED and MODIFIED events together. --- .../src/project/apps/workshops/manager/environments.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/training-portal/src/project/apps/workshops/manager/environments.py b/training-portal/src/project/apps/workshops/manager/environments.py index 8d47b48d..5fa7bb5b 100644 --- a/training-portal/src/project/apps/workshops/manager/environments.py +++ b/training-portal/src/project/apps/workshops/manager/environments.py @@ -230,7 +230,7 @@ def _schedule_session_creation(): f"training.{settings.OPERATOR_API_GROUP}", "v1beta1", "workshopenvironments", - when=lambda event, labels, **_: event["type"] in (None, "MODIFIED") + when=lambda event, labels, **_: event["type"] in (None, "ADDED", "MODIFIED") and labels.get(f"training.{settings.OPERATOR_API_GROUP}/portal.name", "") == settings.PORTAL_NAME, ) @@ -265,6 +265,8 @@ def workshop_environment_event( # workshop specification. if not resource.status.get(f"{settings.OPERATOR_STATUS_KEY}.workshop.uid"): + logger.info("Workshop environment %s not yet ready to be used.", name) + return # Activate the workshop environment, setting status to running if we can. From 321af6571af0cc2b14b9d5faa3b615c2441bcfc4 Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Fri, 14 Jun 2024 14:16:20 +1000 Subject: [PATCH 2/6] Start background tasks if see ADDED event just in case. --- .../src/project/apps/workshops/manager/portal.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/training-portal/src/project/apps/workshops/manager/portal.py b/training-portal/src/project/apps/workshops/manager/portal.py index 7dcd57ce..75051f9c 100644 --- a/training-portal/src/project/apps/workshops/manager/portal.py +++ b/training-portal/src/project/apps/workshops/manager/portal.py @@ -6,6 +6,7 @@ """ import copy +import logging import kopf @@ -39,6 +40,8 @@ from .cleanup import cleanup_old_sessions_and_users, purge_expired_workshop_sessions from .analytics import report_analytics_event +logger = logging.getLogger("educates") + # XXX Disabled here as being handled in parent training_portal_event(). # @resources_lock @@ -377,7 +380,7 @@ def start_reconciliation_task(name): "trainingportals", when=lambda event, name, uid, annotations, **_: name == settings.PORTAL_NAME and uid == settings.PORTAL_UID - and event["type"] in (None, "MODIFIED"), + and event["type"] in (None, "ADDED", "MODIFIED"), ) @resources_lock @transaction.atomic @@ -396,9 +399,13 @@ def training_portal_event(event, name, body, **_): # Event type will be None in case that the process has just started up as # the training portal resource will always exist at that point. For this # case start a background task for performing various reconciliation tasks - # for the training portal. + # for the training portal. Technically we should not ever see an "ADDED" + # event type as the training portal resource should always exist when the + # process starts up. Just in case, we also handle it here. + + if event["type"] in (None, "ADDED"): + logger.info("Starting up training portal background tasks.") - if event["type"] is None: start_reconciliation_task(name).schedule() start_hourly_cleanup_task().schedule() From c7432b57df3cd32d7efcde2887c027a08c0e26fb Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Fri, 14 Jun 2024 14:47:35 +1000 Subject: [PATCH 3/6] Fix forced refresh of workshop environment stuck in STARTING state. --- training-portal/src/project/apps/workshops/admin.py | 6 ++++++ .../src/project/apps/workshops/manager/environments.py | 4 +++- .../src/project/apps/workshops/manager/portal.py | 2 ++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/training-portal/src/project/apps/workshops/admin.py b/training-portal/src/project/apps/workshops/admin.py index 4987b8fe..e4607d67 100644 --- a/training-portal/src/project/apps/workshops/admin.py +++ b/training-portal/src/project/apps/workshops/admin.py @@ -1,3 +1,5 @@ +import logging + from datetime import timedelta from django.contrib import admin @@ -12,6 +14,8 @@ EnvironmentState, ) +logger = logging.getLogger("educates") + class TrainingPortalAdmin(admin.ModelAdmin): list_display = [ @@ -134,6 +138,8 @@ def refresh_environments(self, request, queryset): from .manager.environments import replace_workshop_environment + logger.info("Trigger refresh of workshop environment %s.", environment.name) + replace_workshop_environment(environment) refresh_environments.short_description = "Refresh Environments" diff --git a/training-portal/src/project/apps/workshops/manager/environments.py b/training-portal/src/project/apps/workshops/manager/environments.py index 5fa7bb5b..77e04537 100644 --- a/training-portal/src/project/apps/workshops/manager/environments.py +++ b/training-portal/src/project/apps/workshops/manager/environments.py @@ -356,6 +356,8 @@ def refresh_workshop_environments(training_portal): duration = timezone.now() - environment.created_at if duration.total_seconds() > environment.refresh.total_seconds(): + logger.info("Trigger periodic refresh of workshop environment %s.", environment.name) + replace_workshop_environment(environment) @@ -584,7 +586,7 @@ def replace_workshop_environment(environment): # the existing one as stopping as it clears various values. workshop = { - "name": environment.workshop.name, + "name": environment.workshop_name, "capacity": environment.capacity, "initial": environment.initial, "reserved": environment.reserved, diff --git a/training-portal/src/project/apps/workshops/manager/portal.py b/training-portal/src/project/apps/workshops/manager/portal.py index 75051f9c..fb80093d 100644 --- a/training-portal/src/project/apps/workshops/manager/portal.py +++ b/training-portal/src/project/apps/workshops/manager/portal.py @@ -487,6 +487,8 @@ def workshop_event(event, body, **_): # pylint: disable=unused-argument # Trigger replacement of the workshop environment with a new one. + logger.info("Trigger replacement of workshop environment %s.", environment.name) + replace_workshop_environment(environment) From b8fb2a726c0ea7d025ebc7e4c61b6f1f287c2200 Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Fri, 14 Jun 2024 14:54:01 +1000 Subject: [PATCH 4/6] Adjust logging for replacement of workshop environments. --- .../apps/workshops/manager/environments.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/training-portal/src/project/apps/workshops/manager/environments.py b/training-portal/src/project/apps/workshops/manager/environments.py index 77e04537..aecea2e3 100644 --- a/training-portal/src/project/apps/workshops/manager/environments.py +++ b/training-portal/src/project/apps/workshops/manager/environments.py @@ -356,7 +356,10 @@ def refresh_workshop_environments(training_portal): duration = timezone.now() - environment.created_at if duration.total_seconds() > environment.refresh.total_seconds(): - logger.info("Trigger periodic refresh of workshop environment %s.", environment.name) + logger.info( + "Trigger periodic refresh of workshop environment %s.", + environment.name, + ) replace_workshop_environment(environment) @@ -373,7 +376,9 @@ def delete_workshop_environments(training_portal): for environment in training_portal.stopping_environments(): if environment.active_sessions_count() == 0: - logger.info("Trigger deletion of workshop environment %s.", environment.name) + logger.info( + "Trigger deletion of workshop environment %s.", environment.name + ) delete_workshop_environment(environment).schedule() environment.mark_as_stopped() @@ -616,12 +621,16 @@ def replace_workshop_environment(environment): logger.info( "Stopping workshop environment %s for workshop %s, uid %s, generation %s.", environment.name, - environment.workshop.name, + environment.workshop_name, environment.workshop.uid, environment.workshop.generation, ) else: - logger.info("Stopping workshop environment %s.", environment.name) + logger.info( + "Stopping workshop environment %s for workshop %s.", + environment.name, + environment.workshop_name, + ) update_environment_status(environment.name, "Stopping") environment.mark_as_stopping() From 22b91630d4d5b78870fea7d7a49703fb1060ee28 Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Fri, 14 Jun 2024 14:56:21 +1000 Subject: [PATCH 5/6] Suppress access log messages for static file assets to reduce log noise. --- training-portal/src/httpd.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/training-portal/src/httpd.conf b/training-portal/src/httpd.conf index fea1443b..1f6e907f 100644 --- a/training-portal/src/httpd.conf +++ b/training-portal/src/httpd.conf @@ -14,3 +14,5 @@ SetEnvIf User-Agent "^training-portal-probe/1.0.0" exclude_from_log SetEnvIf Request_URI "^/workshops/session/.*/event/" exclude_from_log SetEnvIf Request_URI "^/workshops/session/.*/schedule/" exclude_from_log + +SetEnvIf Request_URI "^/static/.*" exclude_from_log From 2bda775dad24439b33b9b372632bea16ec6c158a Mon Sep 17 00:00:00 2001 From: Graham Dumpleton Date: Fri, 14 Jun 2024 17:19:19 +1000 Subject: [PATCH 6/6] Add change notes for version 2.7.2. --- project-docs/index.rst | 1 + project-docs/release-notes/version-2.7.2.md | 22 +++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 project-docs/release-notes/version-2.7.2.md diff --git a/project-docs/index.rst b/project-docs/index.rst index 7151cb50..c98857d0 100644 --- a/project-docs/index.rst +++ b/project-docs/index.rst @@ -85,6 +85,7 @@ Educates :maxdepth: 2 :caption: Release Notes: + release-notes/version-2.7.2 release-notes/version-2.7.1 release-notes/version-2.7.0 release-notes/version-2.6.16 diff --git a/project-docs/release-notes/version-2.7.2.md b/project-docs/release-notes/version-2.7.2.md new file mode 100644 index 00000000..ead164ff --- /dev/null +++ b/project-docs/release-notes/version-2.7.2.md @@ -0,0 +1,22 @@ +Version 2.7.2 +============= + +Upcoming Changes +---------------- + +For details on significant changes in future versions, including feature +deprecations and removals which may necessitate updates to existing workshops, +see [Upcoming changes](upcoming-changes). + +Bugs Fixed +---------- + +* A workshop environment could technically get stuck in `STARTING` state as seen + by the training portal if the kopf operator framework coalesced events for + `ADDED` and `MODIFIED` together and only reported a single `ADDED` event. This + is because the training portal was only looking for a `MODIFIED` event. Thus + it could miss when the workshop details were updated in `WorkshopEnvironment` + and so not mark the workshop environment as `RUNNING`. + +* It was not possible through the training portal admin pages to forcibly + refresh a workshop environment that was stuck in `STARTING` state.