Merge branch 'beta'

program/pretalx_sync.py

@@ -1,9 +1,9 @@
 from collections import ChainMap
+from collections.abc import Collection
 from typing import MutableMapping, Any
 
 from program import models
 from program import pretalx
-from collections.abc import Collection
 
 
 class PretalxSync:
@@ -13,11 +13,15 @@ def __init__(self, pretalx_client: pretalx.PretalxClient) -> None:
         self._talks_order = 0
         self._workshops_order = 0
 
-    def full_sync(self):
-        all_speakers = self.sync_speakers()
-        self.sync_submissions(all_speakers)
-
     def update_speakers(self, speakers: Collection[models.Speaker]) -> None:
+        """
+        Fetches updated data for the given list of speakers and updates the database
+        models.
+
+        Note: this method currently performs 1 API call per speaker, therefore
+        updating many speakers might take a long time and might cause the HTTP
+        request to time-out.
+        """
         for speaker in speakers:
             speaker_data = self.client.get_speaker(
                 code=speaker.pretalx_code,
@@ -31,6 +35,14 @@ def update_speakers(self, speakers: Collection[models.Speaker]) -> None:
         )
 
     def update_talks(self, talks: Collection[models.Talk]) -> None:
+        """
+        Fetches updated data for the given list of talks and updates the database
+        models.
+
+        Note: this method currently performs 1 API call per talk, therefore
+        updating many talks might take a long time and might cause the HTTP
+        request to time-out.
+        """
         for talk in talks:
             submission_data = self.client.get_submission(
                 code=talk.pretalx_code,
@@ -44,6 +56,14 @@ def update_talks(self, talks: Collection[models.Talk]) -> None:
         )
 
     def update_workshops(self, workshops: Collection[models.Workshop]) -> None:
+        """
+        Fetches updated data for the given list of workshops and updates the database
+        models.
+
+        Note: this method currently performs 1 API call per workshop, therefore
+        updating many workshops might take a long time and might cause the HTTP
+        request to time-out.
+        """
         for workshop in workshops:
             submission_data = self.client.get_submission(
                 code=workshop.pretalx_code,
@@ -56,11 +76,50 @@ def update_workshops(self, workshops: Collection[models.Workshop]) -> None:
             fields=models.Workshop.PRETALX_FIELDS,
         )
 
-    def sync_speakers(self) -> dict[str, models.Speaker]:
+    def full_sync(self):
+        """
+        Performs full synchronization of speakers, talks and workshops. New entries
+        are created when required, existing entries will be updated.
+
+        This operation currently creates or updates only speakers with at least
+        one confirmed submission.
+        """
+        submissions = self._fetch_submissions()
+        confirmed_speaker_codes = self._extract_confirmed_speaker_codes(submissions)
+        speakers = self._fetch_speakers_by_code(confirmed_speaker_codes)
+
+        all_speakers = self._sync_speakers(speakers)
+        self._sync_submissions(submissions, all_speakers)
+
+    def _fetch_submissions(self) -> list[dict[str, Any]]:
+        # `list_submissions` returns an iterable that can be iterated only once.
+        # We want to use the list of confirmed submissions to filter speakers to update/create,
+        # therefore we need to convert it to a list
+        submissions = self.client.list_submissions(
+            questions=["all"], states=[pretalx.SubmissionState.CONFIRMED]
+        )
+        submissions = list(submissions)
+        return submissions
+
+    def _extract_confirmed_speaker_codes(
+        self, submissions: list[dict[str, Any]]
+    ) -> set[str]:
+        result = set()
+        for submission in submissions:
+            for speaker in submission["speakers"]:
+                result.add(speaker["code"])
+        return result
+
+    def _fetch_speakers_by_code(self, speaker_codes) -> list[dict[str, Any]]:
         speakers = self.client.list_speakers(
             questions=["all"],
         )
+        speakers = filter(
+            lambda speaker_data: speaker_data["code"] in speaker_codes, speakers
+        )
+        return list(speakers)
 
+    def _sync_speakers(self, speakers) -> dict[str, models.Speaker]:
         existing_speakers = models.Speaker.objects.filter(
             pretalx_code__isnull=False,
         ).in_bulk(field_name="pretalx_code")
@@ -88,10 +147,9 @@ def sync_speakers(self) -> dict[str, models.Speaker]:
 
         return dict(all_speakers)
 
-    def sync_submissions(self, speakers: dict[str, models.Speaker]):
-        submissions = self.client.list_submissions(
-            questions=["all"], states=[pretalx.SubmissionState.CONFIRMED]
-        )
+    def _sync_submissions(
+        self, submissions: list[dict[str, Any]], speakers: dict[str, models.Speaker]
+    ):
         talks: list[dict[str, Any]] = []
         workshops: list[dict[str, Any]] = []
         for submission in submissions:
@@ -103,10 +161,10 @@ def sync_submissions(self, speakers: dict[str, models.Speaker]):
             else:
                 workshops.append(submission)
 
-        self.sync_talks(talks, speakers)
-        self.sync_workshops(workshops, speakers)
+        self._sync_talks(talks, speakers)
+        self._sync_workshops(workshops, speakers)
 
-    def sync_talks(
+    def _sync_talks(
         self, submissions: list[dict[str, Any]], speakers: dict[str, models.Speaker]
     ) -> None:
         existing_talks = models.Talk.objects.filter(
@@ -121,7 +179,7 @@ def sync_talks(
         )
 
         for submission_data in submissions:
-            self.update_talk(all_talks, submission_data)
+            self._update_talk(all_talks, submission_data)
 
         models.Talk.objects.bulk_update(
             objs=existing_talks.values(),
@@ -137,7 +195,7 @@ def sync_talks(
                 ]
             )
 
-    def sync_workshops(
+    def _sync_workshops(
         self, submissions: list[dict[str, Any]], speakers: dict[str, models.Speaker]
     ) -> None:
         existing_workshops = models.Workshop.objects.filter(
@@ -152,7 +210,7 @@ def sync_workshops(
         )
 
         for submission_data in submissions:
-            self.update_workshop(all_workshops, submission_data)
+            self._update_workshop(all_workshops, submission_data)
 
         models.Workshop.objects.bulk_update(
             objs=existing_workshops.values(),
@@ -168,7 +226,7 @@ def sync_workshops(
                 ]
             )
 
-    def update_talk(
+    def _update_talk(
         self,
         all_talks: MutableMapping[str, models.Talk],
         submission_data: dict[str, Any],
@@ -188,7 +246,7 @@ def update_talk(
 
         talk.update_from_pretalx(submission_data)
 
-    def update_workshop(
+    def _update_workshop(
         self,
         all_workshops: MutableMapping[str, models.Workshop],
         submission_data: dict[str, Any],