Merge pull request #142 from SpokaneTech/joeriddles/138-scrape-images

src/web/scrapers.py

@@ -3,13 +3,14 @@
 import pathlib
 import re
 import urllib.parse
+import zoneinfo
 from datetime import datetime, timedelta
 from typing import Any, Protocol, TypeAlias, TypeVar
 
 import eventbrite.access_methods
 import requests
-import zoneinfo
-from bs4 import BeautifulSoup, Tag
+from bs4 import BeautifulSoup
+from bs4.element import Tag
 from django.conf import settings
 from django.utils import timezone
 from eventbrite import Eventbrite
@@ -26,7 +27,7 @@ def get_venue(self, id, **data):
 
 
 def get_event_description(self, id, **data):
- return self.get("/events/{0}/description//".format(id), data=data)
+ return self.get("/events/{0}/description/".format(id), data=data)
 
 
 setattr(eventbrite.access_methods.AccessMethodsMixin, "get_venue", get_venue)
@@ -39,10 +40,23 @@ def scrape(self, url: str) -> ST:
  ...
 
 
-EventScraperResult: TypeAlias = tuple[models.Event, list[models.Tag]]
+ImageResult: TypeAlias = tuple[str, bytes]
+EventScraperResult: TypeAlias = tuple[models.Event, list[models.Tag], ImageResult | None]
+
+
+class ScraperMixin:
+ def _get_image(self, image_url: str) -> ImageResult:
+ image_name = self._parse_image_name(image_url)
+ response = requests.get(image_url, timeout=10)
+ response.raise_for_status()
+ image = response.content
+ return image_name, image
 
+ def _parse_image_name(self, image_url: str) -> str:
+ return image_url.rsplit("/", maxsplit=1)[-1].split("?", maxsplit=1)[0]
 
-class MeetupScraperMixin:
+
+class MeetupScraperMixin(ScraperMixin):
  """Common Meetup scraping functionality."""
 
  def _parse_apollo_state(self, soup: BeautifulSoup) -> dict:
@@ -84,8 +98,8 @@ def scrape(self, url: str) -> list[str]:
  else:
  upcoming_section = soup.find_all(id="upcoming-section")[0]
  events = upcoming_section.find_all_next(id=re.compile(r"event-card-"))
- filtered_event_containers = [event for event in events if self._filter_event_tag(event)]
- event_urls = [event_container["href"] for event_container in filtered_event_containers]
+ filtered_event_containers: list[Tag] = [event for event in events if self._filter_event_tag(event)] # type: ignore
+ event_urls: list[str] = [event_container["href"] for event_container in filtered_event_containers] # type: ignore
 
  return [url for url in event_urls if self._filter_repeating_events(url)]
 
@@ -136,27 +150,31 @@ def scrape(self, url: str) -> EventScraperResult:
  location_data = apollo_state[event_json["venue"]["__ref"]]
  location = f"{location_data['address']}, {location_data['city']}, {location_data['state']}"
  external_id = event_json["id"]
+ event_photo = event_json["featuredEventPhoto"]["__ref"]
+ image_url = apollo_state[event_photo].get("highResUrl", apollo_state[event_photo]["baseUrl"])
  except KeyError:
  name = self._parse_name(soup)
  description = self._parse_description(soup)
  date_time = self._parse_date_time(soup)
  duration = self._parse_duration(soup)
  location = self._parse_location(soup)
  external_id = self._parse_external_id(url)
+ image_url = self._parse_image(soup)
+
+ if image_url:
+ image_result = self._get_image(image_url)
 
  tags = self._parse_tags(soup)
- return (
- models.Event(
- name=name,
- description=description,
- date_time=date_time,
- duration=duration,
- location=location,
- external_id=external_id,
- url=url,
- ),
- tags,
+ event = models.Event(
+ name=name,
+ description=description,
+ date_time=date_time,
+ duration=duration,
+ location=location,
+ external_id=external_id,
+ url=url,
  )
+ return (event, tags, image_result)
 
  def _parse_name(self, soup: BeautifulSoup) -> str:
  name: str = soup.find_all("h1")[0].text
@@ -171,10 +189,16 @@ def _parse_description(self, soup: BeautifulSoup) -> str:
  return description
 
  def _parse_date_time(self, soup: BeautifulSoup) -> datetime:
- return datetime.fromisoformat(soup.find_all("time")[0]["datetime"])
+ time: Tag | None = soup.find("time") # type: ignore
+ if not time:
+ raise ValueError("could not find time")
+ dt: str = time["datetime"] # type: ignore
+ return datetime.fromisoformat(dt)
 
  def _parse_duration(self, soup: BeautifulSoup) -> timedelta:
- time: Tag = soup.find_all("time")[0]
+ time: Tag | None = soup.find("time") # type: ignore
+ if not time:
+ raise ValueError("could not find time")
  matches = self.DURATION_PATTERN.findall(time.text)
  if not matches:
  raise ValueError("Could not find duration from:", time.text)
@@ -199,8 +223,18 @@ def _parse_tags(self, soup: BeautifulSoup) -> list[models.Tag]:
  tags = [re.sub(r"\s+", " ", t.text) for t in tags] # Some tags have newlines & extra spaces
  return [models.Tag(value=t) for t in tags]
 
+ def _parse_image(self, soup: BeautifulSoup) -> str | None:
+ picture = soup.find(attrs={"data-testid": "event-description-image"})
+ if not picture:
+ return None
+ img: Tag | None = picture.find("img") # type: ignore
+ if not img:
+ return None
+ src: str = img["src"] # type: ignore
+ return src
+
 
-class EventbriteScraper(Scraper[list[EventScraperResult]]):
+class EventbriteScraper(ScraperMixin, Scraper[list[EventScraperResult]]):
  def __init__(self, api_token: str | None = None):
  self.client = Eventbrite(api_token or settings.EVENTBRITE_API_TOKEN)
  self._location_by_venue_id: dict[str, str] = {}
@@ -209,11 +243,12 @@ def scrape(self, organization_id: str) -> list[EventScraperResult]:
  response = self.client.get_organizer_events(
  organization_id,
  status="live",
+ expand="logo",
  )
- events_and_tags = [self.map_to_event(eventbrite_event) for eventbrite_event in response["events"]]
- return events_and_tags
+ results = [self.map_to_event(eventbrite_event) for eventbrite_event in response["events"]]
+ return results
 
- def map_to_event(self, eventbrite_event: dict) -> tuple[models.Event, list[models.Tag]]:
+ def map_to_event(self, eventbrite_event: dict) -> EventScraperResult:
  name = eventbrite_event["name"]["text"]
  start = datetime.fromisoformat(eventbrite_event["start"]["utc"])
  end = datetime.fromisoformat(eventbrite_event["end"]["utc"])
@@ -230,6 +265,16 @@ def map_to_event(self, eventbrite_event: dict) -> tuple[models.Event, list[model
  # short description
  description = eventbrite_event["description"]["html"]
 
+ try:
+ image_url = eventbrite_event["logo"]["original"]["url"]
+ image_result = self._get_image(image_url)
+ except (KeyError, requests.HTTPError):
+ try:
+ image_url = eventbrite_event["logo"]["url"]
+ image_result = self._get_image(image_url)
+ except KeyError:
+ image_result = None
+
  event = models.Event(
  name=name,
  description=description,
@@ -249,7 +294,7 @@ def map_to_event(self, eventbrite_event: dict) -> tuple[models.Event, list[model
  # if subcategory_name:
  # tags.append(models.Tag(value=subcategory_name))
 
- return event, []
+ return event, [], image_result
 
  @functools.lru_cache
  def _get_venue_location(self, venue_id: str) -> str:

src/web/services.py

@@ -1,43 +1,91 @@
 from datetime import timedelta
 from typing import Protocol
 
+from django.core.files.base import ContentFile
 from django.forms.models import model_to_dict
 from django.utils import timezone
 
 from web import models, scrapers
 
 
+class EventService:
+ def save_event_from_result(
+ self,
+ result: scrapers.EventScraperResult,
+ tech_group: models.TechGroup,
+ ) -> None:
+ event, tags, image_result = result
+ event = self._save_event(event, tech_group)
+ self._save_tags(event, tags)
+ if image_result is not None:
+ self._save_image(event, image_result)
+
+ def _save_event(
+ self,
+ event: models.Event,
+ tech_group: models.TechGroup,
+ ) -> models.Event:
+ event.group = tech_group
+ event.approved_at = timezone.localtime()
+ defaults = model_to_dict(event, exclude=["id"])
+ defaults["group"] = tech_group
+
+ del defaults["tags"] # Can't apply Many-to-Many relationship untill after the event has been saved.
+ del defaults["image"]
+
+ updated_event, _ = models.Event.objects.update_or_create(
+ external_id=event.external_id,
+ defaults=defaults,
+ )
+ return updated_event
+
+ def _save_tags(
+ self,
+ event: models.Event,
+ tags: list[models.Tag],
+ ) -> None:
+ for tag in tags:
+ tag, _ = models.Tag.objects.get_or_create(value=tag)
+ event.tags.add(tag)
+
+ def _save_image(
+ self,
+ event: models.Event,
+ image_result: scrapers.ImageResult,
+ ) -> None:
+ image_name, image = image_result
+
+ # If images are the same, don't re-upload
+ has_existing_image = bool(event.image)
+ if has_existing_image:
+ existing_image = event.image.read()
+ if existing_image == image:
+ return
+
+ file = ContentFile(image, name=image_name)
+ event.image.save(image_name, file)
+
+
 class MeetupService:
  def __init__(
  self,
  homepage_scraper: scrapers.Scraper[list[str]] | None = None,
  event_scraper: scrapers.Scraper[scrapers.EventScraperResult] | None = None,
+ event_service: EventService | None = None,
  ) -> None:
  self.homepage_scraper: scrapers.Scraper[list[str]] = homepage_scraper or scrapers.MeetupHomepageScraper()
  self.event_scraper: scrapers.Scraper[scrapers.EventScraperResult] = (
  event_scraper or scrapers.MeetupEventScraper()
  )
+ self.event_service = event_service or EventService()
 
  def save_events(self) -> None:
  """Scrape upcoming events from Meetup and save them to the database."""
- now = timezone.localtime()
  for tech_group in models.TechGroup.objects.filter(homepage__icontains="meetup.com"):
  event_urls = self.homepage_scraper.scrape(tech_group.homepage) # type: ignore
  for event_url in event_urls: # TODO: parallelize (with async?)
- event, tags = self.event_scraper.scrape(event_url)
- event.group = tech_group
- event.approved_at = now
- defaults = model_to_dict(event, exclude=["id"])
- defaults["group"] = tech_group
-
- del defaults["tags"] # Can't apply Many-to-Many relationship untill after the event has been saved.
- new_event, _ = models.Event.objects.update_or_create(
- external_id=event.external_id,
- defaults=defaults,
- )
- for tag in tags:
- tag, _ = models.Tag.objects.get_or_create(value=tag)
- new_event.tags.add(tag)
+ result = self.event_scraper.scrape(event_url)
+ self.event_service.save_event_from_result(result, tech_group)
 
 
 class EventbriteService:
@@ -46,28 +94,21 @@ class EventbriteService:
  def __init__(
  self,
  events_scraper: scrapers.Scraper[list[scrapers.EventScraperResult]] | None = None,
+ event_service: EventService | None = None,
  ) -> None:
  self.events_scraper = events_scraper or scrapers.EventbriteScraper()
+ self.event_service = event_service or EventService()
 
  def save_events(self) -> None:
  """Fetch upcoming events from Eventbrite and save them.
 
  Note: this uses an API and doesn't actually web scrape.
  """
- now = timezone.localtime()
  for eventbrite_organization in models.EventbriteOrganization.objects.prefetch_related("tech_group"):
  tech_group = eventbrite_organization.tech_group
- events_and_tags = self.events_scraper.scrape(eventbrite_organization.eventbrite_id)
- for event, _ in events_and_tags:
- event.group = tech_group
- event.approved_at = now
- defaults = model_to_dict(event, exclude=["id"])
- defaults["group"] = tech_group
- del defaults["tags"] # Can't apply Many-to-Many relationship untill after the event has been saved.
- models.Event.objects.update_or_create(
- external_id=event.external_id,
- defaults=defaults,
- )
+ results = self.events_scraper.scrape(eventbrite_organization.eventbrite_id)
+ for result in results:
+ self.event_service.save_event_from_result(result, tech_group)
 
 
 class Sender(Protocol):

src/web/tests/data/eventbrite/event_description.json

@@ -0,0 +1,3 @@
+{
+ "description": "<div>Full Day of Panels, Speakers and Vendors on Cybersecurity, AI and Compliance. FREE with pre-registration - space limited - Oct 2, 2024"
+}

src/web/tests/data/eventbrite/event_venue.json

@@ -0,0 +1,25 @@
+{
+ "address": {
+ "address_1": "702 East Desmet Avenue",
+ "address_2": "",
+ "city": "Spokane",
+ "region": "WA",
+ "postal_code": "99202",
+ "country": "US",
+ "latitude": "47.6672448",
+ "longitude": "-117.3999126",
+ "localized_address_display": "702 East Desmet Avenue, Spokane, WA 99202",
+ "localized_area_display": "Spokane, WA",
+ "localized_multi_line_address_display": [
+ "702 East Desmet Avenue",
+ "Spokane, WA 99202"
+ ]
+ },
+ "resource_uri": "https://www.eventbriteapi.com/v3/venues/214450569/",
+ "id": "214450569",
+ "age_restriction": null,
+ "capacity": null,
+ "name": "John J. Hemmingson Center",
+ "latitude": "47.6672448",
+ "longitude": "-117.3999126"
+}