diff --git a/backend/penndata/management/commands/get_engineering_events.py b/backend/penndata/management/commands/get_engineering_events.py index b1f47a0a..319fca23 100644 --- a/backend/penndata/management/commands/get_engineering_events.py +++ b/backend/penndata/management/commands/get_engineering_events.py @@ -17,6 +17,7 @@ def handle(self, *args, **kwargs): try: resp = requests.get(ENGINEERING_EVENTS_WEBSITE) except ConnectionError: + print("Error:", ConnectionError) return None html_content = resp.text @@ -36,14 +37,10 @@ def handle(self, *args, **kwargs): url = event.get("url", "") start = datetime.datetime.fromisoformat(event.get("startDate")) - end = event.get("endDate", "") - if end != "": - end = datetime.datetime.fromisoformat(end) - else: - end = None + end = datetime.datetime.fromisoformat(event["endDate"]) if "endDate" in event else None location = None - if event.get("location"): + if "location" in event: location = event.get("location").get("name") email = None @@ -53,7 +50,7 @@ def handle(self, *args, **kwargs): Event.objects.update_or_create( name=event_name, defaults={ - "event_type": "Engineering", + "event_type": Event.TYPE_PENN_ENGINEERING, "image_url": "", "start": timezone.make_aware(start), "end": timezone.make_aware(end), diff --git a/backend/penndata/management/commands/get_penn_today_events.py b/backend/penndata/management/commands/get_penn_today_events.py index fe774ad4..33f1a5e0 100644 --- a/backend/penndata/management/commands/get_penn_today_events.py +++ b/backend/penndata/management/commands/get_penn_today_events.py @@ -104,7 +104,7 @@ def handle(self, *args, **kwargs): Event.objects.update_or_create( name=name, defaults={ - "event_type": "", + "event_type": Event.TYPE_PENN_TODAY, "image_url": "", "start": start_date, "end": end_date, diff --git a/backend/penndata/management/commands/get_rodin_events.py b/backend/penndata/management/commands/get_rodin_events.py index 2c45e1f3..9ec1e8be 100644 --- a/backend/penndata/management/commands/get_rodin_events.py +++ b/backend/penndata/management/commands/get_rodin_events.py @@ -9,28 +9,26 @@ RODIN_EVENTS_WEBSITE = "https://rodin.house.upenn.edu" -HEADERS = { - "User-Agent": "Mozilla/5.0 AppleWebKit/537.36 Chrome/91.0.4472.124 Safari/537.36" -} class Command(BaseCommand): def handle(self, *args, **kwargs): - self.scrape_calendar_page(RODIN_EVENTS_WEBSITE + "/calendar") + self.scrape_calendar_page(f"{RODIN_EVENTS_WEBSITE}/calendar") now = timezone.localtime() current_day, current_month, current_year = now.day, now.month, now.year if current_day > 25: - next_month_year = current_year if current_month < 12 else current_year + 1 - next_month = current_month + 1 if current_month < 12 else 1 - next_month_url = f"{RODIN_EVENTS_WEBSITE}/calendar/{next_month_year}-{next_month:02d}" + next = now + datetime.timedelta(months=1) + next_month, next_year = next.month, next.year + next_month_url = f"{RODIN_EVENTS_WEBSITE}/calendar/{next_year}-{next_month:02d}" self.scrape_calendar_page(next_month_url) self.stdout.write("Uploaded Rodin College House Events!") def scrape_details(self, event_url): try: - resp = requests.get(event_url, headers=HEADERS) + resp = requests.get(event_url) except ConnectionError: + print("Error:", ConnectionError) return None soup = BeautifulSoup(resp.text, "html.parser") @@ -50,7 +48,7 @@ def scrape_details(self, event_url): def scrape_calendar_page(self, calendar_url): try: - resp = requests.get(calendar_url, headers=HEADERS) + resp = requests.get(calendar_url) except ConnectionError: return None soup = BeautifulSoup(resp.text, "html.parser") @@ -59,29 +57,27 @@ def scrape_calendar_page(self, calendar_url): for cell in event_cells: item = cell.find("div", class_="item") - if item: - event_link = item.find("a", href=True) - if event_link: - name = event_link.text.strip() - url = RODIN_EVENTS_WEBSITE + event_link["href"] + if not item: + continue + event_link = item.find("a", href=True) + if not event_link: + continue + name = event_link.text.strip() + url = f"{RODIN_EVENTS_WEBSITE}{event_link["href"]}" - location, start_time, end_time, description = self.scrape_details(url) - print("Location:", location) - print("Start Time:", start_time) - print("End Time:", end_time) - print("Description:", description) - Event.objects.update_or_create( - name=name, - defaults={ - "event_type": "Rodin College House", - "image_url": "", - "start": timezone.make_aware(start_time), - "end": timezone.make_aware(end_time), - "location": location, - "website": url, - "description": description, - "email": "", - }, - ) - if start_time > timezone.localtime() + datetime.timedelta(days=30): - break + location, start_time, end_time, description = self.scrape_details(url) + Event.objects.update_or_create( + name=name, + defaults={ + "event_type": Event.TYPE_RODIN_COLLEGE_HOUSE, + "image_url": "", + "start": timezone.make_aware(start_time), + "end": timezone.make_aware(end_time), + "location": location, + "website": url, + "description": description, + "email": "", + }, + ) + if start_time > timezone.localtime() + datetime.timedelta(days=30): + break diff --git a/backend/penndata/management/commands/get_venture_events.py b/backend/penndata/management/commands/get_venture_events.py index b5c30ba7..b2913921 100644 --- a/backend/penndata/management/commands/get_venture_events.py +++ b/backend/penndata/management/commands/get_venture_events.py @@ -23,6 +23,7 @@ def handle(self, *args, **kwargs): try: resp = requests.get(VENTURE_EVENTS_WEBSITE, headers=HEADERS) except ConnectionError: + print("Error:", ConnectionError) return None soup = BeautifulSoup(resp.text, "html.parser") @@ -96,7 +97,7 @@ def handle(self, *args, **kwargs): Event.objects.update_or_create( name=title, defaults={ - "event_type": "Venture Lab", + "event_type": Event.TYPE_VENTURE_LAB, "image_url": "", "start": timezone.make_aware(event_start_datetime), "end": timezone.make_aware(event_end_datetime), diff --git a/backend/penndata/models.py b/backend/penndata/models.py index 9279fb52..ac67fb69 100644 --- a/backend/penndata/models.py +++ b/backend/penndata/models.py @@ -9,7 +9,20 @@ class Event(models.Model): - event_type = models.CharField(max_length=255, null=True, blank=True) + TYPE_PENN_TODAY = "PENN TODAY" + TYPE_VENTURE_LAB = "VENTURE LAB" + TYPE_PENN_ENGINEERING = "PENN ENGINEERING" + TYPE_RODIN_COLLEGE_HOUSE = "RODIN COLLEGE HOUSE" + + TYPE_CHOICES = ( + (TYPE_PENN_TODAY, "Penn Today"), + (TYPE_VENTURE_LAB, "Venture Lab"), + (TYPE_PENN_ENGINEERING, "Penn Engineering"), + (TYPE_RODIN_COLLEGE_HOUSE, "Rodin College House"), + ) + + event_type = models.CharField(max_length=63, choices=TYPE_CHOICES, + default=None, null=True, blank=True) name = models.CharField(max_length=255) description = models.TextField(null=True, blank=True) image_url = models.URLField(null=True, blank=True)