Merge pull request #33 from rajiv/apple-store-update-202411

Update the AppleTrustStoreFetcher to support Apple's November 2024 site changes

trust_stores_observatory/store_fetcher/apple_store_fetcher.py

@@ -1,6 +1,7 @@
 import logging
 from typing import Tuple, List
 from urllib.request import urlopen
+from urllib.parse import urljoin
 from bs4 import BeautifulSoup
 from datetime import datetime
 
@@ -18,7 +19,7 @@
 
 class AppleTrustStoreFetcher(StoreFetcherInterface):
 
-    _INDEX_PAGE_URL = "https://support.apple.com/en-us/HT209143"
+    _INDEX_PAGE_URL = "https://support.apple.com/en-us/103272"
 
     def fetch(self, certs_repo: RootCertificatesRepository, should_update_repo: bool = True) -> TrustStore:
         # First find the latest page with the list of root certificates
@@ -30,30 +31,23 @@ def fetch(self, certs_repo: RootCertificatesRepository, should_update_repo: bool
             page_content = response.read()
         parsed_page = BeautifulSoup(page_content, "html.parser")
 
-        # There are two titles on the page, one with trusted certificates and one with blocked certificates
         parsed_trusted_certs: List[ScrapedRootCertificateRecord] = []
-        parsed_blocked_certs: List[ScrapedRootCertificateRecord] = []
         for h2_section in parsed_page.find_all("h2"):
-            if "Trusted certificates" in h2_section:
+            if "Included Root CA Certificates" in h2_section:
                 parsed_trusted_certs = self._parse_root_records_in_div(h2_section.parent)
-            elif "Blocked certificates" in h2_section:
-                parsed_blocked_certs = self._parse_root_records_in_div(h2_section.parent)
 
         # Ensure we did find entries on the page
         assert parsed_trusted_certs
-        assert parsed_blocked_certs
 
         # Look for each certificate in the supplied certs repo
         validated_trusted_certs = RootRecordsValidator.validate_with_repository(certs_repo, parsed_trusted_certs)
-        validated_blocked_certs = RootRecordsValidator.validate_with_repository(certs_repo, parsed_blocked_certs)
 
         return TrustStore(
             PlatformEnum.APPLE,
             os_version,
             trust_store_url,
             datetime.utcnow().date(),
-            validated_trusted_certs,
-            validated_blocked_certs,
+            validated_trusted_certs
         )
 
     @staticmethod
@@ -81,11 +75,12 @@ def _find_latest_root_certificates_page(cls) -> Tuple[str, str]:
         parsed_page = BeautifulSoup(page_content, "html.parser")
 
         # The page contains links to the root certificates page for each version of iOS/macOS - find the latest one
-        section_current = parsed_page.find("h2", text="Current Trust Store").parent
+        section_current = parsed_page.find("h2", text="Current Root Store").parent
         for p_tag in section_current.find_all("p"):
-            if "List of available trusted root certificates in" in p_tag.text:
-                os_and_version = p_tag.text.split("List of available trusted root certificates in")[1].strip()
-                trust_store_url = p_tag.a["href"]
+            if "List of available root certificates in" in p_tag.text:
+                os_and_version = p_tag.text.split("List of available root certificates in")[1].strip()
+                trust_store_path = p_tag.a["href"]
+                trust_store_url = urljoin(cls._INDEX_PAGE_URL, trust_store_path)
                 return os_and_version, trust_store_url
 
         raise ValueError(f"Could not find the store URL at {cls._INDEX_PAGE_URL}")