Merge pull request #55 from thor0215/development/0.1.1

0.1.1

README.md

@@ -16,7 +16,7 @@ Fetch Xfinity Internet Service Usage Data and publish it to a Home Assistant sen
 
 ## Setup
 
-### ***This addon will not work if your Xfinity account is using MFA***
+### ***This addon will not work if your Xfinity account is using Two-step verification***
 
 1. Add this repository `https://github.com/thor0215/hassio-xfinity-usage` to Home Assistant as a source for third-party addons. See the [Home Assistant documentation](https://www.home-assistant.io/common-tasks/os#installing-third-party-add-ons) if you have questions on how to do that. You can also use the button below.
 

xfinity-usage/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## 0.1.1
+
+- Updated AUTH_URL to reflect changes on Xfinity's website
+- Added ability to delete browser profile on startup using environment variable/config
+- Script now hits a logout URL on startup. This will help prevent any issues with any session data that might still exist.
+
+- Dependency Update
+    - playwright v1.49
+    - PyJWT v2.10.1
+    - colorlog v6.9.0
+
 ## 0.1.0
 
 - Moved script polling function out of the Python script and into the run.sh script. 

xfinity-usage/Dockerfile

@@ -34,7 +34,7 @@ RUN \
     && apt-get install -y --no-install-recommends \
         bash=5.2.15-2+b7 \
         ca-certificates=20230311 \
-        curl=7.88.1-10+deb12u7 \
+        curl \
         jq=1.6-2.1 \
         tzdata=2024a-0+deb12u1 \
         xz-utils=5.4.1-0.2 \
@@ -156,7 +156,7 @@ RUN pip3 install --upgrade --break-system-packages pip && \
     pip3 install --no-cache-dir \
   --prefer-binary --break-system-packages \
   -r requirements.txt \
-  playwright==1.44.0
+  playwright==1.49.0
 
 RUN playwright install firefox
 

xfinity-usage/config.yaml

@@ -1,7 +1,7 @@
 name: "Xfinity Internet Usage"
 description: "Get Xfinity Internet Usage Data"
 url: "https://github.com/thor0215/hassio-xfinity-usage"
-version: "0.1.0"
+version: "0.1.1"
 image: "ghcr.io/thor0215/hassio-xfinity-usage-{arch}"
 slug: "xfinity-usage"
 init: false
@@ -14,7 +14,7 @@ homeassistant_api: true
 hassio_api: true
 hassio_role: manager
 panel_icon: mdi:network
-init: false
+video: true
 map:
   - addon_config:rw
 backup_exclude:
@@ -45,4 +45,5 @@ schema:
   mqtt_username: str?
   mqtt_password: password?
   mqtt_raw_usage: bool?
+  profile_cleanup: bool?
   debug_support: bool?

xfinity-usage/requirements.txt

@@ -1,6 +1,6 @@
 typing-extensions==4.12.2
 tenacity==9.0.0
 requests==2.32.3
-PyJWT==2.9.0
+PyJWT==2.10.1
 paho-mqtt==2.1.0
-colorlog==6.8.2
+colorlog==6.9.0

xfinity-usage/run.sh

@@ -41,6 +41,7 @@ if [ $BYPASS = "0" ]; then
     [[ $(bashio::config "mqtt_host") != null ]] && export MQTT_HOST=$(bashio::config "mqtt_host")
     [[ $(bashio::config "mqtt_port") != null ]] && export MQTT_PORT=$(bashio::config "mqtt_port")
     [[ $(bashio::config "mqtt_raw_usage") != null ]] && export MQTT_RAW_USAGE=$(bashio::config "mqtt_raw_usage")
+    [[ $(bashio::config "profile_cleanup") != null ]] && export PROFILE_CLEANUP=$(bashio::config "profile_cleanup")
     [[ $(bashio::config "debug_support") != null ]] && export DEBUG_SUPPORT=$(bashio::config "debug_support")
 
 

xfinity-usage/translations/en.yaml

@@ -39,7 +39,11 @@ configuration:
   mqtt_raw_usage:
     name: MQTT Raw Usage
     description: >-
-      Add raw usage json to MQTT sensor (optional)
+      Add raw usage json to MQTT sensor. (optional)
+  profile_cleanup:
+    name: Cleanup browser profile on startup
+    description: >-
+      Delete browser profile during startup. (optional)
   debug_support:
     name: Debug Support
     description: >-

xfinity-usage/xfinity_usage_addon.py

@@ -37,21 +37,24 @@
 ANDROID_MIN_VERSION = os.environ.get('ANDROID_MIN_VERSION', 10)
 ANDROID_MAX_VERSION = os.environ.get('ANDROID_MAX_VERSION', 10)
 FIREFOX_MIN_VERSION = os.environ.get('FIREFOX_MIN_VERSION', 120)
-FIREFOX_MAX_VERSION = os.environ.get('FIREFOX_MAX_VERSION', 124)
+FIREFOX_MAX_VERSION = os.environ.get('FIREFOX_MAX_VERSION', 120)
 
 # GLOBAL URLS
 VIEW_USAGE_URL = 'https://customer.xfinity.com/#/devices#usage'
 VIEW_WIFI_URL = 'https://customer.xfinity.com/settings/wifi'
 INTERNET_SERVICE_URL = 'https://www.xfinity.com/learn/internet-service/auth'
-AUTH_URL = 'https://content.xfinity.com/securelogin/cima?sc_site=xfinity-learn-ui&continue=https://www.xfinity.com/auth'
-#AUTH_URL = 'https://content.xfinity.com/securelogin/cima?sc_site=xfinity-learn-ui&continue=https://www.xfinity.com/learn/internet-service/auth'
+#AUTH_URL = 'https://content.xfinity.com/securelogin/cima?sc_site=xfinity-learn-ui&continue=https://www.xfinity.com/auth'
+AUTH_URL = 'https://oauth.xfinity.com/oauth/authorize?response_type=token&prompt=select_billing_account&redirect_uri=https%3A%2F%2Fwww.xfinity.com%2Fpost-auth&client_id=shoplearn-web&state=https%3A%2F%2Fwww.xfinity.com%2Fauth'
 LOGIN_URL = 'https://login.xfinity.com/login'
 LOGOUT_URL = 'https://www.xfinity.com/overview'
 USAGE_JSON_URL = 'https://api.sc.xfinity.com/session/csp/selfhelp/account/me/services/internet/usage'
 PLAN_DETAILS_JSON_URL = 'https://api.sc.xfinity.com/session/plan'
 DEVICE_DETAILS_URL = 'https://www.xfinity.com/support/status'
 DEVICE_DETAILS_JSON_URL = 'https://api.sc.xfinity.com/devices/status'
 SESSION_URL = 'https://api.sc.xfinity.com/session'
+XFINITY_START_URL = 'https://oauth.xfinity.com/oauth/sp-logout?client_id=shoplearn-web'
+#AUTH_PAGE_TITLE = 'Internet, TV, Phone, Smart Home and Security - Xfinity by Comcast'
+AUTH_PAGE_TITLE = 'Discovery Hub - News & Technology'
 
 # Xfinity authentication
 XFINITY_USERNAME = os.environ.get('XFINITY_USERNAME', None)
@@ -61,6 +64,9 @@
 BYPASS = int(os.environ.get('BYPASS',0))
 POLLING_RATE = float(int(os.environ.get('POLLING_RATE', 0)))
 
+# Force profile cleanup during startup
+PROFILE_CLEANUP = json.loads(os.environ.get('PROFILE_CLEANUP', 'false').lower()) # Convert PROFILE_CLEANUP string into boolean
+
 # Playwright timeout
 PAGE_TIMEOUT = int(os.environ.get('PAGE_TIMEOUT', 60))
 
@@ -179,11 +185,18 @@ async def get_slow_down_login():
     if SLOW_DOWN_LOGIN:
         await asyncio.sleep(random.uniform(SLOW_DOWN_MIN, SLOW_DOWN_MAX))
 
+async def profile_cleanup():
+    # Remove browser profile path to clean out cookies and cache
+    profile_path = '/config/profile*'
+    directories = glob.glob(profile_path)
+    for directory in directories:
+        if Path(directory).exists() and Path(directory).is_dir(): shutil.rmtree(directory)
 
 class exit_code(Enum):
     SUCCESS = 0
     MISSING_LOGIN_CONFIG = 80
     MISSING_MQTT_CONFIG = 81
+    BAD_AUTHENTICATION = 93
     TOO_MANY_USERNAME = 94
     TOO_MANY_PASSWORD = 95
     BAD_PASSWORD = 96
@@ -542,8 +555,11 @@ async def abort_route(self, route: Route) :
         good_xfinity_domains = ['*.xfinity.com', '*.comcast.net', 'static.cimcontent.net', '*.codebig2.net']
         regex_good_xfinity_domains = ['xfinity.com', 'comcast.net', 'static.cimcontent.net', 'codebig2.net']
 
+        #good_xfinity_domains = ['*.xfinity.com', '*.comcast.net', 'static.cimcontent.net', '*.codebig2.net', '*']
+        #regex_good_xfinity_domains = ['xfinity.com', 'comcast.net', 'static.cimcontent.net', 'codebig2.net', '.*']
+
         # Domains blocked base on Origin Ad Block filters
-        regex_block_xfinity_domains = ['.ico$',
+        regex_block_xfinity_domains = ['.ico$','.mp4$','.vtt$',
                                        '/akam/',
                                        #re.compile('xfinity.com/(?:\w+\/{1}){4,}\w+'), # Will cause Akamai Access Denied
                                        'login.xfinity.com/static/ui-common/',
@@ -556,9 +572,16 @@ async def abort_route(self, route: Route) :
                                        'target.xfinity.com',
                                        'yhm.comcast.net'
                                        ] + xfinity_block_list
-
+        """
+        regex_block_xfinity_domains = ['.ico$','.mp4$','.vtt$'
+                                       ] + xfinity_block_list
+        regex_block_xfinity_domains = ['quantummetric.com',
+                                       'amazonaws.com'] + xfinity_block_list
+        """
+
         # Block unnecessary resources
         bad_resource_types = ['image', 'images', 'stylesheet', 'media', 'font']
+        #bad_resource_types = []
 
         if  route.request.resource_type not in bad_resource_types and \
             any(fnmatch.fnmatch(urllib.parse.urlsplit(route.request.url).netloc, pattern) for pattern in good_xfinity_domains):
@@ -802,14 +825,13 @@ async def check_response(self,response: Response) -> None:
             if  content_type_header is not None:
                 if re.match('application/json', content_type_header):
                     content_type = 'json'
+                    response_json = None
 
-                    if content_length_header is not None and content_length_header != '0' :
+                    if content_length_header is not None and content_length_header != '0' and response.status != 204:
                         page_body = await response.body()
                         if len(page_body) != 0:
                             logger.debug(f"Response: {response.status} {request.resource_type} {content_type} {content_length_header} {response.url}")
                             response_json = await response.json()
-                    else:
-                        response_json = None
 
             if request.is_navigation_request():
                 if  LOG_LEVEL == 'DEBUG' and \
@@ -819,8 +841,8 @@ async def check_response(self,response: Response) -> None:
                         logger.debug(f"Response: {response.status} {page_body}")
                         logger.debug(f"Response: {response.status} {response.headers}")
 
-            if content_type == 'json' and response_json is None:
-                if  response.url == SESSION_URL and 'x-ssm-token' in response.headers:
+            if  content_type == 'json' and \
+                response.url == SESSION_URL and 'x-ssm-token' in response.headers:
                     await self.check_jwt_session(response)
 
             if content_type == 'json' and response_json is not None:
@@ -910,7 +932,16 @@ async def check_response(self,response: Response) -> None:
                         len(response_json['services']['internet']['devices'][0]['deviceDetails']) > 0:
                             self.device_details_data = response_json['services']['internet']['devices'][0]['deviceDetails']
                             logger.info(f"Updating Device Details")
-                    logger.debug(f"Updating Device Details {json.dumps(response_json)}")                
+                    logger.debug(f"Updating Device Details {json.dumps(response_json)}")
+
+        else:
+            if response.url == SESSION_URL:
+                if response.status == 401 and response.request.method == 'POST':
+                    self.is_session_active == False    
+                    #exit(exit_code.BAD_AUTHENTICATION.value)
+                if response.status == 500:
+                    logger.info(f"Session URL returned HTTP 500, attempt page reload")
+                    self.page.reload()
 
 
     async def get_device_details_data(self) -> None:
@@ -945,9 +976,11 @@ async def get_usage_data(self) -> None:
                     await self.page.locator('h2.plan-row-title').wait_for()
                     await self.page.get_by_test_id('planRowDetail').nth(2).filter(has=self.page.locator(f"prism-button[href^=\"https://\"]")).wait_for()
             except Exception:
-                logger.error(f"planRowDetail Count: {await self.page.get_by_test_id('planRowDetail').count()}")
-                logger.error(f"planRowDetail Row 3 inner html: {await self.page.get_by_test_id('planRowDetail').nth(2).inner_html()}")
-                logger.error(f"planRowDetail Row 3 text content: {await self.page.get_by_test_id('planRowDetail').nth(2).text_content()}")
+                planRowDetailCount = await self.page.get_by_test_id('planRowDetail').count()
+                logger.error(f"planRowDetail Count: {planRowDetailCount}")
+                if planRowDetailCount > 0:
+                    logger.error(f"planRowDetail Row 3 inner html: {await self.page.get_by_test_id('planRowDetail').nth(2).inner_html()}")
+                    logger.error(f"planRowDetail Row 3 text content: {await self.page.get_by_test_id('planRowDetail').nth(2).text_content()}")
 
             finally:
                 logger.debug(f"Finished loading page (URL: {self.page.url})")
@@ -987,9 +1020,10 @@ async def get_authenticated(self) -> None:
         await self.page.goto(AUTH_URL)
         logger.info(f"Loading Xfinity Authentication (URL: {parse_url(self.page.url)})")
         _title = await self.get_page_title()
+        _state = await self.page.locator('xc-header').get_attribute('state')
         # xc-header[state="authenticated"]
-        if  _title == 'Xfinity Internet: Fastest Wifi Speeds and the Best Coverage' and \
-            await self.page.locator('xc-header').get_attribute('state') != "authenticated":
+        if  _title == AUTH_PAGE_TITLE and \
+            _state != "authenticated":
                 await self.page.close()
                 await self.goto_logout()
 
@@ -1214,6 +1248,8 @@ async def run(self) -> None:
         """
         await self.start()
 
+        await self.page.goto(XFINITY_START_URL)
+
         await self.get_authenticated()
 
         await self.get_usage_data()
@@ -1304,6 +1340,11 @@ async def main():
 
     Returns: None
     """
+
+    # If PROFILE_CLEANUP, delete profile
+    if PROFILE_CLEANUP:
+        await profile_cleanup()
+
     logger.info(f"Xfinity Internet Usage Starting")
     while True:
         try:
@@ -1322,12 +1363,10 @@ async def main():
         except BaseException as e:
             if (type(e) == SystemExit) and \
                 (e.code == exit_code.TOO_MANY_USERNAME.value or \
-                e.code == exit_code.TOO_MANY_PASSWORD.value):
+                e.code == exit_code.TOO_MANY_PASSWORD.value or \
+                e.code == exit_code.BAD_AUTHENTICATION.value ):
                         # Remove browser profile path to clean out cookies and cache
-                        profile_path = '/config/profile*'
-                        directories = glob.glob(profile_path)
-                        for directory in directories:
-                            if Path(directory).exists() and Path(directory).is_dir(): shutil.rmtree(directory)
+                        profile_cleanup()
 
             if is_mqtt_available():
                 mqtt_client.disconnect_mqtt()