Merge pull request #59 from csobrinho/fix-56

Fix for #56.

xfinity-usage/Dockerfile

@@ -156,7 +156,8 @@ RUN pip3 install --upgrade --break-system-packages pip && \
     pip3 install --no-cache-dir \
   --prefer-binary --break-system-packages \
   -r requirements.txt \
-  playwright==1.49.0
+  playwright==1.49.1 \
+  playwright-stealth==1.0.6
 
 RUN playwright install firefox
 

xfinity-usage/xfinity_usage_addon.py

@@ -24,6 +24,8 @@
 from paho.mqtt import client as mqtt
 from pathlib import Path
 from playwright.async_api import async_playwright, Playwright, Route, Response, Request, Frame, Page, expect
+from playwright_stealth import StealthConfig, stealth_async
+
 
 # Browser mode
 HEADLESS = json.loads(os.environ.get('HEADLESS', 'true').lower()) # Convert HEADLESS string into boolean
@@ -425,11 +427,9 @@ async def start(self):
 
         #logger.info(f"Launching {textwrap.shorten(self.device['user_agent'], width=77, placeholder='...')}")
 
-        self.firefox_user_prefs={'webgl.disabled': True, 'network.http.http2.enabled': False}
+        #self.firefox_user_prefs={'webgl.disabled': True, 'network.http.http2.enabled': False}
         #self.firefox_user_prefs={'webgl.disabled': True}
         #self.firefox_user_prefs={'webgl.disabled': False}
-        self.webdriver_script = "delete Object.getPrototypeOf(navigator).webdriver"
-        #self.webdriver_script = ""
 
         #self.browser = playwright.firefox.launch(headless=False,slow_mo=1000,firefox_user_prefs=self.firefox_user_prefs)
         #self.browser = await self.playwright.firefox.launch(headless=HEADLESS,firefox_user_prefs=self.firefox_user_prefs)
@@ -443,8 +443,10 @@ async def start(self):
 
         #self.context = playwright.firefox.launch_persistent_context(profile_path,headless=False,firefox_user_prefs=self.firefox_user_prefs,**self.device)
         #self.context = playwright.firefox.launch_persistent_context(profile_path,headless=False,firefox_user_prefs=self.firefox_user_prefs,**self.device)
-        self.context = await self.playwright.firefox.launch_persistent_context(self.profile_path,headless=HEADLESS,firefox_user_prefs=self.firefox_user_prefs,**self.device)
+        #self.context = await self.playwright.firefox.launch_persistent_context(self.profile_path,headless=HEADLESS,firefox_user_prefs=self.firefox_user_prefs,**self.device)
 
+        browser = await self.playwright.firefox.launch(headless=HEADLESS)
+        self.context = await browser.new_context()
 
         # Block unnecessary requests
         await self.context.route("**/*", lambda route: self.abort_route(route))
@@ -456,6 +458,28 @@ async def start(self):
         self.context.on("requestfailed", self.check_requestfailed)
         self.context.on("requestfinished", self.check_requestfinished)
 
+        await stealth_async(
+            self.context,
+            StealthConfig(
+                webdriver=True,
+                webgl_vendor=True,
+                chrome_app=False,
+                chrome_csi=False,
+                chrome_load_times=False,
+                chrome_runtime=False,
+                iframe_content_window=True,
+                media_codecs=True,
+                navigator_hardware_concurrency=4,
+                navigator_languages=False,
+                navigator_permissions=True,
+                navigator_platform=False,
+                navigator_plugins=True,
+                navigator_user_agent=False,
+                navigator_vendor=False,
+                outerdimensions=True,
+                hairline=False,
+            ),
+        )
 
         #self.page = await self.context.new_page()
         self.page = await self.get_new_page()
@@ -489,9 +513,6 @@ async def get_new_page(self) -> Page:
         _page.set_default_timeout(self.timeout)
         expect.set_options(timeout=self.timeout)
 
-        # Help reduce bot detection
-        await _page.add_init_script(self.webdriver_script)
-
         return _page
 
     async def get_browser_device(self) -> dict:
@@ -618,7 +639,7 @@ async def debug_support(self) -> None:
 
             datetime_format = datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%f")
             page_content = await self.page.content()
-            page_content_hash = hash(base64.b64encode(await self.page.content().encode()).decode())
+            page_content_hash = hash(base64.b64encode((page_content or "").encode()).decode())
             page_screenshot = await self.page.screenshot()
             page_screenshot_hash = hash(base64.b64encode(page_screenshot).decode())
 
@@ -1304,7 +1325,8 @@ async def run_playwright() -> None:
 
             if is_mqtt_available() and mqtt_client.is_connected_mqtt():
                 mqtt_client.publish_mqtt(usage.usage_data)
-
+        except Exception as e:
+            logger.warning(f"Error: {e}")
         finally:
             await usage.done()