From 8c3c5d1f31ec38f338ca587851b148b415c22bbe Mon Sep 17 00:00:00 2001
From: wangxianqiao <wangxianqiao@cloudfine.com>
Date: Fri, 3 Sep 2021 17:11:56 +0800
Subject: [PATCH] Custom downloader support ShadowDOM

---
 scraper/src/custom_downloader_middleware.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scraper/src/custom_downloader_middleware.py b/scraper/src/custom_downloader_middleware.py
index c3bea000..f8f23e0d 100644
--- a/scraper/src/custom_downloader_middleware.py
+++ b/scraper/src/custom_downloader_middleware.py
@@ -28,7 +28,7 @@ def process_request(self, request, spider):
         self.driver.get(unquote_plus(
             request.url))  # Decode url otherwise firefox is not happy. Ex /#%21/ => /#!/%21
         time.sleep(spider.js_wait)
-        body = self.driver.page_source.encode('utf-8')
+        body = self.driver.execute_script("return document.documentElement.outerHTML;")
         url = self.driver.current_url
 
         return HtmlResponse(