feat: Detailed report

blc/__main__.py

@@ -92,7 +92,7 @@ def main(args):
         pass
 
     checker_threads = []
-    broken_url = {}
+    report = {}
 
     for target in args.host.split(','):
         # We initialize the checker
@@ -102,8 +102,7 @@ def main(args):
             deep_scan=args.deep_scan,
         )
         # We config the shared dict
-        broken_url[target] = {}
-        checker.broken_url = broken_url[target]
+        report[target] = checker.urls
 
         t = threading.Thread(target=checker.run)
         checker_threads.append(t)
@@ -126,19 +125,31 @@ def main(args):
 
     # We build the report
     msg = 'Hello, the report of the broken link checker is ready.\n'
-    for target in broken_url:
-        msg += f"Report of {target}:\n"
-        if broken_url[target]:
-            for data in broken_url[target].items():
-                msg += str(data) + '\n'
+    for target in report:
+        msg += f"--------------\nReport of {target}\n--------------"
+        if report[target]:
+            acc = 0
+            for url, info in report[target].items():
+                if not info['result'][0]:
+                    msg += "\n"\
+                    f"URL:        {url}\n"\
+                    f"Parent URL: {info['parent']}\n"\
+                    f"Real URL:   {info['url']}\n"\
+                    f"Check time: {info['check_time']} seconds\n"\
+                    f"Result:     {info['result'][1]} -> {info['result'][2]}\n"
+                    acc += 1
+                else:
+                    pass
+            msg += f"\nThats it. {acc} errors in {len(report[target])} links found.\n"\
+            "--------------\n\n"
         else:
-            msg += "No broken url found\n"
+            pass
 
     # We verify if the email notifier is configured
     if args.smtp_server:
         # We notify the admin
         logging.info('Sending of the report to %s...' % args.recipient)
-        notifier.send(subject='Broken links found', body=msg, recipient=args.recipient)
+        notifier.send(subject='Broken links found', body=msg or "No broken url found\n", recipient=args.recipient)
     else:
         print(msg)
 

blc/checker.py

@@ -44,14 +44,16 @@ def __init__(self, host: str, delay: int = 1, deep_scan: bool = False):
         # Shallow scan of foreign url
         self.deep_scan = deep_scan
 
-        # Will represent the list of URL to check
-        self.url_to_check = [host]
-
         # Will represent the list of checked URL
-        self.checked_url = []
+        self.urls = {
+            host: {
+                'parent': '',
+                'url': None,
+                'result': None,
+                'check_time': None
 
-        # Will represent the list of broken URL
-        self.broken_url = {}
+            }
+        }
 
         # Will represent the previous webpage content
         self.prev_data = ''
@@ -101,33 +103,31 @@ def check(self, url: str) -> requests.Response:
         :url represent the URL to check
         """
         # We verify the URL is already checked
-        if url in self.checked_url:
+        if [u for u in self.urls if self.urls[u]['result'] and url == u]:
             return None
 
         self.logging.info('Checking of %s...' % url)
 
-        # We mark the URL checked
-        self.checked_url.append(url)
-
         # We make a connection
         try:
             if self.is_same_host(url):
                 response = self.conn.get(url, timeout=2, stream=True)
             else:
                 response = self.conn.head(url, timeout=2)
         except requests.exceptions.ReadTimeout:
-            self.broken_url[url] = "Timeout!"
+            self.urls[url]['result'] = False, None, "Timeout!"
         except requests.exceptions.ConnectionError:
-            self.broken_url[url] = "Connection aborted!"
+            self.urls[url]['result'] = False, None, "Connection aborted!"
         except requests.exceptions.TooManyRedirects:
-            self.broken_url[url] = "Too many redirection!"
+            self.urls[url]['result'] = False, None, "Too many redirection!"
         else:
             # We verify the response status
             # 2xx stand for request was successfully completed
             if response.ok:
+                self.urls[url]['result'] = True, response.status_code, response.reason
                 return response if self.is_same_host(url) else None
             else:
-                self.broken_url[url] = response.reason
+                self.urls[url]['result'] = False, response.status_code, response.reason
 
                 self.logging.warning(
                     '%s maybe broken because status code: %i' %
@@ -177,6 +177,8 @@ def update_list(self, response: requests.Response) -> None:
                 else:
                     continue
 
+                origin_url = url
+
                 # 1.1 and 1.2
                 if self.is_same_host(url):
                     # 1.2
@@ -207,11 +209,15 @@ def update_list(self, response: requests.Response) -> None:
                 # Except if the deep_scan is enable
                 # At this point, the URL belongs to the HOST
                 # We verify that the URL is neither already added nor checked
-                if url not in self.url_to_check \
-                    and url not in self.checked_url \
-                        and url != response.url:
+                if url not in self.urls and url != response.url:
                     self.logging.debug('Add the URL %s' % url)
-                    self.url_to_check.append(url)
+                    self.urls[url] = {
+                        'parent': response.url,
+                        'url': origin_url,
+                        'result': None,
+                        'check_time': None
+
+                    }
                 else:
                     continue
 
@@ -226,8 +232,20 @@ def update_list(self, response: requests.Response) -> None:
     def run(self) -> None:
         """Run the checker."""
         # We check while we have an URL unchecked
-        while (self.url_to_check):
-            response = self.check(self.url_to_check.pop(0))
-            if response:
-                self.update_list(response)
-            time.sleep(self.delay)
+        while 1:
+            url_to_check = [u for u in self.urls if not self.urls[u]['result']]
+
+            if url_to_check:
+                pass
+            else:
+                break
+
+            while (url_to_check):
+                url = url_to_check.pop(0)
+                self.urls[url]['check_time'] = time.time()
+                response = self.check(url)
+                if response:
+                    self.update_list(response)
+                self.urls[url]['check_time'] = time.time() - self.urls[url]['check_time']
+                time.sleep(self.delay)
+

tests/checker_test.py

@@ -69,9 +69,9 @@ def close():
         checker = Checker(Response.url)
         checker.update_list(Response)
 
-        self.assertEqual(len(checker.url_to_check), 18)
+        self.assertEqual(len(checker.urls), 18)
 
         # with deep mode
         checker = Checker('localhost', deep_scan=True)
         checker.update_list(Response)
-        self.assertEqual(len(checker.url_to_check), 36)
+        self.assertEqual(len(checker.urls), 36)

tests/checker_test.sh

@@ -34,7 +34,7 @@ start_server() {
 # We start the test
 start_test() {
     report=$($PYTHON -m blc http://$HOST:$PORT -D -d 0 $BLC_FLAGS)
-    nb_broken_link_got=$(expr $(echo "$report" | grep -c .) - 2)
+    nb_broken_link_got=$(echo "$report"  | grep "\w* errors" -o | grep "[0-9]*" -o)
     if [ ! $nb_broken_link_got -eq $NB_BROKEN_LINK_EXPECTED ]; then
         echo "$NB_BROKEN_LINK_EXPECTED broken links expected, but $nb_broken_link_got got"
         echo "REPORT:"