prometheus metrics and retry updates

brozzler/cli.py

@@ -234,6 +234,25 @@ def brozzle_page(argv=None):
         action="store_true",
         help="Try to avoid web bot detection",
     )
+    arg_parser.add_argument(
+        "--metrics_port",
+        type=int,
+        dest="metrics_port",
+        default=8888,
+        help="Port for brozzler's Prometheus scrape endpoint",
+    )
+    arg_parser.add_argument(
+        "--registry_url",
+        dest="registry_url",
+        default=None,
+        help="http-sd-registry url, for Prometheus metrics discovery",
+    )
+    arg_parser.add_argument(
+        "--env",
+        dest="env",
+        default=None,
+        help="env for Prometheus target registry",
+    )
     arg_parser.add_argument(
         "--screenshot-full-page", dest="screenshot_full_page", action="store_true"
     )
@@ -279,6 +298,9 @@ def brozzle_page(argv=None):
         window_height=args.window_height,
         window_width=args.window_width,
         stealth=args.stealth,
+        metrics_port=int(args.metrics_port),
+        registry_url=args.registry_url,
+        env=args.env,
     )
 
     def on_screenshot(screenshot_jpeg):
@@ -517,6 +539,25 @@ def brozzler_worker(argv=None):
         action="store_true",
         help="Try to avoid web bot detection",
     )
+    arg_parser.add_argument(
+        "--metrics_port",
+        type=int,
+        dest="metrics_port",
+        default=8888,
+        help="Port for brozzler's Prometheus scrape endpoint",
+    )
+    arg_parser.add_argument(
+        "--registry_url",
+        dest="registry_url",
+        default=None,
+        help="http-sd-registry url, for Prometheus metrics discovery",
+    )
+    arg_parser.add_argument(
+        "--env",
+        dest="env",
+        default=None,
+        help="env for Prometheus target registry",
+    )
     add_common_options(arg_parser, argv)
 
     args = arg_parser.parse_args(args=argv[1:])
@@ -573,6 +614,9 @@ def get_skip_av_seeds():
         skip_visit_hashtags=args.skip_visit_hashtags,
         skip_youtube_dl=args.skip_youtube_dl,
         stealth=args.stealth,
+        metrics_port=int(args.metrics_port),
+        registry_url=args.registry_url,
+        env=args.env,
     )
 
     signal.signal(signal.SIGQUIT, dump_state)

brozzler/metrics.py

@@ -0,0 +1,58 @@
+from typing import Optional
+
+try:
+    from http_sd_registry.client import (
+        Client,
+        Env,
+        Registration,
+        Scheme,
+        format_self_target,
+    )
+    from http_sd_registry.config import ClientConfig
+except ImportError:
+    # for users without access to http_sd_registry
+    http_sd_registry = None
+
+
+from prometheus_client import Counter, Gauge, Histogram, start_http_server
+
+# fmt: off
+brozzler_pages_crawled = Counter("brozzler_pages_crawled", "number of pages visited by brozzler")
+brozzler_page_processing_duration_seconds = Histogram("brozzler_page_processing_duration_seconds", "time spent processing a page in brozzler")
+brozzler_outlinks_found = Counter("brozzler_outlinks_found", "number of outlinks found by brozzler")
+brozzler_last_page_crawled_time = Gauge("brozzler_last_page_crawled_time", "time of last page visit, in seconds since UNIX epoch")
+brozzler_in_progress_pages = Gauge("brozzler_in_progress_pages", "number of pages currently processing with brozzler")
+brozzler_ydl_urls_checked = Counter("brozzler_ydl_urls_checked", "count of urls checked by brozzler yt-dlp")
+brozzler_ydl_extract_attempts = Counter("brozzler_ydl_extract_attempts", "count of extracts attempted by brozzler yt-dlp", labelnames=["youtube_host"])
+brozzler_ydl_extract_successes = Counter("brozzler_ydl_extract_successes", "count of extracts completed by brozzler yt-dlp", labelnames=["youtube_host"])
+brozzler_ydl_download_successes = Counter("brozzler_ydl_download_successes", "count of downloads completed by brozzler yt-dlp", labelnames=["youtube_host"])
+# fmt: on
+
+
+def register_prom_metrics(
+    metrics_port: int = 8888,
+    registry_url: Optional[str] = None,
+    env: Optional[str] = None,
+):
+    # Start metrics endpoint for scraping
+    start_http_server(metrics_port)
+
+    if registry_url is None:
+        return
+
+    if env == "qa":
+        env_for_prom = Env.qa
+    elif env == "prod":
+        env_for_prom = Env.prod
+    else:
+        env_for_prom = Env.qa
+
+    config = ClientConfig(server_url_base=registry_url)
+    client = Client(config)
+    target = format_self_target(scrape_port=metrics_port)
+    registration = Registration(
+        target=target,
+        env=env_for_prom,
+        scheme=Scheme.http,
+    )
+    client.keep_registered_threaded(registration)

brozzler/worker.py

@@ -35,6 +35,7 @@
 import urlcanon
 from requests.structures import CaseInsensitiveDict
 import rethinkdb as rdb
+from . import metrics
 from . import ydl
 
 r = rdb.RethinkDB()
@@ -71,6 +72,9 @@ def __init__(
         stealth=False,
         window_height=900,
         window_width=1400,
+        metrics_port=None,
+        registry_url=None,
+        env=None,
     ):
         self._frontier = frontier
         self._service_registry = service_registry
@@ -93,6 +97,9 @@ def __init__(
         self._window_height = window_height
         self._window_width = window_width
         self._stealth = stealth
+        self._metrics_port = metrics_port
+        self._registry_url = registry_url
+        self._env = env
 
         self._browser_pool = brozzler.browser.BrowserPool(
             max_browsers, chrome_exe=chrome_exe, ignore_cert_errors=True
@@ -104,6 +111,9 @@ def __init__(
         self._start_stop_lock = threading.Lock()
         self._shutdown = threading.Event()
 
+        # Setup metrics
+        metrics.register_prom_metrics(self._metrics_port, self._registry_url, self._env)
+
     def _choose_warcprox(self):
         warcproxes = self._service_registry.available_services("warcprox")
         if not warcproxes:
@@ -266,6 +276,7 @@ def brozzle_page(
             ):
                 try:
                     ydl_outlinks = ydl.do_youtube_dl(self, site, page)
+                    metrics.brozzler_ydl_urls_checked.inc(1)
                     outlinks.update(ydl_outlinks)
                 except brozzler.ReachedLimit as e:
                     raise
@@ -311,7 +322,15 @@ def _needs_browsing(self, page_headers):
             return False
         return True
 
+    @metrics.brozzler_page_processing_duration_seconds.time()
+    @metrics.brozzler_in_progress_pages.track_inprogress()
     def _browse_page(self, browser, site, page, on_screenshot=None, on_request=None):
+        def update_page_metrics(page, outlinks):
+            """Update page-level Prometheus metrics."""
+            metrics.brozzler_last_page_crawled_time.set_to_current_time()
+            metrics.brozzler_pages_crawled.inc(1)
+            metrics.brozzler_outlinks_found.inc(len(outlinks))
+
         def _on_screenshot(screenshot_jpeg):
             if on_screenshot:
                 on_screenshot(screenshot_jpeg)
@@ -416,6 +435,7 @@ def _on_service_worker_version_updated(chrome_msg):
         )
         if final_page_url != page.url:
             page.note_redirect(final_page_url)
+        update_page_metrics(page, outlinks)
         return outlinks
 
     def _fetch_url(self, site, url=None, page=None):