add brozzler_ydl_download_successes metric

internetarchive · Sep 20, 2024 · 229d53d · 229d53d
1 parent 7fbee54
commit 229d53d
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 6 deletions.
diff --git a/brozzler/metrics.py b/brozzler/metrics.py
@@ -29,6 +29,7 @@
 brozzler_ydl_urls_checked = Counter("brozzler_ydl_urls_checked", "count of urls checked by brozzler yt-dlp")
 brozzler_ydl_extract_attempts = Counter("brozzler_ydl_extract_attempts", "count of extracts attempted by brozzler yt-dlp", labelnames=["youtube_host"])
 brozzler_ydl_extract_successes = Counter("brozzler_ydl_extract_successes", "count of extracts completed by brozzler yt-dlp", labelnames=["youtube_host"])
+brozzler_ydl_download_successes = Counter("brozzler_ydl_download_successes", "count of downloads completed by brozzler yt-dlp", labelnames=["youtube_host"])
 # fmt: on
 
 

diff --git a/brozzler/ydl.py b/brozzler/ydl.py
@@ -33,7 +33,7 @@
 
 thread_local = threading.local()
 
-PROXYRACK_PROXY = ""
+YTDLP_PROXY = ""
 MAX_YTDLP_ATTEMPTS = 4
 YTDLP_WAIT = 10
 
@@ -239,6 +239,14 @@ def ydl_postprocess_hook(d):
             worker.logger.info(
                 "[ydl_postprocess_hook] postprocessor: {}".format(d["postprocessor"])
             )
+            youtube_host = (
+                "youtube.com"
+                in d["info_dict"]["webpage_url"]
+                .split("//")[-1]
+                .split("/")[0]
+                .split("?")[0]
+            )
+            metrics.brozzler_ydl_download_successes.labels(youtube_host).inc(1)
             if worker._using_warcprox(site):
                 _YoutubeDL._push_video_to_warcprox(
                     _YoutubeDL, site, d["info_dict"], d["postprocessor"]
@@ -281,7 +289,8 @@ def ydl_postprocess_hook(d):
         "youtube.com" in ytdlp_url.split("//")[-1].split("/")[0].split("?")[0]
     )
     if youtube_host:
-        ydl_opts["proxy"] = PROXYRACK_PROXY
+        ydl_opts["proxy"] = YTDLP_PROXY
+        logging.info("using yt-dlp proxy %s", YTDLP_PROXY)
 
     # skip warcprox proxying yt-dlp v.2023.07.06
     # if worker._proxy_for(site):
@@ -361,7 +370,7 @@ def _try_youtube_dl(worker, ydl, site, page):
                         "Failed after %s attempts. Error: %s", MAX_YTDLP_ATTEMPTS, e
                     )
                     raise brozzler.ProxyError(
-                        "yt-dlp hit possible proxyrack proxy error from %s" % ydl.url
+                        "yt-dlp hit possible external proxy error from %s" % ydl.url
                     )
                 else:
                     logging.info(
@@ -371,9 +380,7 @@ def _try_youtube_dl(worker, ydl, site, page):
                     )
                     time.sleep(YTDLP_WAIT)
     else:
-        raise brozzler.ProxyError(
-            "Proxyrack proxy attempt(s) failed for unknown reason(s)"
-        )
+        raise brozzler.ProxyError("Proxy attempt(s) failed for unknown reason(s)")
     logging.info("ytdlp completed successfully")
 
     _remember_videos(page, ydl.pushed_videos)