send newsflashes to telegeram in the order of appearance on website, …

…added retries to widget fetching

anyway/telegram_accident_notifications.py

@@ -50,11 +50,9 @@ def create_accident_text(newsflash_id):
     return f"{first_line}\n\n{newsflash['description']}"
 
 
-def fetch_transcription_by_widget_name(newsflash_id):
-    widgets_url = f"{ANYWAY_BASE_API_URL}/infographics-data?lang=he&news_flash_id={newsflash_id}&years_ago=5"
-    widgets_json = requests.get(widgets_url).json()
+def get_transcription_by_widget_name(widgets):
     transcription_by_widget_name = {widget["name"]: widget["data"]["text"]["transcription"]
-                                    for widget in widgets_json["widgets"]
+                                    for widget in widgets
                                     if "transcription" in widget["data"]["text"]}
     return transcription_by_widget_name
 
@@ -79,14 +77,41 @@ def publish_notification(newsflash_id, chat_id=TELEGRAM_CHANNEL_CHAT_ID):
     db.session.commit()
 
 
+def fetch_widgets_with_retries(newsflash_id, wait_times):
+    url = f"https://www.anyway.co.il/api/infographics-data?news_flash_id={newsflash_id}"
+    for attempt, wait_time in enumerate(wait_times):
+        try:
+            logging.debug(f"Attempt {attempt + 1}: Fetching data widgets for newsflash {newsflash_id}")
+            response = requests.get(url)
+            if response.ok:
+                response_json = response.json()
+                widgets = response_json.get("widgets", [])
+                if len(widgets) > 0:
+                    return widgets
+        except requests.exceptions.RequestException as e:
+            logging.debug(e)
+        time.sleep(wait_time)
+    raise RuntimeError(f"Failed to fetch data from {url}")
+
+
+def fetch_widgets(newsflash_id):
+    retry_timeouts = [10, 20, 30, 60]
+    widgets = fetch_widgets_with_retries(newsflash_id, retry_timeouts)
+    return [widget.get("name") for widget in widgets]
+
+
 def get_items_for_send(newsflash_id):
     items = []
-    transcription_by_widget_name = fetch_transcription_by_widget_name(newsflash_id)
+    retry_timeouts = [10, 20, 30, 60]
+    widgets = fetch_widgets_with_retries(newsflash_id, retry_timeouts)
+    transcription_by_widget_name = get_transcription_by_widget_name(widgets)
     urls_by_infographic_name = create_public_urls_for_infographics_images(str(newsflash_id))
-    for infographic_name, url in urls_by_infographic_name.items():
-        text = transcription_by_widget_name[infographic_name] \
-            if infographic_name in transcription_by_widget_name else None
-        items.append((url, text))
+    for widget in widgets:
+        name = widget.get("name")
+        if name in urls_by_infographic_name:
+            url = urls_by_infographic_name.get(name)
+            text = transcription_by_widget_name.get(name)
+            items.append((url, text))
     return items
 
 
@@ -104,6 +129,7 @@ def send_infographics_to_telegram(root_message_id, newsflash_id, channel_of_init
     send_after_infographics_message(bot, root_message_id, newsflash_id, linked_group)
     logging.info("notification send done")
 
+
 def extract_infographic_name_from_s3_object(s3_object_name):
     left = s3_object_name.rindex("/")
     right = s3_object_name.rindex(".")
@@ -131,4 +157,4 @@ def trigger_generate_infographics_and_send_to_telegram(newsflash_id, pre_verific
     dag_conf = {"news_flash_id": newsflash_id}
     dag_conf["chat_id"] = TELEGRAM_CHANNEL_CHAT_ID if pre_verification_chat \
         else TELEGRAM_POST_VERIFICATION_CHANNEL_CHAT_ID
-    trigger_airflow_dag("generate-and-send-infographics-images", dag_conf)
+    trigger_airflow_dag("generate-and-send-infographics-images", dag_conf)