add RemoteDisconnected retries to gold_client along with 200 error retries

[turbomam]

In my experience, I could always restart the GOLD to MongoDB process immediately after a RemoteDisconnected crash, so I would like to automate handling it.

current state

def _fetch_url(endpoint_url, params, user, passwd) -> JSON:
    attempt = 0
    while attempt < 4:
        results = requests.get(
            endpoint_url, params=params, auth=HTTPBasicAuth(user, passwd)
        )
        logging.info(f"STATUS={results.status_code}")
        if results.status_code == 200:
            return results.json()
        else:
            logging.error(
                f"API call to {endpoint_url} failed, code={results.status_code}; attempt={attempt} [pausing]"
            )
            sleep(5**attempt)
            attempt += 1
    raise Exception(f"API call to {endpoint_url} failed after {attempt} attempts")

ChatGPT suggests a few possible solutions:

Use requests Built-in Retry Adapter

from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry

def _fetch_url(endpoint_url, params, user, passwd) -> JSON:
    session = requests.Session()
    retries = Retry(
        total=4,
        backoff_factor=1,
        status_forcelist=[500, 502, 503, 504],
        allowed_methods=["GET"]
    )
    adapter = HTTPAdapter(max_retries=retries)
    session.mount('https://', adapter)
    session.mount('http://', adapter)

    response = session.get(endpoint_url, params=params, auth=HTTPBasicAuth(user, passwd), timeout=10)
    response.raise_for_status()
    return response.json()

or creating a session

import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry

def create_session():
    retries = Retry(
        total=5,  # Total retry attempts
        backoff_factor=0.3,  # Wait time between retries: 0.3, 0.6, 1.2, etc.
        status_forcelist=[500, 502, 503, 504],  # Retry on server errors
        allowed_methods=["HEAD", "GET", "OPTIONS"],  # Retry for safe methods
    )
    session = requests.Session()
    adapter = HTTPAdapter(max_retries=retries)
    session.mount("http://", adapter)
    session.mount("https://", adapter)
    return session

Then using the session in the requests

session = create_session()
response = session.get(url, params=params, auth=(user, passwd))

or manual, explicit retries. It also suggests using a sooner backoff and an explicit timeout

def _fetch_url(endpoint_url, params, user, passwd) -> JSON:
    attempt = 0
    max_attempts = 4

    while attempt < max_attempts:
        try:
            results = requests.get(
                endpoint_url, params=params, auth=HTTPBasicAuth(user, passwd), timeout=10
            )
            logging.info(f"STATUS={results.status_code}")
            
            if results.status_code == 200:
                return results.json()
            else:
                logging.error(
                    f"API call to {endpoint_url} failed with status code {results.status_code}; attempt={attempt + 1}"
                )

        except (requests.exceptions.ConnectionError, 
                requests.exceptions.Timeout, 
                requests.exceptions.HTTPError) as e:
            logging.error(f"Connection error on attempt {attempt + 1}: {e}")
        
        # Exponential backoff with a cap
        sleep(min(2 ** attempt, 60))  
        attempt += 1

    raise Exception(f"API call to {endpoint_url} failed after {max_attempts} attempts")