Version 0.0.13

.devcontainer/Dockerfile

@@ -12,11 +12,11 @@ RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/
 
 COPY ./app/pip-requirements.txt /tmp/pip-tmp/
 RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/pip-requirements.txt \
-    && rm -rf /tmp/pip-tmp
-
+    && rm -rf /tmp/pip-tmp \
+    && pip3 --disable-pip-version-check --no-cache-dir install --upgrade psutil pylint
 # [Optional] Uncomment this section to install additional OS packages.
-# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-#     && apt-get -y install --no-install-recommends <your-package-list-here>
+RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+     && apt-get -y install --no-install-recommends iputils-ping httpie curl
 
 # [Optional] Uncomment this line to install global node packages.
-# RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g <your-package-here>" 2>&1
+# RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g <your-package-here>" 2>&1

.github/workflows/01_lint_me.yml

@@ -15,7 +15,7 @@ jobs:
     steps:
       # Checks out a copy of your repository on the ubuntu-latest machine
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       # Runs the Super-Linter action
       - name: Run Super-Linter

.github/workflows/02_build_image.yml

@@ -3,23 +3,24 @@ on:
   push:
     branches:
       - 'test'
+      - 'develop'
 jobs:
   docker:
     runs-on: ubuntu-latest
     steps:
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
       - name: Login to DockerHub
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: lint
-        uses: luke142367/Docker-Lint-Action@v1.0.0
+        uses: luke142367/Docker-Lint-Action@v1.1.1
         with:
           target: Dockerfile
         env:

.github/workflows/03_build_and_push_image.yml

@@ -9,18 +9,18 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
       - name: Login to DockerHub
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: lint
-        uses: luke142367/Docker-Lint-Action@v1.0.0
+        uses: luke142367/Docker-Lint-Action@v1.1.1
         with:
           target: Dockerfile
         env:

.vscode/launch.json

@@ -9,14 +9,8 @@
       "type": "python",
       "request": "launch",
       "program": "${file}",
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "Python: Current File",
-      "type": "python",
-      "request": "launch",
-      "program": "${file}",
-      "console": "integratedTerminal"
+      "console": "integratedTerminal",
+      "env": { "PYTHONPATH": "${workspaceRoot}"}
     }
   ]
 }

.vscode/settings.json

@@ -1,5 +1,5 @@
 {
-  "python.pythonPath": "/usr/bin/python3",
+  "python.pythonPath": "/usr/bin/python3.9",
   "python.linting.pylintEnabled": true,
   "python.linting.enabled": true,
   "python.analysis.extraPaths": ["app","/usr/local/lib/python3.10/site-packages"]

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 2022-10-26 0.0.13
+
+* better `json` handling
+* switch to `httpx`
+* application no longer raises system exception when request fails
+* switch to python 3.11
+* removed requirements on distutils
+
 ## 2021-12-12 0.0.12
 
 * fixed main loop logic

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10.1-alpine AS stage
+FROM python:3.11-alpine AS stage
 COPY app/* /app/
 RUN apk add --no-cache --update -t deps curl tzdata \
   && python -m pip install --upgrade pip --no-cache-dir \
@@ -9,7 +9,7 @@ USER python-user
 WORKDIR /app
 ENV PYTHONPATH '/app/'
 ENV TZ "Europe/Warsaw"
-LABEL version="0.0.12"
+LABEL version="0.0.13"
 LABEL release="foreman_exporter"
 LABEL maintainer="marcinbojko"
 HEALTHCHECK --interval=30s --timeout=10s --retries=3 CMD curl -f http://localhost:8000 || exit 1

README.md

@@ -20,6 +20,7 @@
   - [Kubernetes example](#kubernetes-example)
   - [Kubertenes with Service Monitor example](#kubertenes-with-service-monitor-example)
   - [To DO](#to-do)
+  - [Tests](#tests)
   - [About](#about)
 
 <!-- /TOC -->
@@ -69,8 +70,9 @@ It can be used as a stats tool for configuration applying status, hosts status,
 
 ### Tested with
 
+- Foreman 2.0.x
+- Foreman 2.2.x
 - Foreman 1.22.x
-
 - Foreman 1.23.x
 
 ## Usage
@@ -205,13 +207,18 @@ If you'd like to expose your metrics to Prometheus Operator:
 
 ## To DO
 
-- replace prometheus http server with `Flask`
-- change path to /metrics
 - add more API checks (facts maybe?)
 - ~~Kubernetes setup~~
-- switch to `urllib3` instead of `requests`
 - improve python skills
 
+## Tests
+
+- pyreq
+- isort
+- flake8
+- pylint
+- pylance
+
 ## About
 
 That's a small side-project for me to learn Python3

app/foreman_exporter.py

@@ -1,21 +1,20 @@
 #!/bin/python3
 ''' foreman exporter '''
 import datetime
-import distutils.core
 import json
+import logging
 import os
+import sys
 import threading
 import time
 import urllib.parse
 import urllib.request
-import logging
-import requests
+from json.decoder import JSONDecodeError
+
+import httpx
 from packaging import version
 from prometheus_client import Summary, start_http_server
 from prometheus_client.core import REGISTRY, GaugeMetricFamily
-from requests.packages.urllib3.exceptions import InsecureRequestWarning
-
-requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
 
 # Variables
 FOREMAN_HOSTS_BODY = None
@@ -25,7 +24,7 @@
 FOREMAN_STATUS_RESPONSE = None
 FOREMAN_STATUS_BODY = None
 FOREMAN_DASHBOARD_ITEMS = []
-FOREMAN_EXPORTER_VERSION = "0.0.12"
+FOREMAN_EXPORTER_VERSION = "0.0.13"
 FOREMAN_VERSION = None
 R_API_DEPTH = "1000"                 # how much elements get with every request to /api/hosts
 
@@ -61,7 +60,7 @@
         REQUEST_PASSWORD = "api"
     # tls_verify
     if os.getenv("FOREMAN_REQUEST_TLS_VERIFY") is not None:
-        REQUEST_TLS_VERIFY = distutils.util.strtobool((os.getenv("FOREMAN_REQUEST_TLS_VERIFY")))
+        REQUEST_TLS_VERIFY = bool((os.getenv("FOREMAN_REQUEST_TLS_VERIFY")))
     else:
         REQUEST_TLS_VERIFY = False
     print(f"REQUEST_TLS_VERIFY = {REQUEST_TLS_VERIFY}")
@@ -96,9 +95,14 @@ def f_requests_status():
     global FOREMAN_STATUS_BODY
     global R_API_DEPTH
     try:
-        response = requests.get(REQUEST_URI+'api/status', auth=(REQUEST_USER, REQUEST_PASSWORD), verify=REQUEST_TLS_VERIFY, timeout=REQUEST_TIMEOUT)
+        response = httpx.get(REQUEST_URI+'api/status', auth=(REQUEST_USER, REQUEST_PASSWORD), verify=REQUEST_TLS_VERIFY, timeout=REQUEST_TIMEOUT)
         response.raise_for_status()
-        body = json.loads(response.text)
+        try:
+            body = json.loads(response.text)
+        except JSONDecodeError:
+            print(datetime.datetime.now(), f"Request at: {REQUEST_URI}api/status failed, response is not json")
+            time.sleep(1)
+            sys.exit()
         if 200 >= response.status_code <= 399:
             print(datetime.datetime.now(), f"Status request at {REQUEST_URI}api/status with code {response.status_code} took {response.elapsed.seconds} seconds")
             FOREMAN_STATUS_BODY = body
@@ -114,50 +118,60 @@ def f_requests_status():
                 pass
         else:
             print(datetime.datetime.now(), f"Response code not proper: {response.status_code}")
-    except requests.exceptions.RequestException as err:
+    except httpx.HTTPStatusError as err:
         print(datetime.datetime.now(), f"Request at: {REQUEST_URI}api/status failed with code {err}")
         time.sleep(1)
-        raise SystemExit(err) from err
+        # raise SystemExit(err) from err
 
 
 def f_requests_hosts():
     ''' Process Foreman's hosts response '''
     global FOREMAN_HOSTS_RESPONSE
     global FOREMAN_HOSTS_BODY
     try:
-        response = requests.get(REQUEST_URI+'api/hosts?per_page='+str(R_API_DEPTH), auth=(REQUEST_USER, REQUEST_PASSWORD), verify=REQUEST_TLS_VERIFY, timeout=REQUEST_TIMEOUT)
+        response = httpx.get(REQUEST_URI+'api/hosts?per_page='+str(R_API_DEPTH), auth=(REQUEST_USER, REQUEST_PASSWORD), verify=REQUEST_TLS_VERIFY, timeout=REQUEST_TIMEOUT)
         response.raise_for_status()
-        body = json.loads(response.text)
+        try:
+            body = json.loads(response.text)
+        except JSONDecodeError:
+            print(datetime.datetime.now(), f"Request at: {REQUEST_URI}api/hosts failed, response is not json")
+            time.sleep(1)
+            sys.exit()
         if 200 >= response.status_code <= 399:
             print(datetime.datetime.now(), f"Hosts request at {REQUEST_URI}api/hosts with code {response.status_code} took {response.elapsed.seconds} seconds")
             FOREMAN_HOSTS_BODY = body
             FOREMAN_HOSTS_RESPONSE = response
         else:
             print(datetime.datetime.now(), f"Response code not proper: {response.status_code}")
-    except requests.exceptions.RequestException as err:
+    except httpx.HTTPStatusError as err:
         print(datetime.datetime.now(), f"Request at: {REQUEST_URI}api/hosts failed with code {err}")
         time.sleep(1)
-        raise SystemExit(err) from err
+        # raise SystemExit(err) from err
 
 
 def f_requests_dashboard():
     ''' Process Foreman's dashboard response '''
     global FOREMAN_DASHBOARD_RESPONSE
     global FOREMAN_DASHBOARD_BODY
     try:
-        response = requests.get(REQUEST_URI+'api/dashboard?per_page=10000', auth=(REQUEST_USER, REQUEST_PASSWORD), verify=REQUEST_TLS_VERIFY, timeout=REQUEST_TIMEOUT)
+        response = httpx.get(REQUEST_URI+'api/dashboard?per_page=10000', auth=(REQUEST_USER, REQUEST_PASSWORD), verify=REQUEST_TLS_VERIFY, timeout=REQUEST_TIMEOUT)
         response.raise_for_status()
-        body = json.loads(response.text)
+        try:
+            body = json.loads(response.text)
+        except JSONDecodeError:
+            print(datetime.datetime.now(), f"Request at: {REQUEST_URI}api/dashboard failed, response is not json")
+            time.sleep(1)
+            sys.exit()
         if 200 >= response.status_code <= 399:
             print(datetime.datetime.now(), f"Dashboard request at: {REQUEST_URI}api/dashboard with code {response.status_code} took {response.elapsed.seconds} seconds")
             FOREMAN_DASHBOARD_BODY = body
             FOREMAN_DASHBOARD_RESPONSE = response
         else:
             print(datetime.datetime.now(), f"Response code not proper:  {response.status_code}")
-    except requests.exceptions.RequestException as err:
+    except httpx.HTTPStatusError as err:
         print(datetime.datetime.now(), f"Request at: {REQUEST_URI}api/dashboard failed with code {err}")
         time.sleep(1)
-        raise SystemExit(err) from err
+        # raise SystemExit(err) from err
 
 
 # register class
@@ -201,7 +215,7 @@ def collect():
         else:
             pass
         # How long the process was made
-        if FOREMAN_HOSTS_RESPONSE.elapsed.seconds is not None:
+        if FOREMAN_HOSTS_RESPONSE is not None:
             g_hosts_time = GaugeMetricFamily("foreman_exporter_hosts_request_time_seconds", 'foreman host request time seconds', labels=['foreman_hostname'])
             g_hosts_time.add_metric([REQUEST_HOSTNAME], int(FOREMAN_HOSTS_RESPONSE.elapsed.seconds))
             yield g_hosts_time
@@ -240,7 +254,7 @@ def collect():
         else:
             pass
         # How long the process (request dashboard) was made
-        if FOREMAN_DASHBOARD_RESPONSE.elapsed.seconds is not None:
+        if FOREMAN_DASHBOARD_RESPONSE is not None:
             g_dashboard_time = GaugeMetricFamily("foreman_exporter_dashboard_request_time_seconds", 'foreman dashboard request time seconds', labels=['foreman_hostname'])
             g_dashboard_time.add_metric([REQUEST_HOSTNAME], int(FOREMAN_DASHBOARD_RESPONSE.elapsed.seconds))
             yield g_dashboard_time
@@ -258,7 +272,7 @@ def collect():
         ''' Register Prometheus Metrics for Foremant's Status '''
 #        global R_API_DEPTH
         # How long the process (request dashboard) was made
-        if FOREMAN_STATUS_RESPONSE.elapsed.seconds is not None:
+        if FOREMAN_STATUS_RESPONSE is not None:
             g_status_time = GaugeMetricFamily("foreman_exporter_status_request_time_seconds", 'foreman status request time seconds', labels=['foreman_hostname'])
             g_status_time.add_metric([REQUEST_HOSTNAME], int(FOREMAN_STATUS_RESPONSE.elapsed.seconds))
             yield g_status_time

app/pip-requirements.txt

@@ -1,3 +1,4 @@
 prometheus-client
-requests
-packaging
+#requests
+packaging
+httpx

buildme.sh

@@ -24,9 +24,9 @@ if [ "$build_status" == 0 ]; then
   echo "Docker build succeed"
   rm -rfv dive.log||true
   rm -rfv ./.coverage.*||true
-  trivy --output .coverage."$version"_trivy.txt "$release":"$version"
+  trivy image --output .coverage."$version"_trivy.txt "$release":"$version"
   dive --ci "$release":"$version" > .coverage."$version"_dive.txt
-  dockle -f json -o .coverage."$version"_dockle.txt "$release":"$version"
+  sudo dockle -f json -o .coverage."$version"_dockle.txt "$release":"$version"
 else
  echo "Docker build failed, exiting now"
 fi