Flower monitoring dashboard for Celery

.fides/fides.toml

@@ -50,15 +50,15 @@ task_retry_backoff = 1
 subject_identity_verification_required = false
 task_retry_count = 0
 task_retry_delay = 1
-use_dsr_3_0 = false
+use_dsr_3_0 = true
 
 [admin_ui]
 enabled = true
 
 [celery]
 event_queue_prefix = "fides_worker"
 task_default_queue = "fides"
-task_always_eager = true
+task_always_eager = false
 
 [notifications]
 notification_service_type = "mailgun"

docker-compose.yml

@@ -155,6 +155,19 @@ services:
         target: /usr/local/etc/redis
         read_only: False
 
+  flower:
+    image: mher/flower:2.0
+    command: celery flower --broker=redis://:redispassword@redis:6379/0
+    ports:
+      - "5555:5555"
+    depends_on:
+      - redis
+      - worker
+    environment:
+      - FLOWER_BASIC_AUTH=admin:adminpassword # Optional: basic auth protection
+      - CELERY_BROKER_URL=redis://:redispassword@redis:6379/0
+      - CELERY_RESULT_BACKEND=redis://:redispassword@redis:6379/0
+
 volumes:
   postgres: null
 

noxfiles/dev_nox.py

@@ -51,6 +51,7 @@ def dev(session: Session) -> None:
         - pc = Build and run the Privacy Center
         - remote_debug = Run with remote debugging enabled (see docker-compose.remote-debug.yml)
         - worker = Run a Fides worker
+        - flower = Run Flower monitoring dashboard for Celery
         - child = Run a Fides child node
         - <datastore(s)> = Run a test datastore (e.g. 'mssql', 'mongodb')
 
@@ -64,6 +65,15 @@ def dev(session: Session) -> None:
     if "worker" in session.posargs:
         session.run("docker", "compose", "up", "--wait", "worker", external=True)
 
+    if "flower" in session.posargs:
+        # Only start Flower if worker is also enabled
+        if "worker" in session.posargs:
+            session.run("docker", "compose", "up", "-d", "flower", external=True)
+        else:
+            session.error(
+                "Flower requires the worker service. Please add 'worker' to your arguments."
+            )
+
     datastores = [
         datastore for datastore in session.posargs if datastore in ALL_DATASTORES
     ] or None

requirements.txt

@@ -19,6 +19,7 @@ fastapi[all]==0.111.0
 fastapi-pagination[sqlalchemy]==0.12.25
 fideslog==1.2.10
 firebase-admin==5.3.0
+flower==2.0.1
 GitPython==3.1.41
 httpx==0.23.1
 hvac==0.11.2