Removing dependency of flask-sse

docker/Dockerfile.core

@@ -1,14 +1,23 @@
-FROM python:3.7-alpine3.14 AS build_shared
+FROM python:3.10-alpine3.14 AS build_shared
 
 WORKDIR /build_shared/
 
 RUN pip install --no-cache-dir build
 COPY ./src/shared/. .
 RUN python -m build
 
+RUN apk add --no-cache  \
+    gcc \
+    build-base\
+    libc-dev\
+    linux-headers
+
+COPY ./src/core/sse/forward.c .
+
+RUN gcc -o forward forward.c
 
 
-FROM python:3.7-alpine3.14 AS production
+FROM python:3.10-alpine3.14 AS production
 
 WORKDIR /app/
 
@@ -36,6 +45,10 @@ RUN \
     apk add --no-cache --virtual .build-deps \
     gcc \
     g++ \
+    build-base\
+    libc-dev\
+    zlib-dev \
+    linux-headers \
     make \
     glib-dev \
     musl-dev \
@@ -45,6 +58,9 @@ RUN \
     pip install --no-cache-dir -r /app/requirements.txt && \
     apk --purge del .build-deps
 
+
+COPY --from=build_shared /build_shared/forward /usr/local/bin/forward
+
 COPY ./docker/start.sh /start.sh
 RUN chmod +x /start.sh
 

docker/prestart_core.sh

@@ -2,6 +2,9 @@
 
 echo "Running inside /app/prestart.sh..."
 
+echo "Running sse forward in the background..."
+/usr/local/bin/forward --sender-port 5000 --client-port 5001 &
+
 echo "Running migrations..."
 /app/db_migration.py db upgrade head
 

src/core/api/__init__.py

@@ -1,2 +1,2 @@
 __all__ = ['collectors', 'assess', 'auth', 'config', 'analyze', 'publish',
-           'user', 'assets', 'bots', 'isalive', 'remote', 'dashboard']
+           'user', 'assets', 'bots', 'isalive', 'remote', 'dashboard', 'sse']

src/core/api/analyze.py

@@ -2,7 +2,8 @@
 from flask import request, jsonify, send_file
 from flask_restful import Resource
 
-from managers import asset_manager, auth_manager, sse_manager, log_manager
+from managers import asset_manager, auth_manager, log_manager
+from managers.sse_manager import sse_manager
 from managers.auth_manager import auth_required, ACLCheck
 from model import attribute, report_item, report_item_type
 from model.permission import Permission

src/core/api/assess.py

@@ -2,7 +2,8 @@
 from flask import request, send_file
 from flask_restful import Resource
 
-from managers import auth_manager, sse_manager, log_manager
+from managers import auth_manager, log_manager
+from managers.sse_manager import sse_manager
 from managers.auth_manager import ACLCheck, auth_required
 from model import news_item, osint_source
 from model.permission import Permission

src/core/api/bots.py

@@ -1,7 +1,8 @@
 from flask import request
 from flask_restful import Resource, reqparse
 
-from managers import sse_manager, log_manager
+from managers import log_manager
+from managers.sse_manager import sse_manager
 from managers.auth_manager import api_key_required
 from model import bot_preset, news_item, word_list
 

src/core/api/collectors.py

@@ -1,7 +1,7 @@
 from flask import request
 from flask_restful import Resource, reqparse
 
-from managers import sse_manager
+from managers.sse_manager import sse_manager
 from managers.auth_manager import api_key_required
 from managers.log_manager import log_debug
 from model import osint_source, collectors_node, news_item

src/core/api/config.py

@@ -3,8 +3,9 @@
 from flask import request, send_file
 from flask_restful import Resource
 
-from managers import auth_manager, sse_manager, remote_manager, presenters_manager, \
+from managers import auth_manager, remote_manager, presenters_manager, \
     publishers_manager, bots_manager, external_auth_manager, log_manager, collectors_manager
+from managers.sse_manager import sse_manager
 from managers.auth_manager import auth_required, get_user_from_jwt
 from model import acl_entry, remote, presenters_node, publisher_preset, publishers_node, \
     bots_node, bot_preset, attribute, collectors_node, organization, osint_source, product_type, \

src/core/api/sse.py

@@ -0,0 +1,80 @@
+from flask_restful import Resource
+import socket
+import json
+from flask import Response, stream_with_context, Flask
+from flask import request, abort
+from managers import auth_manager, bots_manager, time_manager, remote_manager
+
+
+class TaranisSSE(Resource):
+
+    @stream_with_context
+    def stream(self):
+        try:
+            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            sock.settimeout(60)
+        except Exception as e:
+            print(f"Failed to create socket: {e}")
+            return
+
+        try:
+            sock.connect(('localhost', 5001))
+            sock.settimeout(1)
+        except Exception as e:
+            print(f"Failed to connect to server: {e}")
+            sock.close()
+            return
+
+        buffer = ""
+        try:
+            data = b""
+            while True:
+                try:
+                    b = sock.recv(1)
+                except socket.timeout:
+                    yield ":\n\n"
+                    continue
+
+                if not b:
+                    break
+                data += b
+                try:
+                    char = data.decode('utf-8')
+                    data = b""
+                except UnicodeDecodeError:
+                    yield ":\n\n"
+                    continue
+
+                buffer += char
+
+                if char == '}':
+                    try:
+                        json_data = json.loads(buffer)
+                    except json.JSONDecodeError:
+                        continue
+
+                    yield f"event: {json_data['event']}\ndata: {json.dumps(json_data['data'])}\n\n"
+                    buffer = ""
+        except Exception as e:
+            print(f"Error during data reception or processing: {e}")
+        finally:
+            sock.close()
+
+    def get(self):
+        try:
+            if request.args.get('jwt') is not None:
+                if auth_manager.decode_user_from_jwt(request.args.get('jwt')) is None:
+                    return "", 403
+            elif request.args.get('api_key') is not None:
+                if bots_manager.verify_api_key(request.args.get('api_key')) is False:
+                    return "", 403
+            else:
+                return "", 403
+
+            return Response(self.stream(), mimetype="text/event-stream")
+        except Exception as e:
+            return str(e), 500
+
+
+def initialize(api):
+    api.add_resource(TaranisSSE, "/sse")

src/core/app.py

@@ -5,6 +5,7 @@
 from model import *  # just until all new model classes are used regularly
 
 
+
 def create_app():
     app = Flask(__name__)
     app.config.from_object('config.Config')
@@ -17,7 +18,6 @@ def create_app():
 
         auth_manager.initialize(app)
         api_manager.initialize(app)
-
         sse_manager.initialize(app)
         remote_manager.initialize(app)
         tagcloud_manager.initialize(app)

src/core/managers/api_manager.py

@@ -1,5 +1,4 @@
 from flask_restful import Api
-
 from api import *
 
 
@@ -11,6 +10,7 @@ def initialize(app):
     collectors.initialize(api)
     isalive.initialize(api)
     config.initialize(api)
+    sse.initialize(api)
     analyze.initialize(api)
     publish.initialize(api)
     user.initialize(api)

src/core/managers/remote_manager.py

@@ -52,7 +52,7 @@ def run(self):
                                         {'last_sync_time': data['last_sync_time']})
 
                                     with EventThread.app.app_context():
-                                        sse_manager.news_items_updated()
+                                        sse_manager.sse_manager.news_items_updated()
 
                     elif event.event == 'remote_access_report_items_updated':
                         if self.remote_node.sync_report_items:
@@ -68,7 +68,7 @@ def run(self):
                                         {'last_sync_time': data['last_sync_time']})
 
                                     with EventThread.app.app_context():
-                                        sse_manager.report_items_updated()
+                                        sse_manager.sse_manager.report_items_updated()
 
             except Exception:
                 time.sleep(5)