diff --git a/.gitignore b/.gitignore index 68e1d41f4..c4afe0ea7 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ src/.env *.key *.log *.crt +*.asc local/ # settings of editors diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ebbd284b2..982895371 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,11 +1,23 @@ repos: - repo: https://github.com/psf/black - rev: 22.6.0 + rev: 23.3.0 hooks: - id: black + language_version: python3 + args: [--line-length=142] + + - repo: https://github.com/PyCQA/flake8 + rev: 6.0.0 + hooks: + - id: flake8 + additional_dependencies: [flake8-docstrings] + args: [--max-line-length=142] + types: ['python'] - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v4.4.0 hooks: - - id: trailing-whitespace + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace diff --git a/docker/Dockerfile.publishers b/docker/Dockerfile.publishers index 50cd13b0a..6b0367b56 100644 --- a/docker/Dockerfile.publishers +++ b/docker/Dockerfile.publishers @@ -1,4 +1,4 @@ -FROM python:3.7-alpine3.14 AS build_shared +FROM python:3.9-alpine3.17 AS build_shared WORKDIR /build_shared/ @@ -8,7 +8,7 @@ RUN python -m build -FROM python:3.7-alpine3.14 AS production +FROM python:3.9-alpine3.17 AS production WORKDIR /app/ @@ -24,6 +24,10 @@ RUN pip install --no-cache-dir ./custom_packages/taranis_ng_shared-*.whl && rm - # install dependencies COPY ./src/publishers/requirements.txt /app/requirements.txt +RUN apk add --no-cache \ + swig\ + gnupg + RUN \ apk add --no-cache --virtual .build-deps build-base \ gcc \ @@ -32,6 +36,7 @@ RUN \ musl-dev \ python3-dev \ libffi-dev \ + openssl-dev \ rust && \ pip install --no-cache-dir -r /app/requirements.txt && \ apk --purge del .build-deps diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 8f0fa8eac..7e3698391 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -33,7 +33,7 @@ services: max-file: "10" core: - depends_on: + depends_on: - "redis" - "database" restart: unless-stopped @@ -46,7 +46,7 @@ services: HTTPS_PROXY: "${HTTPS_PROXY}" http_proxy: "${HTTP_PROXY}" https_proxy: "${HTTPS_PROXY}" - environment: + environment: REDIS_URL: "redis://redis" DB_URL: "database" DB_DATABASE: "taranis-ng" @@ -55,6 +55,9 @@ services: DB_POOL_SIZE: 100 DB_POOL_RECYCLE: 300 DB_POOL_TIMEOUT: 30 + TARANIS_NG_AUTHENTICATOR: "${TARANIS_NG_AUTHENTICATOR}" + LDAP_SERVER: "${LDAP_SERVER}" + LDAP_BASE_DN: "${LDAP_BASE_DN}" JWT_SECRET_KEY: "${JWT_SECRET_KEY}" OPENID_LOGOUT_URL: "" @@ -116,7 +119,7 @@ services: options: max-size: "200k" max-file: "10" - + collectors: depends_on: core: @@ -144,7 +147,7 @@ services: options: max-size: "200k" max-file: "10" - + presenters: depends_on: core: @@ -173,7 +176,7 @@ services: options: max-size: "200k" max-file: "10" - + publishers: depends_on: core: @@ -212,7 +215,7 @@ services: HTTPS_PROXY: "${HTTPS_PROXY}" http_proxy: "${HTTP_PROXY}" https_proxy: "${HTTPS_PROXY}" -# ports: +# ports: # - "8080:80" environment: NGINX_WORKERS: "4" @@ -253,7 +256,7 @@ services: image: "traefik:latest" environment: TZ: "${TZ}" - ports: + ports: - "${TARANIS_NG_HTTP_PORT}:80" - "${TARANIS_NG_HTTPS_PORT}:443" - "${TRAEFIK_MANAGEMENT_PORT}:9090" diff --git a/src/core/README.md b/src/core/README.md index 829dcb151..0a83bf2df 100644 --- a/src/core/README.md +++ b/src/core/README.md @@ -38,7 +38,10 @@ Keycloak is not needed to run test version of TaranisNG at the moment. You can u 11. In **taranis-ng-core** add environment variable TARANIS_NG_AUTHENTICATOR=openid (just for sign in) or TARANIS_NG_AUTHENTICATOR=keycloak (for identy management) 12. In **taranis-ng-core** add environment variable OPENID_LOGOUT_URL and set it according to your Keycloak installation e.g. http://127.0.0.1:8081/auth/realms/taranisng/protocol/openid-connect/logout?redirect_uri= 13. In **taranis-ng-gui** add these environment variables to activate external login: - VUE_APP_TARANIS_NG_LOGIN_URL=http://127.0.0.1:5000/api/auth/login;VUE_APP_TARANIS_NG_LOGOUT_URL=http://127.0.0.1:5000/api/auth/logout + ``` + VUE_APP_TARANIS_NG_LOGIN_URL=http://127.0.0.1:5000/api/auth/login + VUE_APP_TARANIS_NG_LOGOUT_URL=http://127.0.0.1:5000/api/auth/logout + ``` ## Keycloak example of docker-compose.yml for taranis-ng-core: ``` @@ -50,4 +53,11 @@ TARANIS_NG_KEYCLOAK_CLIENT_SECRET: "XXXXXX" TARANIS_NG_AUTHENTICATOR: "keycloak" KEYCLOAK_REALM_NAME: "taranis-ng" KEYCLOAK_USER_MANAGEMENT: "false" +``` +# **LDAP authentication** +If you prefer to authenticate users with LDAP, you need to set environment variables similarly to this: +``` +TARANIS_NG_AUTHENTICATOR: "ldap" +LDAP_SERVER: "ldaps://ldap.example.com" +LDAP_BASE_DN: "ou=people,dc=example,dc=com" ``` \ No newline at end of file diff --git a/src/core/auth/README.md b/src/core/auth/README.md new file mode 100644 index 000000000..b6263d9a8 --- /dev/null +++ b/src/core/auth/README.md @@ -0,0 +1 @@ +Place certificate for LDAP authentication in this folder and name it ldap_ca.pem \ No newline at end of file diff --git a/src/core/auth/ldap_authenticator.py b/src/core/auth/ldap_authenticator.py new file mode 100644 index 000000000..8baa25b12 --- /dev/null +++ b/src/core/auth/ldap_authenticator.py @@ -0,0 +1,56 @@ +from managers import log_manager +from auth.base_authenticator import BaseAuthenticator +from flask import request +from ldap3 import Server, Connection, ALL, Tls +import ssl +import time +import random +import os + + +class LDAPAuthenticator(BaseAuthenticator): + """Authenticates users against an LDAP server. + + Args: + BaseAuthenticator (_type_): _description_ + + Returns: + _type_: _description_ + """ + + LDAP_SERVER = os.getenv('LDAP_SERVER') + LDAP_BASE_DN = os.getenv('LDAP_BASE_DN') + LDAP_CA_CERT_PATH = 'auth/ldap_ca.pem' + if not os.path.isfile(LDAP_CA_CERT_PATH): + LDAP_CA_CERT_PATH = None + log_manager.store_auth_error_activity("No LDAP CA certificate found. LDAP authentication might not work.") + + def get_required_credentials(self): + """Gets the username and the password. + + Returns: + _type_: _description_ + """ + return ["username", "password"] + + def authenticate(self, credentials): + """Tries to authenticate the user against the LDAP server. + + Args: + credentials (_type_): _description_ + + Returns: + _type_: _description_ + """ + tls = Tls(ca_certs_file=self.LDAP_CA_CERT_PATH, validate=ssl.CERT_REQUIRED, version=ssl.PROTOCOL_TLSv1_2) + server = Server(self.LDAP_SERVER, use_ssl=True, tls=tls, get_info=ALL) + conn = Connection(server, user=f'uid={credentials["username"]},{self.LDAP_BASE_DN}', password=credentials["password"], read_only=True) + + if not conn.bind(): + data = request.get_json() + data["password"] = log_manager.sensitive_value(data["password"]) + log_manager.store_auth_error_activity("Authentication failed for user: " + credentials["username"], request_data=data) + time.sleep(random.uniform(1, 3)) + return BaseAuthenticator.generate_error() + + return BaseAuthenticator.generate_jwt(credentials["username"]) diff --git a/src/core/managers/auth_manager.py b/src/core/managers/auth_manager.py index da18bb490..e9a3ed415 100644 --- a/src/core/managers/auth_manager.py +++ b/src/core/managers/auth_manager.py @@ -11,6 +11,7 @@ from auth.keycloak_authenticator import KeycloakAuthenticator from auth.openid_authenticator import OpenIDAuthenticator from auth.password_authenticator import PasswordAuthenticator +from auth.ldap_authenticator import LDAPAuthenticator from model.collectors_node import CollectorsNode from model.news_item import NewsItem from model.osint_source import OSINTSourceGroup @@ -37,13 +38,15 @@ def initialize(app): JWTManager(app) - which = os.getenv('TARANIS_NG_AUTHENTICATOR') + which = os.getenv('TARANIS_NG_AUTHENTICATOR').casefold() if which == 'openid': current_authenticator = OpenIDAuthenticator() elif which == 'keycloak': current_authenticator = KeycloakAuthenticator() elif which == 'password': current_authenticator = PasswordAuthenticator() + elif which == 'ldap': + current_authenticator = LDAPAuthenticator() else: current_authenticator = PasswordAuthenticator() @@ -139,8 +142,8 @@ def get_user_from_api_key(): user: User object or None """ try: - if not request.headers.has_key('Authorization') or not request.headers['Authorization'].__contains__('Bearer '): - return None + if 'Authorization' not in request.headers or not request.headers['Authorization'].__contains__('Bearer '): + return None key_string = request.headers['Authorization'].replace('Bearer ', '') api_key = ApiKey.find_by_key(key_string) if not api_key: @@ -151,6 +154,7 @@ def get_user_from_api_key(): log_manager.store_auth_error_activity("Apikey check presence error: " + str(ex)) return None + def get_perm_from_user(user): """ Get user permmisions @@ -171,6 +175,7 @@ def get_perm_from_user(user): log_manager.store_auth_error_activity("Get permmision from user error: " + str(ex)) return None + def get_user_from_jwt_token(): """ Try to authenticate the user by API key @@ -197,6 +202,7 @@ def get_user_from_jwt_token(): return None return user + def get_perm_from_jwt_token(user): """ Get user permmisions @@ -218,6 +224,7 @@ def get_perm_from_jwt_token(user): log_manager.store_auth_error_activity("Get permmision from JWT error: " + str(ex)) return None + def auth_required(required_permissions, *acl_args): def auth_required_wrap(fn): @wraps(fn) @@ -264,7 +271,7 @@ def wrapper(*args, **kwargs): error = ({'error': 'not authorized'}, 401) # do we have the authorization header? - if not request.headers.has_key('Authorization'): + if 'Authorization' not in request.headers.has_key: log_manager.store_auth_error_activity("Missing Authorization header for external access") return error @@ -293,7 +300,7 @@ def wrapper(*args, **kwargs): error = ({'error': 'not authorized'}, 401) # do we have the authorization header? - if not request.headers.has_key('Authorization'): + if 'Authorization' not in request.headers: log_manager.store_auth_error_activity("Missing Authorization header for remote access") return error @@ -358,7 +365,7 @@ def decode_user_from_jwt(jwt_token): decoded = None try: decoded = jwt.decode(jwt_token, os.getenv('JWT_SECRET_KEY')) - except Exception as ex: # e.g. "Signature has expired" + except Exception as ex: # e.g. "Signature has expired" log_manager.store_auth_error_activity("Invalid JWT: " + str(ex)) if decoded is None: return None diff --git a/src/core/managers/publishers_manager.py b/src/core/managers/publishers_manager.py index 944ad78b0..374b9c188 100644 --- a/src/core/managers/publishers_manager.py +++ b/src/core/managers/publishers_manager.py @@ -1,3 +1,8 @@ +"""Manager for publishers. + +Returns: + _type_: _description_ +""" from model.publishers_node import PublishersNode from model.publisher import Publisher from model.publisher_preset import PublisherPreset @@ -7,6 +12,14 @@ def add_publishers_node(data): + """_summary_. + + Args: + data (_type_): _description_ + + Returns: + _type_: _description_ + """ node = PublishersNodeSchema.create(data) publishers_info, status_code = PublishersApi(node.api_url, node.api_key).get_publishers_info() if status_code == 200: @@ -17,6 +30,15 @@ def add_publishers_node(data): def update_publishers_node(node_id, data): + """_summary_. + + Args: + node_id (_type_): _description_ + data (_type_): _description_ + + Returns: + _type_: _description_ + """ node = PublishersNodeSchema.create(data) publishers_info, status_code = PublishersApi(node.api_url, node.api_key).get_publishers_info() if status_code == 200: @@ -27,20 +49,38 @@ def update_publishers_node(node_id, data): def add_publisher_preset(data): + """_summary_. + + Args: + data (_type_): _description_ + """ PublisherPreset.add_new(data) def publish(preset, data, message_title, message_body, recipients): + """_summary_. + + Args: + preset (_type_): _description_ + data (_type_): _description_ + message_title (_type_): _description_ + message_body (_type_): _description_ + recipients (_type_): _description_ + + Returns: + _type_: _description_ + """ publisher = preset.publisher node = publisher.node data_data = None data_mime = None if data is not None: - data_data = data['data'] - data_mime = data['mime_type'] + data_data = data["data"] + data_mime = data["mime_type"] + message_title = data["message_title"] + message_body = data["message_body"] - input_data = PublisherInput(publisher.type, preset.parameter_values, data_mime, data_data, message_title, - message_body, recipients) + input_data = PublisherInput(publisher.type, preset.parameter_values, data_mime, data_data, message_title, message_body, recipients) input_schema = PublisherInputSchema() return PublishersApi(node.api_url, node.api_key).publish(input_schema.dump(input_data)) diff --git a/src/core/requirements.txt b/src/core/requirements.txt old mode 100755 new mode 100644 index 9b5033c98..5784a289b --- a/src/core/requirements.txt +++ b/src/core/requirements.txt @@ -14,12 +14,13 @@ greenlet==1.1.1 gunicorn==20.0.4 idna==2.9 Jinja2==2.11.3 +ldap3==2.9.1 Mako==1.1.0 MarkupSafe==1.1.0 marshmallow==3.18.0 marshmallow-enum==1.5.1 psycogreen==1.0.2 -psycopg2-binary==2.8.4 +psycopg2-binary==2.9.6 PyJWT==1.7.1 python-dateutil==2.8.1 python-dotenv==0.10.3 diff --git a/src/presenters/managers/presenters_manager.py b/src/presenters/managers/presenters_manager.py index 4a96599bc..ea7780b97 100644 --- a/src/presenters/managers/presenters_manager.py +++ b/src/presenters/managers/presenters_manager.py @@ -1,26 +1,44 @@ +"""Manager of all presenters. + +Returns: + _description_ +""" from presenters.pdf_presenter import PDFPresenter from presenters.html_presenter import HTMLPresenter from presenters.text_presenter import TEXTPresenter from presenters.misp_presenter import MISPPresenter from presenters.json_presenter import JSONPresenter +from presenters.message_presenter import MESSAGEPresenter from shared.schema.presenter import PresenterInputSchema, PresenterOutputSchema presenters = {} def initialize(): + """Initialize all presenters.""" register_presenter(PDFPresenter()) register_presenter(HTMLPresenter()) register_presenter(TEXTPresenter()) register_presenter(MISPPresenter()) register_presenter(JSONPresenter()) + register_presenter(MESSAGEPresenter()) def register_presenter(presenter): + """Register a presenter. + + Arguments: + presenter -- Presenter module + """ presenters[presenter.type] = presenter def get_registered_presenters_info(): + """Get info about all presenters. + + Returns: + List with presenter type as key and info as value + """ presenters_info = [] for key in presenters: presenters_info.append(presenters[key].get_info()) @@ -29,6 +47,14 @@ def get_registered_presenters_info(): def generate(presenter_input_json): + """Generate. + + Arguments: + presenter_input_json -- JSON + + Returns: + _description_ + """ presenter_input_schema = PresenterInputSchema() presenter_input = presenter_input_schema.load(presenter_input_json) diff --git a/src/presenters/presenters/base_presenter.py b/src/presenters/presenters/base_presenter.py index 067c95511..a1f00da22 100644 --- a/src/presenters/presenters/base_presenter.py +++ b/src/presenters/presenters/base_presenter.py @@ -1,6 +1,10 @@ from shared.schema.presenter import PresenterSchema from managers import log_manager -import json, datetime, types +import json +import datetime +import types +import re + class BasePresenter: type = "BASE_PRESENTER" @@ -18,14 +22,17 @@ def json_default(value): return dict(value) else: return value.__dict__ + # helper class class AttributesObject: def toJSON(self): return json.dumps(self, default=BasePresenter.json_default, sort_keys=True, indent=4) + # helper class class ReportItemObject: def toJSON(self): return json.dumps(self, default=BasePresenter.json_default, sort_keys=True, indent=4) + def __init__(self, report_item, report_types, attribute_map): # report item itself self.name = report_item.title @@ -69,6 +76,41 @@ class InputDataObject: def toJSON(self): return json.dumps(self, default=BasePresenter.json_default, sort_keys=True, indent=4) + def get_max_tlp(self, reports): + """Returns the highest TLP value from a list of reports + + Args: + reports (list): list of reports + + Returns: + max_tlp: Highest TLP value from the list of reports + """ + color_values = { + 'WHITE': 0, + 'CLEAR': 1, + 'GREEN': 2, + 'AMBER': 3, + 'AMBER+STRICT': 4, + 'RED': 5 + } + colors = [] + + for report in reports: + if report.type == "Vulnerability Report": + colors.append(report.attrs.tlp) + + max_tlp = max(colors, key=lambda color: color_values.get(color, 0)) + if not max_tlp: + max_tlp = "CLEAR" + return max_tlp + + def add_link_prefix(self, report, letter): + pattern = r'\[(\d+)\]' + description = re.sub(pattern, lambda match: f"[{letter}{match.group(1)}]", report.attrs.description) + recommendations = re.sub(pattern, lambda match: f"[{letter}{match.group(1)}]", report.attrs.recommendations) + + return description, recommendations + def __init__(self, presenter_input): # types of report items (e.g. vuln report, disinfo report) report_types = dict() @@ -85,9 +127,21 @@ def __init__(self, presenter_input): self.product = presenter_input.product self.report_items = list() + for report in presenter_input.reports: self.report_items.append(BasePresenter.ReportItemObject(report, report_types, attribute_map)) + letter = 'A' + vul_report_count = 0 + for report in self.report_items: + if report.type == "Vulnerability Report": + report.attrs.description, report.attrs.recommendations = self.add_link_prefix(report, letter) + report.attrs.link_prefix = letter + letter = chr(ord(letter) + 1) + vul_report_count += 1 + if vul_report_count > 0: + self.product.max_tlp = self.get_max_tlp(self.report_items) + def get_info(self): info_schema = PresenterSchema() return info_schema.dump(self) diff --git a/src/presenters/presenters/message_presenter.py b/src/presenters/presenters/message_presenter.py new file mode 100644 index 000000000..23a61403e --- /dev/null +++ b/src/presenters/presenters/message_presenter.py @@ -0,0 +1,70 @@ +"""Create a message presenter. + +Returns: + _description_ +""" +import os +from base64 import b64encode +import jinja2 + +from .base_presenter import BasePresenter +from shared.schema.parameter import Parameter, ParameterType + + +class MESSAGEPresenter(BasePresenter): + """Class for MESSAGE presenter. + + Arguments: + BasePresenter -- Superclass + + Returns: + _description_ + """ + + type = "MESSAGE_PRESENTER" + name = "MESSAGE Presenter" + description = "Presenter for generating message title and body" + + parameters = [ + Parameter(0, "TITLE_TEMPLATE_PATH", "Title template", "Path of message title template file", ParameterType.STRING), + Parameter(0, "BODY_TEMPLATE_PATH", "Body template", "Path to message body template file", ParameterType.STRING), + ] + + parameters.extend(BasePresenter.parameters) + + def generate(self, presenter_input): + """Generate message parts from Jinja templates. + + Arguments: + presenter_input -- Input data for templating + + Returns: + presenter_output -- dict with keys mime_type and data with message parts as subkeys + """ + message_title_template_path = presenter_input.parameter_values_map["TITLE_TEMPLATE_PATH"] + message_body_template_path = presenter_input.parameter_values_map["BODY_TEMPLATE_PATH"] + presenter_output = {"mime_type": "text/plain", "message_title": None, "message_body": None, "data": None} + + def generate_part(template_path): + head, tail = os.path.split(template_path) + input_data = BasePresenter.generate_input_data(presenter_input) + env = jinja2.Environment(loader=jinja2.FileSystemLoader(head)) + func_dict = { + "vars": vars, + } + template = env.get_template(tail) + template.globals.update(func_dict) + output_text = template.render(data=input_data).encode() + base64_bytes = b64encode(output_text) + data = base64_bytes.decode("UTF-8") + return data + + try: + presenter_output["message_title"] = generate_part(message_title_template_path) + presenter_output["message_body"] = generate_part(message_body_template_path) + return presenter_output + + except Exception as error: + BasePresenter.print_exception(self, error) + presenter_output = {"mime_type": "text/plain", "data": b64encode(("TEMPLATING ERROR\n" + str(error)).encode()).decode("UTF-8")} + return presenter_output diff --git a/src/presenters/presenters/misp_presenter.py b/src/presenters/presenters/misp_presenter.py index e70d16554..7493882ea 100644 --- a/src/presenters/presenters/misp_presenter.py +++ b/src/presenters/presenters/misp_presenter.py @@ -41,3 +41,8 @@ def generate(self, presenter_input): return presenter_output except Exception as error: BasePresenter.print_exception(self, error) + presenter_output = { + 'mime_type': 'text/plain', + 'data': b64encode(("TEMPLATING ERROR\n"+str(error)).encode()).decode('UTF-8') + } + return presenter_output diff --git a/src/presenters/templates/email_body_template.txt b/src/presenters/templates/email_body_template.txt new file mode 100644 index 000000000..2a07bb265 --- /dev/null +++ b/src/presenters/templates/email_body_template.txt @@ -0,0 +1,25 @@ +Hi, + +{{ data.product.description }} +{% for report_item in data.report_items %}{% if report_item.type != "Vulnerability Report" %}This template cannot be used for item of type "{{ report_item.type }}". It can only handle "Vulnerability Report".{% else %} + +{{ report_item.name }} + +{{ report_item.attrs.description }} + +{% if report_item.attrs.recommendations %} +### Recommendations + +{{ report_item.attrs.recommendations }} + +{% endif %} +### CVE: {% for cve in report_item.attrs.cve %}{{ cve }}{{ ", " if not loop.last else "" }}{% endfor %} +{% if report_item.attrs.links %} +### Links +{% for entry in report_item.attrs.links %} +[{{ report_item.attrs.link_prefix }}{{ loop.index }}] - {{ entry }} +{% endfor %}{% endif %}{% endif %}{% endfor %} +-- +{{ data.product.user.name }} +Cyber Security Team +Acme Corporation diff --git a/src/presenters/templates/email_subject_template.txt b/src/presenters/templates/email_subject_template.txt new file mode 100644 index 000000000..42bdaa589 --- /dev/null +++ b/src/presenters/templates/email_subject_template.txt @@ -0,0 +1 @@ +[TLP: {{ data.product.max_tlp }}] {{ data.product.title }} diff --git a/src/presenters/templates/misp.json b/src/presenters/templates/misp.json index dede39d8c..ffe2fb611 100644 --- a/src/presenters/templates/misp.json +++ b/src/presenters/templates/misp.json @@ -1,20 +1,21 @@ -{ -{% for report_item in data.report_items %} - "threat_level_id": {% if report_item.attrs.event_threat_level == 'High' %}"1"{% elif report_item.attrs.event_threat_level == 'Medium' %}"2"{% elif report_item.attrs.event_threat_level == 'Low' %}"3"{% elif report_item.attrs.event_threat_level == 'Undefined' %}"4"{% else %}null{% endif %}, +{ {% for report_item in data.report_items %} + "Event": { + "analysis": {% if report_item.attrs.event_analysis == 'Initial' %}"0"{% elif report_item.attrs.event_analysis == 'Ongoing' %}"1"{% elif report_item.attrs.event_analysis == 'Complete' %}"2"{% else %}null{% endif %}, + "distribution": {% if report_item.attrs.event_distribution == 'Your Organisation only' %}"0"{% elif report_item.attrs.event_distribution == 'This Community Only' %}"1"{% elif report_item.attrs.event_distribution == 'Conected Communities' %}"2"{% elif report_item.attrs.event_distribution == 'All Communities' %}"3"{% elif report_item.attrs.event_distribution == 'Sharing Group' %}"4"{% else %}null{% endif %}, "info": {% if report_item.attrs.event_info %}"{{ report_item.attrs.event_info }}"{% else %}null{% endif %}, "published": false, - "distribution": {% if report_item.attrs.event_distribution == 'Your Organisation only' %}"0"{% elif report_item.attrs.event_distribution == 'This Community Only' %}"1"{% elif report_item.attrs.event_distribution == 'Conected Communities' %}"2"{% elif report_item.attrs.event_distribution == 'All Communities' %}"3"{% elif report_item.attrs.event_distribution == 'Sharing Group' %}"4"{% else %}null{% endif %}, - "analysis": {% if report_item.attrs.event_analysis == 'Initial' %}"0"{% elif report_item.attrs.event_analysis == 'Ongoing' %}"1"{% elif report_item.attrs.event_analysis == 'Complete' %}"2"{% else %}null{% endif %}, + "threat_level_id": {% if report_item.attrs.event_threat_level == 'High' %}"1"{% elif report_item.attrs.event_threat_level == 'Medium' %}"2"{% elif report_item.attrs.event_threat_level == 'Low' %}"3"{% elif report_item.attrs.event_threat_level == 'Undefined' %}"4"{% else %}null{% endif %}, + "uuid": "{{ report_item.uuid }}", "Attribute": [ - { - "type": {% if report_item.attrs.attribute_type %}"{{ report_item.attrs.attribute_type }}"{% else %}null{% endif %}, + { "category": {% if report_item.attrs.attribute_category %}"{{ report_item.attrs.attribute_category }}"{% else %}null{% endif %}, - "to_ids": {% if report_item.attrs.attribute_additional_information == 'For Intrusion Detection System' %}true{% else %}false{% endif %}, + "comment": {% if report_item.attrs.attribute_contextual_comment %}"{{ report_item.attrs.attribute_contextual_comment }}"{% else %}null{% endif %}, "disable_correlation": {% if report_item.attrs.attribute_additional_information == 'Disable Correlation' %}true{% else %}false{% endif %}, "distribution": {% if report_item.attrs.attribute_distribution == 'Your Organisation Only' %}"0"{% elif report_item.attrs.attribute_distribution == 'This Community Only' %}"1"{% elif report_item.attrs.attribute_distribution == 'Connected Communities' %}"2"{% elif report_item.attrs.attribute_distribution == 'All Communities' %}"3"{% elif report_item.attrs.attribute_distribution == 'Sharing Group' %}"4"{% elif report_item.attrs.attribute_distribution == 'Inherit Event' %}"5"{% else %}null{% endif %}, - "comment": {% if report_item.attrs.attribute_contextual_comment %}"{{ report_item.attrs.attribute_contextual_comment }}"{% else %}null{% endif %}, + "to_ids": {% if report_item.attrs.attribute_additional_information == 'For Intrusion Detection System' %}true{% else %}false{% endif %}, + "type": {% if report_item.attrs.attribute_type %}"{{ report_item.attrs.attribute_type }}"{% else %}null{% endif %}, "value": {% if report_item.attrs.attribute_value %}"{{ report_item.attrs.attribute_value }}"{% else %}null{% endif %} - } + } ] -{% endfor %} + }{{ ", " if not loop.last else "" }}{% endfor %} } diff --git a/src/publishers/crypto/README.md b/src/publishers/crypto/README.md new file mode 100644 index 000000000..41eebfb22 --- /dev/null +++ b/src/publishers/crypto/README.md @@ -0,0 +1,5 @@ +# A directory for certificates and private keys + +If you wish to sign or encrypt messages of Taranis NG, place the certificates here. Use either PEM file with both certificate and key for S/MIME or armored PGP for signing. +The signing file might be encrypted, you can specify the password in Taranis NG configuration. +For encryption include S/MIME certificate or armored PGP public key. diff --git a/src/publishers/managers/publishers_manager.py b/src/publishers/managers/publishers_manager.py index 3acfb9665..df6a1fd38 100644 --- a/src/publishers/managers/publishers_manager.py +++ b/src/publishers/managers/publishers_manager.py @@ -4,6 +4,7 @@ from publishers.wordpress_publisher import WORDPRESSPublisher from publishers.misp_publisher import MISPPublisher from shared.schema.publisher import PublisherInputSchema +from managers import log_manager publishers = {} @@ -24,6 +25,8 @@ def get_registered_publishers_info(): publishers_info = [] for key in publishers: publishers_info.append(publishers[key].get_info()) + log_manager.log_critical(publishers_info) + return publishers_info diff --git a/src/publishers/publishers/email_publisher.py b/src/publishers/publishers/email_publisher.py index bf8c70c0f..9b0e03f18 100644 --- a/src/publishers/publishers/email_publisher.py +++ b/src/publishers/publishers/email_publisher.py @@ -1,154 +1,120 @@ -import datetime -import smtplib -from email.message import Message -from email.mime.base import MIMEBase -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText -import gnupg - +"""Publisher for publishing by email.""" +from datetime import datetime +from base64 import b64decode +import os +from managers import log_manager from .base_publisher import BasePublisher from shared.schema.parameter import Parameter, ParameterType +from envelope import Envelope +import mimetypes class EMAILPublisher(BasePublisher): + """_summary_. + + Arguments: + BasePublisher -- _description_ + + Returns: + _description_ + """ + type = "EMAIL_PUBLISHER" name = "EMAIL Publisher" description = "Publisher for publishing by email" parameters = [ Parameter(0, "SMTP_SERVER", "SMTP server", "SMTP server for sending emails", ParameterType.STRING), - Parameter(0, "SMTP_SERVER_PORT", "SMTP server port", "SMTP server port for sending emails", - ParameterType.STRING), + Parameter(0, "SMTP_SERVER_PORT", "SMTP server port", "SMTP server port for sending emails", ParameterType.STRING), Parameter(0, "EMAIL_USERNAME", "Email username", "Username for email account", ParameterType.STRING), Parameter(0, "EMAIL_PASSWORD", "Email password", "Password for email account", ParameterType.STRING), - Parameter(0, "EMAIL_RECIPIENT", "Email recipient", "Email address of recipient", ParameterType.STRING), + Parameter(0, "EMAIL_SENDER", "Email sender", "Email address of the sender", ParameterType.STRING), + Parameter(0, "EMAIL_RECIPIENT", "Email recipient", "Email address of the recipient", ParameterType.STRING), Parameter(0, "EMAIL_SUBJECT", "Email subject", "Text of email subject", ParameterType.STRING), Parameter(0, "EMAIL_MESSAGE", "Email message", "Text of email message", ParameterType.STRING), - Parameter(0, "EMAIL_ENCRYPTION", "Do you want use email encrypt (yes/no)", "Turn ON/OFF email encryption", - ParameterType.STRING) + Parameter(0, "EMAIL_SIGN", "Email signature", "File used for signing or auto", ParameterType.STRING), + Parameter(0, "EMAIL_SIGN_PASSWORD", "Email signature password", "Password for signing file", ParameterType.STRING), + Parameter(0, "EMAIL_ENCRYPT", "Email encryption", "File used for encryption or auto", ParameterType.STRING), ] parameters.extend(BasePublisher.parameters) def publish(self, publisher_input): - - smtp_server = publisher_input.parameter_values_map['SMTP_SERVER'] - smtp_server_port = publisher_input.parameter_values_map['SMTP_SERVER_PORT'] - email_user = publisher_input.parameter_values_map['EMAIL_USERNAME'] - email_password = publisher_input.parameter_values_map['EMAIL_PASSWORD'] - email_recipients = publisher_input.parameter_values_map['EMAIL_RECIPIENT'] - email_subject = publisher_input.parameter_values_map['EMAIL_SUBJECT'] - email_message = publisher_input.parameter_values_map['EMAIL_MESSAGE'] - email_encryption = publisher_input.parameter_values_map['EMAIL_ENCRYPTION'] - - file = 'file_' + datetime.datetime.now().strftime("%d-%m-%Y_%H:%M") + '.pdf' - - if publisher_input.data is not None: - data = publisher_input.data[:] + """_summary_. + + Arguments: + publisher_input -- _description_ + + Returns: + _description_ + """ + smtp_server = publisher_input.parameter_values_map["SMTP_SERVER"] + smtp_server_port = publisher_input.parameter_values_map["SMTP_SERVER_PORT"] + user = publisher_input.parameter_values_map["EMAIL_USERNAME"] + password = publisher_input.parameter_values_map["EMAIL_PASSWORD"] + sender = publisher_input.parameter_values_map["EMAIL_SENDER"] + recipients = publisher_input.parameter_values_map["EMAIL_RECIPIENT"] + subject = publisher_input.parameter_values_map["EMAIL_SUBJECT"] + message = publisher_input.parameter_values_map["EMAIL_MESSAGE"] + sign = publisher_input.parameter_values_map["EMAIL_SIGN"] + sign_password = publisher_input.parameter_values_map["EMAIL_SIGN_PASSWORD"] + encrypt = publisher_input.parameter_values_map["EMAIL_ENCRYPT"] + + now = datetime.now().strftime("%Y%m%d%H%M%S") + + smtp = {"host": smtp_server, "port": smtp_server_port, "user": user, "password": password} + + envelope = Envelope() + + # if attachment data available from presenter + if publisher_input.mime_type and publisher_input.data: + attachment_mimetype = publisher_input.mime_type + attachment_extension = mimetypes.guess_extension(attachment_mimetype) + attachment_data = publisher_input.data[:] + attachment_list = [ + ( + b64decode(attachment_data), + attachment_mimetype, + f"file_{now}{attachment_extension}", + False, + ) + ] + # it is possible to attach multiple files + envelope.attach(attachment_list) + + # when title available from presenter + if publisher_input.message_title: + subject = b64decode(publisher_input.message_title).decode("UTF-8") + # when body available from presenter + if publisher_input.message_body: + message = b64decode(publisher_input.message_body).decode("UTF-8") + + if not message: + envelope.message(" ") else: - data = None - - def get_attachment(file_name): - msg_attachment = Message() - msg_attachment.add_header(_name="Content-Type", _value='application/pdf', name=file_name) - msg_attachment.add_header(_name="Content-Transfer-Encoding", _value="base64") - msg_attachment.add_header(_name="Content-Disposition", _value="attachment", filename=file_name) - msg_attachment.set_payload(data) - return msg_attachment - - def get_body(message): - msg_body = Message() - msg_body.add_header(_name="Content-Type", _value="text/plain", charset="utf-8") - msg_body.add_header(_name="Content-Transfer-Encoding", _value="quoted-printable") - msg_body.set_payload(message + 2 * "\n") - return msg_body - - def get_encrypted_email_string(email_address_recipient, file_name, message): - def get_gpg_cipher_text(string, recipient_email_address): - gpg = gnupg.GPG() - encrypted_str = str(gpg.encrypt(string, recipient_email_address)) - return encrypted_str - - msg = Message() - msg.add_header(_name="Content-Type", _value="multipart/mixed") - msg["From"] = email_user - msg["To"] = email_address_recipient - msg['Subject'] = email_subject - - msg_text = Message() - msg_text.add_header(_name="Content-Type", _value="multipart/mixed") - msg_text.add_header(_name="Content-Language", _value="en-US") - - msg_body = get_body(message) - msg_attachment = get_attachment(file_name) - - msg_text.attach(msg_body) - msg_text.attach(msg_attachment) - msg.attach(msg_text) - - pgp_msg = MIMEBase(_maintype="multipart", _subtype="encrypted", protocol="application/pgp-encrypted") - pgp_msg["From"] = email_user - pgp_msg["To"] = email_address_recipient - pgp_msg['Subject'] = email_subject - - pgp_msg_part1 = Message() - pgp_msg_part1.add_header(_name="Content-Type", _value="application/pgp-encrypted") - pgp_msg_part1.add_header(_name="Content-Description", _value="PGP/MIME version identification") - pgp_msg_part1.set_payload("Version: 2" + "\n") - - pgp_msg_part2 = Message() - pgp_msg_part2.add_header(_name="Content-Type", _value="application/octet-stream", name="encrypted.asc") - pgp_msg_part2.add_header(_name="Content-Description", _value="OpenPGP encrypted message") - pgp_msg_part2.add_header(_name="Content-Disposition", _value="inline", filename="encrypted.asc") - pgp_msg_part2.set_payload(get_gpg_cipher_text(msg.as_string(), email_address_recipient)) - - pgp_msg.attach(pgp_msg_part1) - pgp_msg.attach(pgp_msg_part2) - - return pgp_msg.as_string() + envelope.message(message) + if not subject: + envelope.subject(" ") + else: + envelope.subject(subject) + envelope.from_(sender) + envelope.to(recipients) + envelope.smtp(smtp) + + if sign == "auto": + envelope.signature(key=sign) + elif os.path.isfile(sign): + log_manager.log_info(f"Signing email with file {sign}") + envelope.signature(key=open(sign), passphrase=sign_password) + + if encrypt == "auto": + envelope.encryption(key=encrypt) + elif os.path.isfile(encrypt): + log_manager.log_info(f"Encrypting email with file {encrypt}") + envelope.encryption(key=open(encrypt)) try: - - server = smtplib.SMTP(smtp_server, smtp_server_port) - server.starttls() - server.login(email_user, email_password) - - if publisher_input.recipients is not None: - recipients = publisher_input.recipients - else: - recipients = email_recipients.split(',') - - if email_encryption.lower() == 'yes': - for recipient in recipients: - email_msg = email_message - email_msg = get_encrypted_email_string(recipient, file, email_msg) - server.sendmail(email_user, recipient, email_msg) - else: - email_msg = MIMEMultipart() - email_msg['From'] = email_user - email_msg['To'] = email_recipients - - if publisher_input.message_title is not None: - email_msg['Subject'] = publisher_input.message_title - else: - email_msg['Subject'] = email_subject - - if publisher_input.message_body is not None: - body = publisher_input.message_body - else: - body = email_message - - email_msg.attach(MIMEText(body + 2 * "\n", 'plain')) - - if data is not None: - attachment = get_attachment(file) - email_msg.attach(attachment) - - text = email_msg.as_string() - - server.sendmail(email_user, recipients, text) - - server.quit() + envelope.send() except Exception as error: BasePublisher.print_exception(self, error) diff --git a/src/publishers/requirements.txt b/src/publishers/requirements.txt index 9b81bb629..74fbb9ec9 100644 --- a/src/publishers/requirements.txt +++ b/src/publishers/requirements.txt @@ -1,21 +1,22 @@ -certifi==2019.11.28 +certifi==2023.5.7 +envelope==2.0.2 Flask==1.1.4 Flask-Cors==3.0.10 Flask-RESTful==0.3.7 gevent==21.8.0 greenlet==1.1.1 gunicorn==20.0.4 -httplib2==0.18.1 +httplib2==0.22.0 idna==2.9 marshmallow==3.18.0 marshmallow-enum==1.5.1 oauth2client==4.1.3 Jinja2==2.11.3 +M2Crypto==0.38.0 MarkupSafe==1.1.0 -paramiko==2.10.1 +paramiko==3.2.0 python-dateutil==2.8.1 python-dotenv==0.10.3 -python-gnupg==0.4.6 pytz==2019.3 requests==2.26.0 schedule==0.6.0 @@ -25,4 +26,4 @@ urllib3==1.26.7 Werkzeug==0.16.0 zope.event==4.4 zope.interface==5.1.0 -pymisp==2.4.128 +pymisp==2.4.128 \ No newline at end of file diff --git a/src/shared/shared/schema/presenter.py b/src/shared/shared/schema/presenter.py index 04657dc7e..2727de543 100644 --- a/src/shared/shared/schema/presenter.py +++ b/src/shared/shared/schema/presenter.py @@ -1,31 +1,74 @@ +"""Schema for presenters. + +Returns: + _type_: _description_ +""" from marshmallow import Schema, fields, post_load from shared.schema.parameter import ParameterSchema from shared.schema.parameter_value import ParameterValueSchema from shared.schema.report_item import ReportItemSchema from shared.schema.report_item_type import ReportItemTypeSchema +from shared.schema.user import UserSchemaBase class PresenterSchema(Schema): + """Schema for presenter. + + Args: + Schema (_type_): Base schema class. + """ + id = fields.Str() type = fields.Str() name = fields.Str() description = fields.Str() parameters = fields.List(fields.Nested(ParameterSchema)) -# schema for "presenter input product" - a dumbed down product suitable for presenters + class PresenterInputProductSchema(Schema): + """Schema for "presenter input product". + + A dumbed down product suitable for presenters. + + Args: + Schema (_type_): Base schema class. + + Returns: + _type_: _description_ + """ + title = fields.Str() description = fields.Str() product_type = fields.Str() product_type_description = fields.Str() + user = fields.Nested(UserSchemaBase, exclude=("password",)) @post_load def make(self, data, **kwargs): + """Make a PresenterInputProduct object from JSON data. + + Args: + data (_type_): JSON data. + + Returns: + _type_: PresenterInputProduct object. + """ return PresenterInputProduct(**data) -# schema for "presenter input" - a complex package of data for presenters + class PresenterInputSchema(Schema): + """Schema for "presenter input". + + A complex package of data for presenters. + + Args: + Schema (_type_): Base schema class. + + Returns: + _type_: _description_ + """ + type = fields.Str() parameter_values = fields.List(fields.Nested(ParameterValueSchema)) reports = fields.List(fields.Nested(ReportItemSchema)) @@ -34,55 +77,90 @@ class PresenterInputSchema(Schema): @post_load def make(self, data, **kwargs): + """Make a PresenterInput object from JSON data. + + Args: + data (_type_): JSON data. + + Returns: + _type_: PresenterInput object. + """ return PresenterInput(**data) -# real data holding object presented by PresenterInputProductSchema + class PresenterInputProduct: - def __init__(self, title, description, product_type, product_type_description): + """Real data holding object presented by PresenterInputProductSchema.""" + + def __init__(self, title, description, product_type, product_type_description, user): + """Initialize the "presenter input product". + + Args: + title (_type_): Title of the product. + description (_type_): Description of the product. + product_type (_type_): Basicaly name of the product. + product_type_description (_type_): Product description. + user (_type_): Data about the user who created the product. + """ self.title = title self.description = description self.product_type = product_type self.product_type_description = product_type_description + self.user = user @classmethod def make_from_product(cls, product): + """Make a PresenterInputProduct object from a Product object. + + Args: + product (_type_): Product object. + + Returns: + _type_: PresenterInputProduct object. + """ return PresenterInputProduct( - product.title, - product.description, - product.product_type.title, - product.product_type.description + product.title, product.description, product.product_type.title, product.product_type.description, product.user ) + class PresenterInput: - def __init__(self, type, product, parameter_values = None, reports = None, report_types = None): - # creating from JSON data + """Class for "presenter input".""" + + def __init__(self, type, product, parameter_values=None, reports=None, report_types=None): + """Initialize the "presenter input". + + Args: + type (_type_): Which presenter to use (e.g. PDF presenter) + product (_type_): Product consisting of report items. + parameter_values (_type_, optional): Arguments for the presenter (e.g. template path). Defaults to None. + reports (_type_, optional): Report items themselves. Defaults to None. + report_types (_type_, optional): Description of the report item types used. Defaults to None. + """ + # Creating from JSON data. if parameter_values is not None: self.load(type, product, parameter_values, reports, report_types) return - # creating from OBJECTS - - # PRESENTER - # - which presenter to use (e.g. PDF presenter) + # Creating from OBJECTS self.type = type - # - arguments for the presenter (e.g. template path) self.parameter_values = product.product_type.parameter_values - - # REPORT ITEMS - # - report items themselves self.reports = product.report_items - # - description of the report item types used report_types = {} for report in self.reports: report_types[report.report_item_type.id] = report.report_item_type self.report_types = list(report_types.values()) - - # PRODUCT self.product = PresenterInputProduct.make_from_product(product) def load(self, type, product, parameter_values, reports, report_types): + """Load data from JSON. + + Args: + type (_type_): _description_ + product (_type_): _description_ + parameter_values (_type_): The same arguments, parsed differently. + reports (_type_): _description_ + report_types (_type_): _description_ + """ self.type = type self.parameter_values = parameter_values - # - the same arguments, parsed differently self.parameter_values_map = dict() for parameter_value in self.parameter_values: self.parameter_values_map.update({parameter_value.parameter.key: parameter_value.value}) @@ -92,15 +170,46 @@ def load(self, type, product, parameter_values, reports, report_types): class PresenterOutputSchema(Schema): + """Schema for "presenter output". + + Args: + Schema (_type_): Base schema class. + + Returns: + _type_: _description_ + """ + mime_type = fields.Str() data = fields.Str() + message_body = fields.Str() + message_title = fields.Str() @post_load def make(self, data, **kwargs): + """Make a PresenterOutput object from JSON data. + + Args: + data (_type_): JSON data. + + Returns: + _type_: PresenterOutput object. + """ return PresenterOutput(**data) class PresenterOutput: - def __init__(self, mime_type, data): + """Class for "presenter output".""" + + def __init__(self, mime_type, data, message_body, message_title): + """Initialize the "presenter output". + + Args: + mime_type (_type_): MIME type of the output. + data (_type_): The data itself. + message_body (_type_): Body of the message. + message_title (_type_): Title of the message. + """ self.mime_type = mime_type self.data = data + self.message_body = message_body + self.message_title = message_title