From 0976be1534ebf1a5fad565c1ac35bef0209576a8 Mon Sep 17 00:00:00 2001 From: ekneg54 Date: Thu, 27 Jun 2024 11:06:39 +0200 Subject: [PATCH 01/14] update changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96cf82f38..cf9f10bc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,12 +3,15 @@ ## next release ### Breaking ### Features + ### Improvements * a result object was added to processors and pipelines * each processor returns an object including the processor name, generated extra_data, warnings and errors * the pipeline returns an object with the list of all processor result objects +* add deployment examples for compose and helm +* remove quickstart setup ### Bugfix From ff029a73649a151b03b2a527bfe56c277ddd9c7b Mon Sep 17 00:00:00 2001 From: ekneg54 Date: Fri, 12 Jul 2024 11:02:00 +0200 Subject: [PATCH 02/14] move quickstart to examples/compose --- .github/secret_scanning.yml | 6 +-- CHANGELOG.md | 2 +- README.md | 2 +- .../programaticly_start_logprep.rst | 8 ++-- doc/source/examples/compose.rst | 38 +++++++-------- .../compose}/docker-compose.yml | 34 +++++++------- .../exampledata/config/credentials.yml | 10 ++-- .../exampledata/config/dummy-output.yml | 0 .../exampledata/config/fda/config.json | 0 .../exampledata/config/fda/default.conf | 0 .../config/grafana/dashboards/kafka.json | 0 .../grafana/dashboards/logprep-dashboard.json | 0 .../dashboards/logprep-processors.json | 0 .../grafana/dashboards/logprep-provider.yml | 0 .../config/grafana/dashboards/sample.yaml | 0 .../config/grafana/datasources/prometheus.yml | 0 .../config/grafana/datasources/sample.yaml | 0 .../exampledata/config/http_pipeline.yml | 0 .../config/keycloak/logprep_realm_export.json | 0 .../exampledata/config/nginx/conf.d/.htpasswd | 0 .../config/nginx/conf.d/default.conf | 0 .../config/nginx/mtls.conf.d/ca.crt | 0 .../config/nginx/mtls.conf.d/ca.key | 0 .../config/nginx/mtls.conf.d/client.crt | 0 .../config/nginx/mtls.conf.d/client.csr | 0 .../config/nginx/mtls.conf.d/client.key | 0 .../config/nginx/mtls.conf.d/default.conf | 0 .../config/nginx/mtls.conf.d/server.crt | 0 .../config/nginx/mtls.conf.d/server.csr | 0 .../config/nginx/mtls.conf.d/server.key | 0 .../exampledata/config/nginx/nginx.conf | 0 .../exampledata/config/pipeline.yml | 38 +++++++-------- .../config/postgresql/keycloak_db.sql | 0 .../config/prometheus/prometheus.yml | 0 .../config/ucl/general.example.json | 0 .../config/ucl/keycloak.example.json | 0 .../exampledata/config/ucl/nginx.example.conf | 0 .../exampledata/config/user_password.txt | 0 .../input_logdata/logclass/config.yaml | 0 .../input_logdata/logclass/test_input.jsonl | 0 .../exampledata/models/model.zip | Bin .../rules/amides/generic/amides_generic.yml | 0 .../rules/amides/specific/amides_specific.yml | 0 .../rules/dissector/generic/example_rule.yml | 0 .../rules/dissector/specific/example_rule.yml | 0 .../rules/dropper/generic/example_rule.yml | 0 .../rules/dropper/specific/example_rule.yml | 0 .../rules/labeler/generic/example_rule.yml | 0 .../exampledata/rules/labeler/schema.json | 0 .../rules/labeler/specific/example_rule.yml | 0 .../rules/pre_detector/alert_ips.yml | 0 .../pre_detector/generic/example_rule.yml | 0 .../pre_detector/specific/example_rule.yml | 0 .../rules/pre_detector/tree_config.json | 0 .../pseudonymizer/example_analyst_pub.pem | 0 .../pseudonymizer/example_depseudo_pub.pem | 0 .../pseudonymizer/generic/example_rule.yml | 0 .../rules/pseudonymizer/regex_mapping.yml | 0 .../pseudonymizer/specific/example_rule.yml | 0 logprep/connector/http/input.py | 2 +- logprep/metrics/metrics.py | 2 +- logprep/processor/amides/processor.py | 2 +- logprep/util/configuration.py | 44 +++++++++--------- tests/acceptance/test_full_configuration.py | 6 +-- .../requester/generic_rules/requester.json | 2 +- .../requester/specific_rules/requester.json | 2 +- tests/unit/connector/test_real_kafka.py | 2 +- tests/unit/test_quickstart.py | 10 ++-- tests/unit/test_run_logprep.py | 4 +- tests/unit/util/test_configuration.py | 36 +++++++------- tests/unit/util/test_getter.py | 4 +- 71 files changed, 126 insertions(+), 128 deletions(-) rename {quickstart => examples/compose}/docker-compose.yml (86%) rename {quickstart => examples}/exampledata/config/credentials.yml (65%) rename {quickstart => examples}/exampledata/config/dummy-output.yml (100%) rename {quickstart => examples}/exampledata/config/fda/config.json (100%) rename {quickstart => examples}/exampledata/config/fda/default.conf (100%) rename {quickstart => examples}/exampledata/config/grafana/dashboards/kafka.json (100%) rename {quickstart => examples}/exampledata/config/grafana/dashboards/logprep-dashboard.json (100%) rename {quickstart => examples}/exampledata/config/grafana/dashboards/logprep-processors.json (100%) rename {quickstart => examples}/exampledata/config/grafana/dashboards/logprep-provider.yml (100%) rename {quickstart => examples}/exampledata/config/grafana/dashboards/sample.yaml (100%) rename {quickstart => examples}/exampledata/config/grafana/datasources/prometheus.yml (100%) rename {quickstart => examples}/exampledata/config/grafana/datasources/sample.yaml (100%) rename {quickstart => examples}/exampledata/config/http_pipeline.yml (100%) rename {quickstart => examples}/exampledata/config/keycloak/logprep_realm_export.json (100%) rename {quickstart => examples}/exampledata/config/nginx/conf.d/.htpasswd (100%) rename {quickstart => examples}/exampledata/config/nginx/conf.d/default.conf (100%) rename {quickstart => examples}/exampledata/config/nginx/mtls.conf.d/ca.crt (100%) rename {quickstart => examples}/exampledata/config/nginx/mtls.conf.d/ca.key (100%) rename {quickstart => examples}/exampledata/config/nginx/mtls.conf.d/client.crt (100%) rename {quickstart => examples}/exampledata/config/nginx/mtls.conf.d/client.csr (100%) rename {quickstart => examples}/exampledata/config/nginx/mtls.conf.d/client.key (100%) rename {quickstart => examples}/exampledata/config/nginx/mtls.conf.d/default.conf (100%) rename {quickstart => examples}/exampledata/config/nginx/mtls.conf.d/server.crt (100%) rename {quickstart => examples}/exampledata/config/nginx/mtls.conf.d/server.csr (100%) rename {quickstart => examples}/exampledata/config/nginx/mtls.conf.d/server.key (100%) rename {quickstart => examples}/exampledata/config/nginx/nginx.conf (100%) rename {quickstart => examples}/exampledata/config/pipeline.yml (67%) rename {quickstart => examples}/exampledata/config/postgresql/keycloak_db.sql (100%) rename {quickstart => examples}/exampledata/config/prometheus/prometheus.yml (100%) rename {quickstart => examples}/exampledata/config/ucl/general.example.json (100%) rename {quickstart => examples}/exampledata/config/ucl/keycloak.example.json (100%) rename {quickstart => examples}/exampledata/config/ucl/nginx.example.conf (100%) rename {quickstart => examples}/exampledata/config/user_password.txt (100%) rename {quickstart => examples}/exampledata/input_logdata/logclass/config.yaml (100%) rename {quickstart => examples}/exampledata/input_logdata/logclass/test_input.jsonl (100%) rename {quickstart => examples}/exampledata/models/model.zip (100%) rename {quickstart => examples}/exampledata/rules/amides/generic/amides_generic.yml (100%) rename {quickstart => examples}/exampledata/rules/amides/specific/amides_specific.yml (100%) rename {quickstart => examples}/exampledata/rules/dissector/generic/example_rule.yml (100%) rename {quickstart => examples}/exampledata/rules/dissector/specific/example_rule.yml (100%) rename {quickstart => examples}/exampledata/rules/dropper/generic/example_rule.yml (100%) rename {quickstart => examples}/exampledata/rules/dropper/specific/example_rule.yml (100%) rename {quickstart => examples}/exampledata/rules/labeler/generic/example_rule.yml (100%) rename {quickstart => examples}/exampledata/rules/labeler/schema.json (100%) rename {quickstart => examples}/exampledata/rules/labeler/specific/example_rule.yml (100%) rename {quickstart => examples}/exampledata/rules/pre_detector/alert_ips.yml (100%) rename {quickstart => examples}/exampledata/rules/pre_detector/generic/example_rule.yml (100%) rename {quickstart => examples}/exampledata/rules/pre_detector/specific/example_rule.yml (100%) rename {quickstart => examples}/exampledata/rules/pre_detector/tree_config.json (100%) rename {quickstart => examples}/exampledata/rules/pseudonymizer/example_analyst_pub.pem (100%) rename {quickstart => examples}/exampledata/rules/pseudonymizer/example_depseudo_pub.pem (100%) rename {quickstart => examples}/exampledata/rules/pseudonymizer/generic/example_rule.yml (100%) rename {quickstart => examples}/exampledata/rules/pseudonymizer/regex_mapping.yml (100%) rename {quickstart => examples}/exampledata/rules/pseudonymizer/specific/example_rule.yml (100%) diff --git a/.github/secret_scanning.yml b/.github/secret_scanning.yml index 3d036d4d4..3e87bf78b 100644 --- a/.github/secret_scanning.yml +++ b/.github/secret_scanning.yml @@ -1,4 +1,4 @@ paths-ignore: - - "quickstart/exampledata/config/credentials.yml" - - "quickstart/exampledata/config/nginx/mtls.conf.d/*.key" - - "quickstart/exampledata/config/nginx/mtls.conf.d/*.crt" + - "examples/exampledata/config/credentials.yml" + - "examples/exampledata/config/nginx/mtls.conf.d/*.key" + - "examples/exampledata/config/nginx/mtls.conf.d/*.crt" diff --git a/CHANGELOG.md b/CHANGELOG.md index cf9f10bc3..dd3ffde10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -319,7 +319,7 @@ * add possibility to convert hex to int in `calculator` processor with new added function `from_hex` * add metrics on rule level -* add grafana example dashboards under `quickstart/exampledata/config/grafana/dashboards` +* add grafana example dashboards under `examples/exampledata/config/grafana/dashboards` * add new configuration field `id` for all rules to identify rules in metrics and logs - if no `id` is given, the `id` will be generated in a stable way - add verification of rule `id` uniqueness on processor level over both rule trees to ensure metrics are counted correctly on rule level diff --git a/README.md b/README.md index 8e33a94b9..bf7d2371c 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ allowing further applications besides log handling. This readme provides basic information about the following topics: - [About Logprep](#about-logprep) - [Getting Started](https://logprep.readthedocs.io/en/latest/getting_started.html) -- [Docker Quickstart](https://logprep.readthedocs.io/en/latest/getting_started.html#logprep-quickstart-environment) +- [Deployment Examples](https://logprep.readthedocs.io/en/latest/examples/index.html) - [Event Generation](https://logprep.readthedocs.io/en/latest/user_manual/execution.html#event-generation) - [Documentation](https://logprep.readthedocs.io/en/latest) - [Contributing](#contributing) diff --git a/doc/source/development/programaticly_start_logprep.rst b/doc/source/development/programaticly_start_logprep.rst index 4c1426ab7..51de7c9a4 100644 --- a/doc/source/development/programaticly_start_logprep.rst +++ b/doc/source/development/programaticly_start_logprep.rst @@ -23,10 +23,10 @@ An example with input connector and preprocessors could look like this: "predetector": { "type": "pre_detector", "specific_rules": [ - "quickstart/exampledata/rules/pre_detector/specific" + "examples/exampledata/rules/pre_detector/specific" ], "generic_rules": [ - "quickstart/exampledata/rules/pre_detector/generic" + "examples/exampledata/rules/pre_detector/generic" ], "pre_detector_topic": "output_topic" } @@ -61,10 +61,10 @@ An example without input connector and preprocessors could look like this: "predetector": { "type": "pre_detector", "specific_rules": [ - "quickstart/exampledata/rules/pre_detector/specific" + "examples/exampledata/rules/pre_detector/specific" ], "generic_rules": [ - "quickstart/exampledata/rules/pre_detector/generic" + "examples/exampledata/rules/pre_detector/generic" ], "pre_detector_topic": "output_topic" } diff --git a/doc/source/examples/compose.rst b/doc/source/examples/compose.rst index 9e1747f80..98c9769bc 100644 --- a/doc/source/examples/compose.rst +++ b/doc/source/examples/compose.rst @@ -5,7 +5,7 @@ Docker Compose Example Deployment To demonstrate the functionality of logprep this repo comes with a complete `kafka`, `logprep` and `opensearch` stack. To get it running `docker` with compose support must be first installed. -The docker compose file is located in the directory `quickstart`. +The docker compose file is located in the directory `examples/compose`. A prerequisite is to run `sysctl -w vm.max_map_count=262144`, otherwise Opensearch might not properly start. @@ -14,7 +14,7 @@ The environment can either be started with a Logprep container or without one: Run without Logprep Container (default) --------------------------------------- - 1. Run from within the `quickstart` directory: + 1. Run from within the `examples/compose` directory: .. code-block:: bash @@ -25,14 +25,14 @@ Run without Logprep Container (default) .. code-block:: bash - logprep run quickstart/exampledata/config/pipeline.yml + logprep run examples/exampledata/config/pipeline.yml If logprep is run with the metrics enabled, the necessary environment variable has to be set first: .. code-block:: bash export PROMETHEUS_MULTIPROC_DIR="tmp/logprep" - logprep run quickstart/exampledata/config/pipeline.yml + logprep run examples/exampledata/config/pipeline.yml @@ -40,7 +40,7 @@ Run without Logprep Container (default) Run with Logprep Container -------------------------- - * Run from within the `quickstart` directory: + * Run from within the `examples/compose` directory: .. code-block:: bash @@ -50,7 +50,7 @@ Run with Logprep Container Run with getting config from http server with basic authentication ------------------------------------------------------------------ - * Run from within the `quickstart` directory: + * Run from within the `examples/compose` directory: .. code-block:: bash @@ -60,14 +60,14 @@ Run with getting config from http server with basic authentication .. code-block:: bash - export LOGPREP_CREDENTIALS_FILE="quickstart/exampledata/config/credentials.yml" + export LOGPREP_CREDENTIALS_FILE="examples/exampledata/config/credentials.yml" logprep run http://localhost:8081/config/pipeline.yml Run with getting config from http server with mTLS authentication ----------------------------------------------------------------- - * Run from within the `quickstart` directory: + * Run from within the `examples/compose` directory: .. code-block:: bash @@ -77,12 +77,12 @@ Run with getting config from http server with mTLS authentication .. code-block:: bash - export LOGPREP_CREDENTIALS_FILE="quickstart/exampledata/config/credentials.yml" + export LOGPREP_CREDENTIALS_FILE="examples/exampledata/config/credentials.yml" logprep run https://localhost:8082/config/pipeline.yml -Interacting with the Quickstart Environment -------------------------------------------- +Interacting with the Compose Environment +---------------------------------------- The start up takes a few seconds to complete, but once everything is up and running it is possible to write JSON events into Kafka and read the processed events in @@ -111,9 +111,9 @@ UCL Postgres: `localhost:5432` ucl ucl ====================== ================= ======================== ======================= The example rules that are used in the docker instance of Logprep can be found -in `quickstart/exampledata/rules`. +in `examples/exampledata/rules`. Example events that trigger for the example rules can be found in -`quickstart/exampledata/input_logdata/logclass/test_input.jsonl`. +`examples/exampledata/input_logdata/logclass/test_input.jsonl`. These events can be added to Kafka with the following command: .. code-block:: bash @@ -133,12 +133,12 @@ Utilizing FDA and UCL If you want to try out the FDA and UCL you first have to do some preparations. -0. Run the quickstart setup with the :code:`oauth2` profile: +0. Run the example compose setup with the :code:`oauth2` profile: :code:`docker compose --profile oauth2 up -d`. 1. Sign into the keycloak admin panel and create a logprep user in the :code:`logprep` realm. Make sure that the user is part of the :code:`logprep-admin` group and has a password. If you choose a password other than :code:`logprep` you have to update the credentials file - :code:`quickstart/exampledata/config/credentials.yml`, such that the password of + :code:`examples/exampledata/config/credentials.yml`, such that the password of :code:`http://localhost:3001` and :code:`http://localhost:3002` reflects your choice. 2. You have to login to the FDA with the previously created user and create a release, as well as your first logclass. It is also necessary to add an example event to this logclass in order @@ -148,7 +148,7 @@ If you want to try out the FDA and UCL you first have to do some preparations. your created logprep user and then configure required Use-Cases. At the current moment these configuration are not yet processed by logprep though, as the ucl only provides a mock endpoint which doesn't contain your Use-Case configurations. -4. Set the env :code:`LOGPREP_CREDENTIALS_FILE` to :code:`quickstart/exampledata/config/credentials.yml` +4. Set the env :code:`LOGPREP_CREDENTIALS_FILE` to :code:`examples/exampledata/config/credentials.yml` Once you have set everything up you can run logprep with the following command. Just consider that the first :code:`pipeline.yml` argument is used to define a proper :code:`input` @@ -157,10 +157,10 @@ you should ensure that the :code:`stage` and :code:`loglcass` are set properly. .. code-block:: bash - logprep run quickstart/exampledata/config/pipeline.yml "http://localhost:3002/api/v1/pipelines?stage=prod&logclass=ExampleClass" "http://localhost:3001/api/v1/general-predetection" + logprep run examples/exampledata/config/pipeline.yml "http://localhost:3002/api/v1/pipelines?stage=prod&logclass=ExampleClass" "http://localhost:3001/api/v1/general-predetection" .. note:: - If you did use the quickstart setup before and run into problems it is advised to first pull + If you did use the example compose setup before and run into problems it is advised to first pull all images again to update them to the latest version: - :code:`docker compose -f ./quickstart/docker-compose.yml pull`. + :code:`docker compose -f ./example/compose/docker-compose.yml pull`. diff --git a/quickstart/docker-compose.yml b/examples/compose/docker-compose.yml similarity index 86% rename from quickstart/docker-compose.yml rename to examples/compose/docker-compose.yml index 63f893e0d..4cb8e7b4d 100644 --- a/quickstart/docker-compose.yml +++ b/examples/compose/docker-compose.yml @@ -100,21 +100,21 @@ services: - kafka - opensearch volumes: - - ../quickstart/:/home/logprep/quickstart/ + - ../../examples/:/home/logprep/examples/ tmpfs: - /tmp/logprep/prometheus_multiproc entrypoint: - logprep - run - - /home/logprep/quickstart/exampledata/config/pipeline.yml + - /home/logprep/examples/exampledata/config/pipeline.yml grafana: image: bitnami/grafana:latest container_name: grafana ports: - "3000:3000" volumes: - - ../quickstart/exampledata/config/grafana/datasources:/opt/bitnami/grafana/conf/provisioning/datasources - - ../quickstart/exampledata/config/grafana/dashboards:/opt/bitnami/grafana/conf/provisioning/dashboards + - ../../examples/exampledata/config/grafana/datasources:/opt/bitnami/grafana/conf/provisioning/datasources + - ../../examples/exampledata/config/grafana/dashboards:/opt/bitnami/grafana/conf/provisioning/dashboards networks: - prometheus prometheus: @@ -123,7 +123,7 @@ services: ports: - "9090:9090" volumes: - - ../quickstart/exampledata/config/prometheus/prometheus.yml:/opt/bitnami/prometheus/conf/prometheus.yml + - ../../examples/exampledata/config/prometheus/prometheus.yml:/opt/bitnami/prometheus/conf/prometheus.yml networks: - kafka - prometheus @@ -138,9 +138,9 @@ services: ports: - "8081:8081" volumes: - - ../quickstart/exampledata:/usr/share/nginx/html:ro - - ../quickstart/exampledata/config/nginx/nginx.conf:/etc/nginx/nginx.conf:ro - - ../quickstart/exampledata/config/nginx/conf.d:/etc/nginx/conf.d:ro + - ../../examples/exampledata:/usr/share/nginx/html:ro + - ../../examples/exampledata/config/nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ../../examples/exampledata/config/nginx/conf.d:/etc/nginx/conf.d:ro mtls-config: image: nginx:latest container_name: mtls-config @@ -149,9 +149,9 @@ services: ports: - "8082:8082" volumes: - - ../quickstart/exampledata:/usr/share/nginx/html:ro - - ../quickstart/exampledata/config/nginx/nginx.conf:/etc/nginx/nginx.conf:ro - - ../quickstart/exampledata/config/nginx/mtls.conf.d:/etc/nginx/conf.d:ro + - ../../examples/exampledata:/usr/share/nginx/html:ro + - ../../examples/exampledata/config/nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ../../examples/exampledata/config/nginx/mtls.conf.d:/etc/nginx/conf.d:ro keycloak: image: bitnami/keycloak:latest container_name: keycloak @@ -160,7 +160,7 @@ services: profiles: - oauth2 volumes: - - ../quickstart/exampledata/config/keycloak:/opt/keycloak/data/import + - ../../examples/exampledata/config/keycloak:/opt/keycloak/data/import environment: KEYCLOAK_ADMIN_USER: admin KEYCLOAK_ADMIN_PASSWORD: admin @@ -263,8 +263,8 @@ services: ports: - "3002:80" volumes: - - ../quickstart/exampledata/config/fda/config.json:/usr/share/nginx/html/assets/config/keycloak/config.json - - ../quickstart/exampledata/config/fda/default.conf:/etc/nginx/conf.d/default.conf + - ../../examples/exampledata/config/fda/config.json:/usr/share/nginx/html/assets/config/keycloak/config.json + - ../../examples/exampledata/config/fda/default.conf:/etc/nginx/conf.d/default.conf networks: - fda ucl-api: @@ -338,9 +338,9 @@ services: ports: - "3001:80" volumes: - - ../quickstart/exampledata/config/ucl/nginx.example.conf:/etc/nginx/conf.d/default.conf - - ../quickstart/exampledata/config/ucl/general.example.json:/usr/share/nginx/html/assets/config/general/config.json - - ../quickstart/exampledata/config/ucl/keycloak.example.json:/usr/share/nginx/html/assets/config/keycloak/config.json + - ../../examples/exampledata/config/ucl/nginx.example.conf:/etc/nginx/conf.d/default.conf + - ../../examples/exampledata/config/ucl/general.example.json:/usr/share/nginx/html/assets/config/general/config.json + - ../../examples/exampledata/config/ucl/keycloak.example.json:/usr/share/nginx/html/assets/config/keycloak/config.json networks: - ucl diff --git a/quickstart/exampledata/config/credentials.yml b/examples/exampledata/config/credentials.yml similarity index 65% rename from quickstart/exampledata/config/credentials.yml rename to examples/exampledata/config/credentials.yml index 6a8c59496..8ca296a8a 100644 --- a/quickstart/exampledata/config/credentials.yml +++ b/examples/exampledata/config/credentials.yml @@ -15,14 +15,14 @@ getter: username: user password: password "https://localhost:8082": - client_key: quickstart/exampledata/config/nginx/mtls.conf.d/client.key - cert: quickstart/exampledata/config/nginx/mtls.conf.d/client.crt - ca_cert: quickstart/exampledata/config/nginx/mtls.conf.d/ca.crt + client_key: examples/exampledata/config/nginx/mtls.conf.d/client.key + cert: examples/exampledata/config/nginx/mtls.conf.d/client.crt + ca_cert: examples/exampledata/config/nginx/mtls.conf.d/ca.crt input: endpoints: /auth-json: username: user - password_file: quickstart/exampledata/config/user_password.txt + password_file: examples/exampledata/config/user_password.txt /lab/123/ABC/auditlog: username: user - password_file: quickstart/exampledata/config/user_password.txt + password_file: examples/exampledata/config/user_password.txt diff --git a/quickstart/exampledata/config/dummy-output.yml b/examples/exampledata/config/dummy-output.yml similarity index 100% rename from quickstart/exampledata/config/dummy-output.yml rename to examples/exampledata/config/dummy-output.yml diff --git a/quickstart/exampledata/config/fda/config.json b/examples/exampledata/config/fda/config.json similarity index 100% rename from quickstart/exampledata/config/fda/config.json rename to examples/exampledata/config/fda/config.json diff --git a/quickstart/exampledata/config/fda/default.conf b/examples/exampledata/config/fda/default.conf similarity index 100% rename from quickstart/exampledata/config/fda/default.conf rename to examples/exampledata/config/fda/default.conf diff --git a/quickstart/exampledata/config/grafana/dashboards/kafka.json b/examples/exampledata/config/grafana/dashboards/kafka.json similarity index 100% rename from quickstart/exampledata/config/grafana/dashboards/kafka.json rename to examples/exampledata/config/grafana/dashboards/kafka.json diff --git a/quickstart/exampledata/config/grafana/dashboards/logprep-dashboard.json b/examples/exampledata/config/grafana/dashboards/logprep-dashboard.json similarity index 100% rename from quickstart/exampledata/config/grafana/dashboards/logprep-dashboard.json rename to examples/exampledata/config/grafana/dashboards/logprep-dashboard.json diff --git a/quickstart/exampledata/config/grafana/dashboards/logprep-processors.json b/examples/exampledata/config/grafana/dashboards/logprep-processors.json similarity index 100% rename from quickstart/exampledata/config/grafana/dashboards/logprep-processors.json rename to examples/exampledata/config/grafana/dashboards/logprep-processors.json diff --git a/quickstart/exampledata/config/grafana/dashboards/logprep-provider.yml b/examples/exampledata/config/grafana/dashboards/logprep-provider.yml similarity index 100% rename from quickstart/exampledata/config/grafana/dashboards/logprep-provider.yml rename to examples/exampledata/config/grafana/dashboards/logprep-provider.yml diff --git a/quickstart/exampledata/config/grafana/dashboards/sample.yaml b/examples/exampledata/config/grafana/dashboards/sample.yaml similarity index 100% rename from quickstart/exampledata/config/grafana/dashboards/sample.yaml rename to examples/exampledata/config/grafana/dashboards/sample.yaml diff --git a/quickstart/exampledata/config/grafana/datasources/prometheus.yml b/examples/exampledata/config/grafana/datasources/prometheus.yml similarity index 100% rename from quickstart/exampledata/config/grafana/datasources/prometheus.yml rename to examples/exampledata/config/grafana/datasources/prometheus.yml diff --git a/quickstart/exampledata/config/grafana/datasources/sample.yaml b/examples/exampledata/config/grafana/datasources/sample.yaml similarity index 100% rename from quickstart/exampledata/config/grafana/datasources/sample.yaml rename to examples/exampledata/config/grafana/datasources/sample.yaml diff --git a/quickstart/exampledata/config/http_pipeline.yml b/examples/exampledata/config/http_pipeline.yml similarity index 100% rename from quickstart/exampledata/config/http_pipeline.yml rename to examples/exampledata/config/http_pipeline.yml diff --git a/quickstart/exampledata/config/keycloak/logprep_realm_export.json b/examples/exampledata/config/keycloak/logprep_realm_export.json similarity index 100% rename from quickstart/exampledata/config/keycloak/logprep_realm_export.json rename to examples/exampledata/config/keycloak/logprep_realm_export.json diff --git a/quickstart/exampledata/config/nginx/conf.d/.htpasswd b/examples/exampledata/config/nginx/conf.d/.htpasswd similarity index 100% rename from quickstart/exampledata/config/nginx/conf.d/.htpasswd rename to examples/exampledata/config/nginx/conf.d/.htpasswd diff --git a/quickstart/exampledata/config/nginx/conf.d/default.conf b/examples/exampledata/config/nginx/conf.d/default.conf similarity index 100% rename from quickstart/exampledata/config/nginx/conf.d/default.conf rename to examples/exampledata/config/nginx/conf.d/default.conf diff --git a/quickstart/exampledata/config/nginx/mtls.conf.d/ca.crt b/examples/exampledata/config/nginx/mtls.conf.d/ca.crt similarity index 100% rename from quickstart/exampledata/config/nginx/mtls.conf.d/ca.crt rename to examples/exampledata/config/nginx/mtls.conf.d/ca.crt diff --git a/quickstart/exampledata/config/nginx/mtls.conf.d/ca.key b/examples/exampledata/config/nginx/mtls.conf.d/ca.key similarity index 100% rename from quickstart/exampledata/config/nginx/mtls.conf.d/ca.key rename to examples/exampledata/config/nginx/mtls.conf.d/ca.key diff --git a/quickstart/exampledata/config/nginx/mtls.conf.d/client.crt b/examples/exampledata/config/nginx/mtls.conf.d/client.crt similarity index 100% rename from quickstart/exampledata/config/nginx/mtls.conf.d/client.crt rename to examples/exampledata/config/nginx/mtls.conf.d/client.crt diff --git a/quickstart/exampledata/config/nginx/mtls.conf.d/client.csr b/examples/exampledata/config/nginx/mtls.conf.d/client.csr similarity index 100% rename from quickstart/exampledata/config/nginx/mtls.conf.d/client.csr rename to examples/exampledata/config/nginx/mtls.conf.d/client.csr diff --git a/quickstart/exampledata/config/nginx/mtls.conf.d/client.key b/examples/exampledata/config/nginx/mtls.conf.d/client.key similarity index 100% rename from quickstart/exampledata/config/nginx/mtls.conf.d/client.key rename to examples/exampledata/config/nginx/mtls.conf.d/client.key diff --git a/quickstart/exampledata/config/nginx/mtls.conf.d/default.conf b/examples/exampledata/config/nginx/mtls.conf.d/default.conf similarity index 100% rename from quickstart/exampledata/config/nginx/mtls.conf.d/default.conf rename to examples/exampledata/config/nginx/mtls.conf.d/default.conf diff --git a/quickstart/exampledata/config/nginx/mtls.conf.d/server.crt b/examples/exampledata/config/nginx/mtls.conf.d/server.crt similarity index 100% rename from quickstart/exampledata/config/nginx/mtls.conf.d/server.crt rename to examples/exampledata/config/nginx/mtls.conf.d/server.crt diff --git a/quickstart/exampledata/config/nginx/mtls.conf.d/server.csr b/examples/exampledata/config/nginx/mtls.conf.d/server.csr similarity index 100% rename from quickstart/exampledata/config/nginx/mtls.conf.d/server.csr rename to examples/exampledata/config/nginx/mtls.conf.d/server.csr diff --git a/quickstart/exampledata/config/nginx/mtls.conf.d/server.key b/examples/exampledata/config/nginx/mtls.conf.d/server.key similarity index 100% rename from quickstart/exampledata/config/nginx/mtls.conf.d/server.key rename to examples/exampledata/config/nginx/mtls.conf.d/server.key diff --git a/quickstart/exampledata/config/nginx/nginx.conf b/examples/exampledata/config/nginx/nginx.conf similarity index 100% rename from quickstart/exampledata/config/nginx/nginx.conf rename to examples/exampledata/config/nginx/nginx.conf diff --git a/quickstart/exampledata/config/pipeline.yml b/examples/exampledata/config/pipeline.yml similarity index 67% rename from quickstart/exampledata/config/pipeline.yml rename to examples/exampledata/config/pipeline.yml index 71fd7e70b..b45ff72f2 100644 --- a/quickstart/exampledata/config/pipeline.yml +++ b/examples/exampledata/config/pipeline.yml @@ -15,26 +15,26 @@ metrics: pipeline: - labelername: type: labeler - schema: quickstart/exampledata/rules/labeler/schema.json + schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true specific_rules: - - quickstart/exampledata/rules/labeler/specific + - examples/exampledata/rules/labeler/specific generic_rules: - - quickstart/exampledata/rules/labeler/generic + - examples/exampledata/rules/labeler/generic - dissector: type: dissector specific_rules: - - quickstart/exampledata/rules/dissector/specific/ + - examples/exampledata/rules/dissector/specific/ generic_rules: - - quickstart/exampledata/rules/dissector/generic/ + - examples/exampledata/rules/dissector/generic/ - dropper: type: dropper specific_rules: - - quickstart/exampledata/rules/dropper/specific + - examples/exampledata/rules/dropper/specific generic_rules: - - quickstart/exampledata/rules/dropper/generic + - examples/exampledata/rules/dropper/generic - filter: "test_dropper" dropper: drop: @@ -44,37 +44,37 @@ pipeline: - pre_detector: type: pre_detector specific_rules: - - quickstart/exampledata/rules/pre_detector/specific + - examples/exampledata/rules/pre_detector/specific generic_rules: - - quickstart/exampledata/rules/pre_detector/generic + - examples/exampledata/rules/pre_detector/generic outputs: - opensearch: sre - tree_config: quickstart/exampledata/rules/pre_detector/tree_config.json - alert_ip_list_path: quickstart/exampledata/rules/pre_detector/alert_ips.yml + tree_config: examples/exampledata/rules/pre_detector/tree_config.json + alert_ip_list_path: examples/exampledata/rules/pre_detector/alert_ips.yml - amides: type: amides specific_rules: - - quickstart/exampledata/rules/amides/specific + - examples/exampledata/rules/amides/specific generic_rules: - - quickstart/exampledata/rules/amides/generic - models_path: quickstart/exampledata/models/model.zip + - examples/exampledata/rules/amides/generic + models_path: examples/exampledata/models/model.zip num_rule_attributions: 10 max_cache_entries: 1000000 decision_threshold: 0.32 - pseudonymizer: type: pseudonymizer - pubkey_analyst: quickstart/exampledata/rules/pseudonymizer/example_analyst_pub.pem - pubkey_depseudo: quickstart/exampledata/rules/pseudonymizer/example_depseudo_pub.pem - regex_mapping: quickstart/exampledata/rules/pseudonymizer/regex_mapping.yml + pubkey_analyst: examples/exampledata/rules/pseudonymizer/example_analyst_pub.pem + pubkey_depseudo: examples/exampledata/rules/pseudonymizer/example_depseudo_pub.pem + regex_mapping: examples/exampledata/rules/pseudonymizer/regex_mapping.yml hash_salt: a_secret_tasty_ingredient outputs: - opensearch: pseudonyms specific_rules: - - quickstart/exampledata/rules/pseudonymizer/specific/ + - examples/exampledata/rules/pseudonymizer/specific/ generic_rules: - - quickstart/exampledata/rules/pseudonymizer/generic/ + - examples/exampledata/rules/pseudonymizer/generic/ max_cached_pseudonyms: 1000000 - calculator: diff --git a/quickstart/exampledata/config/postgresql/keycloak_db.sql b/examples/exampledata/config/postgresql/keycloak_db.sql similarity index 100% rename from quickstart/exampledata/config/postgresql/keycloak_db.sql rename to examples/exampledata/config/postgresql/keycloak_db.sql diff --git a/quickstart/exampledata/config/prometheus/prometheus.yml b/examples/exampledata/config/prometheus/prometheus.yml similarity index 100% rename from quickstart/exampledata/config/prometheus/prometheus.yml rename to examples/exampledata/config/prometheus/prometheus.yml diff --git a/quickstart/exampledata/config/ucl/general.example.json b/examples/exampledata/config/ucl/general.example.json similarity index 100% rename from quickstart/exampledata/config/ucl/general.example.json rename to examples/exampledata/config/ucl/general.example.json diff --git a/quickstart/exampledata/config/ucl/keycloak.example.json b/examples/exampledata/config/ucl/keycloak.example.json similarity index 100% rename from quickstart/exampledata/config/ucl/keycloak.example.json rename to examples/exampledata/config/ucl/keycloak.example.json diff --git a/quickstart/exampledata/config/ucl/nginx.example.conf b/examples/exampledata/config/ucl/nginx.example.conf similarity index 100% rename from quickstart/exampledata/config/ucl/nginx.example.conf rename to examples/exampledata/config/ucl/nginx.example.conf diff --git a/quickstart/exampledata/config/user_password.txt b/examples/exampledata/config/user_password.txt similarity index 100% rename from quickstart/exampledata/config/user_password.txt rename to examples/exampledata/config/user_password.txt diff --git a/quickstart/exampledata/input_logdata/logclass/config.yaml b/examples/exampledata/input_logdata/logclass/config.yaml similarity index 100% rename from quickstart/exampledata/input_logdata/logclass/config.yaml rename to examples/exampledata/input_logdata/logclass/config.yaml diff --git a/quickstart/exampledata/input_logdata/logclass/test_input.jsonl b/examples/exampledata/input_logdata/logclass/test_input.jsonl similarity index 100% rename from quickstart/exampledata/input_logdata/logclass/test_input.jsonl rename to examples/exampledata/input_logdata/logclass/test_input.jsonl diff --git a/quickstart/exampledata/models/model.zip b/examples/exampledata/models/model.zip similarity index 100% rename from quickstart/exampledata/models/model.zip rename to examples/exampledata/models/model.zip diff --git a/quickstart/exampledata/rules/amides/generic/amides_generic.yml b/examples/exampledata/rules/amides/generic/amides_generic.yml similarity index 100% rename from quickstart/exampledata/rules/amides/generic/amides_generic.yml rename to examples/exampledata/rules/amides/generic/amides_generic.yml diff --git a/quickstart/exampledata/rules/amides/specific/amides_specific.yml b/examples/exampledata/rules/amides/specific/amides_specific.yml similarity index 100% rename from quickstart/exampledata/rules/amides/specific/amides_specific.yml rename to examples/exampledata/rules/amides/specific/amides_specific.yml diff --git a/quickstart/exampledata/rules/dissector/generic/example_rule.yml b/examples/exampledata/rules/dissector/generic/example_rule.yml similarity index 100% rename from quickstart/exampledata/rules/dissector/generic/example_rule.yml rename to examples/exampledata/rules/dissector/generic/example_rule.yml diff --git a/quickstart/exampledata/rules/dissector/specific/example_rule.yml b/examples/exampledata/rules/dissector/specific/example_rule.yml similarity index 100% rename from quickstart/exampledata/rules/dissector/specific/example_rule.yml rename to examples/exampledata/rules/dissector/specific/example_rule.yml diff --git a/quickstart/exampledata/rules/dropper/generic/example_rule.yml b/examples/exampledata/rules/dropper/generic/example_rule.yml similarity index 100% rename from quickstart/exampledata/rules/dropper/generic/example_rule.yml rename to examples/exampledata/rules/dropper/generic/example_rule.yml diff --git a/quickstart/exampledata/rules/dropper/specific/example_rule.yml b/examples/exampledata/rules/dropper/specific/example_rule.yml similarity index 100% rename from quickstart/exampledata/rules/dropper/specific/example_rule.yml rename to examples/exampledata/rules/dropper/specific/example_rule.yml diff --git a/quickstart/exampledata/rules/labeler/generic/example_rule.yml b/examples/exampledata/rules/labeler/generic/example_rule.yml similarity index 100% rename from quickstart/exampledata/rules/labeler/generic/example_rule.yml rename to examples/exampledata/rules/labeler/generic/example_rule.yml diff --git a/quickstart/exampledata/rules/labeler/schema.json b/examples/exampledata/rules/labeler/schema.json similarity index 100% rename from quickstart/exampledata/rules/labeler/schema.json rename to examples/exampledata/rules/labeler/schema.json diff --git a/quickstart/exampledata/rules/labeler/specific/example_rule.yml b/examples/exampledata/rules/labeler/specific/example_rule.yml similarity index 100% rename from quickstart/exampledata/rules/labeler/specific/example_rule.yml rename to examples/exampledata/rules/labeler/specific/example_rule.yml diff --git a/quickstart/exampledata/rules/pre_detector/alert_ips.yml b/examples/exampledata/rules/pre_detector/alert_ips.yml similarity index 100% rename from quickstart/exampledata/rules/pre_detector/alert_ips.yml rename to examples/exampledata/rules/pre_detector/alert_ips.yml diff --git a/quickstart/exampledata/rules/pre_detector/generic/example_rule.yml b/examples/exampledata/rules/pre_detector/generic/example_rule.yml similarity index 100% rename from quickstart/exampledata/rules/pre_detector/generic/example_rule.yml rename to examples/exampledata/rules/pre_detector/generic/example_rule.yml diff --git a/quickstart/exampledata/rules/pre_detector/specific/example_rule.yml b/examples/exampledata/rules/pre_detector/specific/example_rule.yml similarity index 100% rename from quickstart/exampledata/rules/pre_detector/specific/example_rule.yml rename to examples/exampledata/rules/pre_detector/specific/example_rule.yml diff --git a/quickstart/exampledata/rules/pre_detector/tree_config.json b/examples/exampledata/rules/pre_detector/tree_config.json similarity index 100% rename from quickstart/exampledata/rules/pre_detector/tree_config.json rename to examples/exampledata/rules/pre_detector/tree_config.json diff --git a/quickstart/exampledata/rules/pseudonymizer/example_analyst_pub.pem b/examples/exampledata/rules/pseudonymizer/example_analyst_pub.pem similarity index 100% rename from quickstart/exampledata/rules/pseudonymizer/example_analyst_pub.pem rename to examples/exampledata/rules/pseudonymizer/example_analyst_pub.pem diff --git a/quickstart/exampledata/rules/pseudonymizer/example_depseudo_pub.pem b/examples/exampledata/rules/pseudonymizer/example_depseudo_pub.pem similarity index 100% rename from quickstart/exampledata/rules/pseudonymizer/example_depseudo_pub.pem rename to examples/exampledata/rules/pseudonymizer/example_depseudo_pub.pem diff --git a/quickstart/exampledata/rules/pseudonymizer/generic/example_rule.yml b/examples/exampledata/rules/pseudonymizer/generic/example_rule.yml similarity index 100% rename from quickstart/exampledata/rules/pseudonymizer/generic/example_rule.yml rename to examples/exampledata/rules/pseudonymizer/generic/example_rule.yml diff --git a/quickstart/exampledata/rules/pseudonymizer/regex_mapping.yml b/examples/exampledata/rules/pseudonymizer/regex_mapping.yml similarity index 100% rename from quickstart/exampledata/rules/pseudonymizer/regex_mapping.yml rename to examples/exampledata/rules/pseudonymizer/regex_mapping.yml diff --git a/quickstart/exampledata/rules/pseudonymizer/specific/example_rule.yml b/examples/exampledata/rules/pseudonymizer/specific/example_rule.yml similarity index 100% rename from quickstart/exampledata/rules/pseudonymizer/specific/example_rule.yml rename to examples/exampledata/rules/pseudonymizer/specific/example_rule.yml diff --git a/logprep/connector/http/input.py b/logprep/connector/http/input.py index 5f412d568..55690affd 100644 --- a/logprep/connector/http/input.py +++ b/logprep/connector/http/input.py @@ -45,7 +45,7 @@ endpoints: /firstendpoint: username: user - password_file: quickstart/exampledata/config/user_password.txt + password_file: examples/exampledata/config/user_password.txt /second*: username: user password: secret_password diff --git a/logprep/metrics/metrics.py b/logprep/metrics/metrics.py index 99ae80813..caf12bcf7 100644 --- a/logprep/metrics/metrics.py +++ b/logprep/metrics/metrics.py @@ -3,7 +3,7 @@ :code:`logprep_number_of_processed_events_total` or :code:`logprep_processing_time_per_event_sum`. Examples of grafana dashboards can be found in `the logprep github repo -`_ +`_ Configuration ============= diff --git a/logprep/processor/amides/processor.py b/logprep/processor/amides/processor.py index d6652d105..5e0a6de87 100644 --- a/logprep/processor/amides/processor.py +++ b/logprep/processor/amides/processor.py @@ -28,7 +28,7 @@ Since there is a plethora of different SIEM event types, the current implementation focuses on events that provide process command lines. Command lines are most commonly targeted by SIEM rules while they are also highly vulnerable to evasions. The rules and models for AMIDES provided in the -quickstart example are for Sysmon Process Creation events. In general, the Amides rule format allows +deployment examples are for Sysmon Process Creation events. In general, the Amides rule format allows to create rules for other event types that provide process command lines, e.g. Process Creation events generated by Windows Security Auditing. diff --git a/logprep/util/configuration.py b/logprep/util/configuration.py index f6f8f09c8..3dc5a6a90 100644 --- a/logprep/util/configuration.py +++ b/logprep/util/configuration.py @@ -53,26 +53,26 @@ pipeline: - labelername: type: labeler - schema: quickstart/exampledata/rules/labeler/schema.json + schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true specific_rules: - - quickstart/exampledata/rules/labeler/specific + - examples/exampledata/rules/labeler/specific generic_rules: - - quickstart/exampledata/rules/labeler/generic + - examples/exampledata/rules/labeler/generic - dissectorname: type: dissector specific_rules: - - quickstart/exampledata/rules/dissector/specific/ + - examples/exampledata/rules/dissector/specific/ generic_rules: - - quickstart/exampledata/rules/dissector/generic/ + - examples/exampledata/rules/dissector/generic/ - dropper: type: dropper specific_rules: - - quickstart/exampledata/rules/dropper/specific + - examples/exampledata/rules/dropper/specific generic_rules: - - quickstart/exampledata/rules/dropper/generic + - examples/exampledata/rules/dropper/generic - filter: "test_dropper" dropper: drop: @@ -82,37 +82,37 @@ - pre_detector: type: pre_detector specific_rules: - - quickstart/exampledata/rules/pre_detector/specific + - examples/exampledata/rules/pre_detector/specific generic_rules: - - quickstart/exampledata/rules/pre_detector/generic + - examples/exampledata/rules/pre_detector/generic outputs: - opensearch: sre - tree_config: quickstart/exampledata/rules/pre_detector/tree_config.json - alert_ip_list_path: quickstart/exampledata/rules/pre_detector/alert_ips.yml + tree_config: examples/exampledata/rules/pre_detector/tree_config.json + alert_ip_list_path: examples/exampledata/rules/pre_detector/alert_ips.yml - amides: type: amides specific_rules: - - quickstart/exampledata/rules/amides/specific + - examples/exampledata/rules/amides/specific generic_rules: - - quickstart/exampledata/rules/amides/generic - models_path: quickstart/exampledata/models/model.zip + - examples/exampledata/rules/amides/generic + models_path: examples/exampledata/models/model.zip num_rule_attributions: 10 max_cache_entries: 1000000 decision_threshold: 0.32 - pseudonymizer: type: pseudonymizer - pubkey_analyst: quickstart/exampledata/rules/pseudonymizer/example_analyst_pub.pem - pubkey_depseudo: quickstart/exampledata/rules/pseudonymizer/example_depseudo_pub.pem - regex_mapping: quickstart/exampledata/rules/pseudonymizer/regex_mapping.yml + pubkey_analyst: examples/exampledata/rules/pseudonymizer/example_analyst_pub.pem + pubkey_depseudo: examples/exampledata/rules/pseudonymizer/example_depseudo_pub.pem + regex_mapping: examples/exampledata/rules/pseudonymizer/regex_mapping.yml hash_salt: a_secret_tasty_ingredient outputs: - opensearch: pseudonyms specific_rules: - - quickstart/exampledata/rules/pseudonymizer/specific/ + - examples/exampledata/rules/pseudonymizer/specific/ generic_rules: - - quickstart/exampledata/rules/pseudonymizer/generic/ + - examples/exampledata/rules/pseudonymizer/generic/ max_cached_pseudonyms: 1000000 - calculator: @@ -173,12 +173,12 @@ pipeline: - labelername: type: labeler - schema: quickstart/exampledata/rules/labeler/schema.json + schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true specific_rules: - - quickstart/exampledata/rules/labeler/specific + - examples/exampledata/rules/labeler/specific generic_rules: - - quickstart/exampledata/rules/labeler/generic" + - examples/exampledata/rules/labeler/generic" export LOGPREP_OUTPUT=" output: kafka: diff --git a/tests/acceptance/test_full_configuration.py b/tests/acceptance/test_full_configuration.py index a09dd3fb6..5175c31ab 100644 --- a/tests/acceptance/test_full_configuration.py +++ b/tests/acceptance/test_full_configuration.py @@ -87,12 +87,12 @@ def test_start_of_logprep_from_http_with_templated_url_and_config(): pipeline: - labelername: type: labeler - schema: quickstart/exampledata/rules/labeler/schema.json + schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true specific_rules: - - quickstart/exampledata/rules/labeler/specific + - examples/exampledata/rules/labeler/specific generic_rules: - - quickstart/exampledata/rules/labeler/generic + - examples/exampledata/rules/labeler/generic """, "LOGPREP_OUTPUT": """ output: diff --git a/tests/testdata/unit/requester/generic_rules/requester.json b/tests/testdata/unit/requester/generic_rules/requester.json index 2564b1e24..d7154a894 100644 --- a/tests/testdata/unit/requester/generic_rules/requester.json +++ b/tests/testdata/unit/requester/generic_rules/requester.json @@ -2,7 +2,7 @@ { "filter": "message1", "requester": { - "url": "http://localhost:32000/quickstart/exampledata/config/pipeline.yml", + "url": "http://localhost:32000/examples/exampledata/config/pipeline.yml", "method": "GET" } } diff --git a/tests/testdata/unit/requester/specific_rules/requester.json b/tests/testdata/unit/requester/specific_rules/requester.json index 1ae41c251..701c4e95e 100644 --- a/tests/testdata/unit/requester/specific_rules/requester.json +++ b/tests/testdata/unit/requester/specific_rules/requester.json @@ -2,7 +2,7 @@ { "filter": "message", "requester": { - "url": "http://localhost:32000/quickstart/exampledata/config/pipeline.yml", + "url": "http://localhost:32000/examples/exampledata/config/pipeline.yml", "method": "GET" } } diff --git a/tests/unit/connector/test_real_kafka.py b/tests/unit/connector/test_real_kafka.py index 25eb6a909..a35d00f8b 100644 --- a/tests/unit/connector/test_real_kafka.py +++ b/tests/unit/connector/test_real_kafka.py @@ -24,7 +24,7 @@ def setup_module(): if not in_ci: subprocess.run( - ["docker", "compose", "-f", "quickstart/docker-compose.yml", "up", "-d", "kafka"] + ["docker", "compose", "-f", "example/compose/docker-compose.yml", "up", "-d", "kafka"] ) diff --git a/tests/unit/test_quickstart.py b/tests/unit/test_quickstart.py index 18be79ee7..fbd686676 100644 --- a/tests/unit/test_quickstart.py +++ b/tests/unit/test_quickstart.py @@ -5,19 +5,19 @@ from logprep import run_logprep -class TestQuickstart: - QUICKSTART_CONFIG_PATH = "quickstart/exampledata/config/pipeline.yml" +class TestExampleCompose: + EXAMPLE_CONFIG_PATH = "examples/exampledata/config/pipeline.yml" @mock.patch("os.environ", new={"PROMETHEUS_MULTIPROC_DIR": "/tmp"}) - def test_quickstart_setup_is_valid(self): - """ensures the quickstart rules are valid""" + def test_example_compose_setup_is_valid(self): + """ensures the example rules are valid""" with mock.patch( "sys.argv", [ "logprep", "test", "config", - self.QUICKSTART_CONFIG_PATH, + self.EXAMPLE_CONFIG_PATH, ], ): with pytest.raises(SystemExit) as e_info: diff --git a/tests/unit/test_run_logprep.py b/tests/unit/test_run_logprep.py index 7f61b55d0..cf4e4870a 100644 --- a/tests/unit/test_run_logprep.py +++ b/tests/unit/test_run_logprep.py @@ -55,7 +55,7 @@ def setup_method(self): "logprep.util.configuration.Configuration.as_yaml", ), ( - "test dry-run tests/testdata/config/config.yml quickstart/exampledata/input_logdata/test_input.jsonl", + "test dry-run tests/testdata/config/config.yml examples/exampledata/input_logdata/test_input.jsonl", "logprep.util.rule_dry_runner.DryRunner.run", ), ( @@ -255,7 +255,7 @@ def test_logprep_exits_on_request_exception(self): @mock.patch("logprep.util.rule_dry_runner.DryRunner.run") def test_test_dry_run_starts_dry_runner(self, mock_dry_runner): config_path = ("tests/testdata/config/config.yml",) - events_path = "quickstart/exampledata/input_logdata/test_input.jsonl" + events_path = "examples/exampledata/input_logdata/test_input.jsonl" result = self.cli_runner.invoke(cli, ["test", "dry-run", *config_path, events_path]) assert result.exit_code == 0 mock_dry_runner.assert_called() diff --git a/tests/unit/util/test_configuration.py b/tests/unit/util/test_configuration.py index 3c067d58a..4a027b339 100644 --- a/tests/unit/util/test_configuration.py +++ b/tests/unit/util/test_configuration.py @@ -196,7 +196,7 @@ def test_pipeline_property_is_merged_from_configs(self, tmp_path): pipeline: - labelername: type: labeler - schema: quickstart/exampledata/rules/labeler/schema.json + schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true specific_rules: [] generic_rules: [] @@ -377,9 +377,9 @@ def test_verify_passes_for_valid_configuration(self): { "labelername": { "type": "labeler", - "schema": "quickstart/exampledata/rules/labeler/schema.json", + "schema": "examples/exampledata/rules/labeler/schema.json", "include_parent_labels": "on", - "specific_rules": ["quickstart/exampledata/rules/labeler/specific"], + "specific_rules": ["examples/exampledata/rules/labeler/specific"], } } ] @@ -393,10 +393,10 @@ def test_verify_passes_for_valid_configuration(self): { "labelername": { "type": "labeler", - "schema": "quickstart/exampledata/rules/labeler/schema.json", + "schema": "examples/exampledata/rules/labeler/schema.json", "include_parent_labels": "on", - "specific_rules": ["quickstart/exampledata/rules/labeler/specific"], - "generic_rules": ["quickstart/exampledata/rules/labeler/generic"], + "specific_rules": ["examples/exampledata/rules/labeler/specific"], + "generic_rules": ["examples/exampledata/rules/labeler/generic"], "SOME_UNKNOWN_OPTION": "FOO", } } @@ -411,10 +411,10 @@ def test_verify_passes_for_valid_configuration(self): { "labelername": { "type": "labeler", - "schema": "quickstart/exampledata/rules/labeler/schema.json", + "schema": "examples/exampledata/rules/labeler/schema.json", "include_parent_labels": "on", - "specific_rules": ["quickstart/exampledata/rules/labeler/specific"], - "generic_rules": ["quickstart/exampledata/rules/labeler/generic"], + "specific_rules": ["examples/exampledata/rules/labeler/specific"], + "generic_rules": ["examples/exampledata/rules/labeler/generic"], "SOME UNKNOWN OPTION": "FOO", } } @@ -444,10 +444,10 @@ def test_verify_passes_for_valid_configuration(self): { "labelername": { "type": "labeler", - "schema": "quickstart/exampledata/rules/labeler/schema.json", + "schema": "examples/exampledata/rules/labeler/schema.json", "include_parent_labels": "on", - "specific_rules": ["quickstart/exampledata/rules/labeler/specific"], - "generic_rules": ["quickstart/exampledata/rules/labeler/generic"], + "specific_rules": ["examples/exampledata/rules/labeler/specific"], + "generic_rules": ["examples/exampledata/rules/labeler/generic"], "SOME UNKNOWN OPTION": "FOO", } }, @@ -523,12 +523,12 @@ def test_verify_verifies_config(self, tmp_path, test_case, test_config, error_co pipeline: - labelername: type: labeler - schema: quickstart/exampledata/rules/labeler/schema.json + schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true specific_rules: - - quickstart/exampledata/rules/labeler/specific + - examples/exampledata/rules/labeler/specific generic_rules: - - quickstart/exampledata/rules/labeler/generic + - examples/exampledata/rules/labeler/generic """, "LOGPREP_OUTPUT": """ output: @@ -780,7 +780,7 @@ def test_reload_raises_on_invalid_processor_config(self, tmp_path): pipeline: - labelername: type: labeler - schema: quickstart/exampledata/rules/labeler/schema.json + schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true specific_rules: [] generic_rules: [] @@ -805,7 +805,7 @@ def test_reload_raises_on_invalid_processor_config(self, tmp_path): pipeline: - labelername: type: labeler - schema: quickstart/exampledata/rules/labeler/schema.json + schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true specific_rules: [] generic_rules: [] @@ -854,7 +854,7 @@ def test_reload_raises_on_same_version(self, tmp_path): pipeline: - labelername: type: labeler - schema: quickstart/exampledata/rules/labeler/schema.json + schema: examples/exampledata/rules/labeler/schema.json include_parent_labels: true specific_rules: [] generic_rules: [] diff --git a/tests/unit/util/test_getter.py b/tests/unit/util/test_getter.py index c64d51d39..ddbeb2c27 100644 --- a/tests/unit/util/test_getter.py +++ b/tests/unit/util/test_getter.py @@ -545,9 +545,7 @@ def test_get_raw_raises_if_credential_file_env_set_and_unauthorizes(self): http_getter: HttpGetter = GetterFactory.from_string(f"https://{domain}/bar") with mock.patch.dict( "os.environ", - { - ENV_NAME_LOGPREP_CREDENTIALS_FILE: "quickstart/exampledata/config/credentials.yml" - }, + {ENV_NAME_LOGPREP_CREDENTIALS_FILE: "examples/exampledata/config/credentials.yml"}, ): http_getter.get_json() assert error.value.response.status_code == 401 From 8654d9fc7f5adae99607b7cb9e0931d5d1ad92e3 Mon Sep 17 00:00:00 2001 From: ekneg54 Date: Mon, 15 Jul 2024 08:30:31 +0200 Subject: [PATCH 03/14] add helm installation option --- doc/source/installation.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/source/installation.rst b/doc/source/installation.rst index 47e1859f9..40ecf5f6c 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -61,4 +61,7 @@ Helm This option can be used to deploy logprep on a kubernetes cluster. -tbd +.. code-block:: bash + + helm repo add logprep https://fkie-cad.github.io/Logprep + helm install logprep logprep/logprep From 989c830a92b0b39370ec7717d4480981e04d24f7 Mon Sep 17 00:00:00 2001 From: ekneg54 Date: Mon, 15 Jul 2024 12:14:38 +0200 Subject: [PATCH 04/14] add opensiem example --- .gitignore | 1 + charts/logprep/templates/deployment.yaml | 3 +- charts/logprep/values.yaml | 3 + doc/source/examples/minikube.rst | 15 +++- doc/source/installation.rst | 21 ++++- examples/k8s/.helmignore | 23 +++++ examples/k8s/Chart.yaml | 24 ++++++ examples/k8s/values.yaml | 104 +++++++++++++++++++++++ 8 files changed, 189 insertions(+), 5 deletions(-) create mode 100644 examples/k8s/.helmignore create mode 100644 examples/k8s/Chart.yaml create mode 100644 examples/k8s/values.yaml diff --git a/.gitignore b/.gitignore index 30c797895..bb0cf0161 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ experiments **/_static/*.xlsx logprep.log /charts/logprep/charts +examples/k8s/charts diff --git a/charts/logprep/templates/deployment.yaml b/charts/logprep/templates/deployment.yaml index 87e3bebf6..c47a4d632 100644 --- a/charts/logprep/templates/deployment.yaml +++ b/charts/logprep/templates/deployment.yaml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "logprep.name" . }} + name: {{ include "logprep.fullname" . }} labels: {{- include "logprep.labels" . | nindent 4 }} spec: @@ -14,6 +14,7 @@ spec: {{- include "logprep.selectorLabels" . | nindent 6 }} template: metadata: + name: {{ include "logprep.fullname" . }} labels: {{- include "logprep.labels" . | nindent 8 }} annotations: diff --git a/charts/logprep/values.yaml b/charts/logprep/values.yaml index 9ec46ba9e..3a12a402a 100644 --- a/charts/logprep/values.yaml +++ b/charts/logprep/values.yaml @@ -118,6 +118,9 @@ logger: # # See https://logprep.readthedocs.io/en/latest/user_manual/configuration/input.html # for available configuration options. +# Note: +# For the `http_input` endpoints you have to add the endpoint `/health: plaintext` to ensure +# readiness probes are working. input: {} # The logprep output connector configuration diff --git a/doc/source/examples/minikube.rst b/doc/source/examples/minikube.rst index eded3b3a1..f1fdc2d7b 100644 --- a/doc/source/examples/minikube.rst +++ b/doc/source/examples/minikube.rst @@ -62,9 +62,18 @@ At first you have to install the prometheus PodMonitor CRD: kubectl apply -f https://raw.githubusercontent.com/prometheus-community/helm-charts/main/charts/kube-prometheus-stack/charts/crds/crds/crd-podmonitors.yaml -Next you can install logprep using: +Then you have to update and build the helm subcharts repository: .. code-block:: bash - :caption: Install logprep + :caption: Add the bitnami helm repository - helm install logprep charts/logprep + helm dependencies update ./examples/k8s + helm dependencies build ./examples/k8s + + +Next you can install the opensiem example using: + +.. code-block:: bash + :caption: Install opensiem + + helm install opensiem examples/k8s diff --git a/doc/source/installation.rst b/doc/source/installation.rst index 40ecf5f6c..34189b618 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -61,7 +61,26 @@ Helm This option can be used to deploy logprep on a kubernetes cluster. +At first you have to install the prometheus PodMonitor CRD: + +.. code-block:: bash + :caption: Install the prometheus PodMonitor CRD + + kubectl apply -f https://raw.githubusercontent.com/prometheus-community/helm-charts/main/charts/kube-prometheus-stack/charts/crds/crds/crd-podmonitors.yaml + + +To install latest stable release: + .. code-block:: bash helm repo add logprep https://fkie-cad.github.io/Logprep - helm install logprep logprep/logprep + helm install logprep logprep/logprep + + +To install from cloned github repository: + +.. code-block:: bash + + git clone https://github.com/fkie-cad/Logprep.git + cd Logprep + helm install logprep charts/logprep diff --git a/examples/k8s/.helmignore b/examples/k8s/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/examples/k8s/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/examples/k8s/Chart.yaml b/examples/k8s/Chart.yaml new file mode 100644 index 000000000..5345e5217 --- /dev/null +++ b/examples/k8s/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: opensiem +description: a deployment example for logprep siem deployment +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: "1.0.0" + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.0.0" + +dependencies: + - name: logprep + repository: file://../../charts/logprep + version: 13.1.0 + alias: connector + - name: kafka + repository: https://charts.bitnami.com/bitnami + version: 29.3.7 diff --git a/examples/k8s/values.yaml b/examples/k8s/values.yaml new file mode 100644 index 000000000..6992184b8 --- /dev/null +++ b/examples/k8s/values.yaml @@ -0,0 +1,104 @@ +connector: + # The replica count + replicas: 1 + + # The pod resources + resources: + limits: + memory: "2Gi" + cpu: "1" + requests: + memory: "2Gi" + cpu: "250m" + + # Optional secrets that will be mounted into the pod + # Listed secrets are handled specially by the logprep deployment. + # Additional secrets will be mounted as usual. + # secrets: + # certificates: + # name: ca-cert # Name of the secret containing the ca certificate (or chain) in one data block + # credentials: + # name: logprep-credentials # Name of the secret containing the logprep credentials file + # imagePullSecret: + # name: logprep-image-pull-secret # Name of the secret containing the image pull secret + secrets: {} + + # Logprep logging configuration. + # See: https://logprep.readthedocs.io/en/latest/user_manual/configuration/index.html#configuration-file-structure + # for available configuration options. + logger: + level: DEBUG + input: + type: http_input + message_backlog_size: 150 + collect_meta: True + metafield_name: "@metadata" + uvicorn_config: + host: 0.0.0.0 + port: 9000 + workers: 2 + access_log: true + server_header: false + date_header: false + endpoints: + /auth-json: json + /json: json + /lab/123/(ABC|DEF)/pl.*: plaintext + /lab/123/ABC/auditlog: jsonl + /health: plaintext + + output: + type: confluentkafka_output + topic: consumer + error_topic: errors + flush_timeout: 300 + send_timeout: 0 + kafka_config: + bootstrap.servers: opensiem-kafka:9092 + compression.type: gzip + statistics.interval.ms: "60000" + + # Additional logprep configurations + # Note: The configuration keys 'logger', 'metrics', 'input' and 'output' will overwrite the + # corresponding keys in these configurations as they will be merged into + # the final configuration, after the configurations that are specified here. + # for configurations with data from an external API put the url in the name field and leave the data field undefined + # + # Example: + # + # configurations: + # - name: logprep-config + # data: + # pipeline: [] + # - name: https://rule-server.de + # + # See https://logprep.readthedocs.io/en/latest/user_manual/configuration/index.html#configuration-file-structure + # for available configuration options. + configurations: + - name: logprep-config + data: + pipeline: [] + + # Additional artifacts needed by the logprep deployment. These follow the same mechanic + # as the configurations. The artifacts will be mounted into the pod under the name given + # in the artifacts configuration + # Beware, that configMaps are limited to 1MiB in size. + # Example: + # artifacts: + # - name: admin-list + # data: | + # alfalfa + # admin + # admin2 + artifacts: [] +## for additional configurations see: `https://github.com/bitnami/charts/blob/main/bitnami/kafka/values.yaml` +kafka: + controller: + replicaCount: 1 + metrics: + jmx: + enabled: true + provision: + topics: + - consumer + - producer From 5235e521322dd70814f27e6cd24392a4c7636de8 Mon Sep 17 00:00:00 2001 From: ekneg54 Date: Mon, 15 Jul 2024 15:56:21 +0200 Subject: [PATCH 05/14] add istio installation --- doc/source/examples/minikube.rst | 39 +++++++++++ examples/k8s/Chart.yaml | 6 ++ examples/k8s/templates/_helpers.tpl | 77 +++++++++++++++++++++ examples/k8s/templates/gateway.yaml | 7 ++ examples/k8s/values.yaml | 102 ++++++++++++++-------------- 5 files changed, 179 insertions(+), 52 deletions(-) create mode 100644 examples/k8s/templates/_helpers.tpl create mode 100644 examples/k8s/templates/gateway.yaml diff --git a/doc/source/examples/minikube.rst b/doc/source/examples/minikube.rst index f1fdc2d7b..e1752188b 100644 --- a/doc/source/examples/minikube.rst +++ b/doc/source/examples/minikube.rst @@ -49,6 +49,8 @@ with the following commands: :caption: Configure and start minikube minikube config set driver docker + minikube config set cpus 16 + minikube config set memory 16GB minikube start Deploy the example @@ -70,6 +72,43 @@ Then you have to update and build the helm subcharts repository: helm dependencies update ./examples/k8s helm dependencies build ./examples/k8s +Then install istio (for details see: `https://istio.io/latest/docs/setup/install/helm/`_. ): + +.. code-block:: bash + :caption: Create the istio-system namespace + + kubectl create namespace istio-system + +.. code-block:: bash + :caption: Install istio + + helm repo add istio https://istio-release.storage.googleapis.com/charts + helm repo update + helm install istio-base istio/base -n istio-system --set defaultRevision=opensiem --wait + helm install istiod istio/istiod -n istio-system --wait + + +.. code-block:: bash + :caption: Install istio ingress gateway + + kubectl create namespace istio-ingress + helm install istio-ingress istio/gateway -n istio-ingress + +.. code-block:: bash + :caption: Verifiy the istio installation + + ❯ helm ls -n istio-system + NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION + istio-base istio-system 1 2024-07-15 14:54:54.029747408 +0200 CEST deployed base-1.22.2 1.22.2 + istiod istio-system 1 2024-07-15 14:57:41.496783572 +0200 CEST deployed istiod-1.22.2 1.22.2 + + ❯ kubectl get deployments -n istio-system --output wide + NAME READY UP-TO-DATE AVAILABLE AGE CONTAINERS IMAGES SELECTOR + istiod 1/1 1 1 24m discovery docker.io/istio/pilot:1.22.2 istio=pilot + + ❯ kubectl get pods -n istio-ingress + NAME READY STATUS RESTARTS AGE + istio-ingress-7f5f6f58b8-sv6gk 1/1 Running 0 16m Next you can install the opensiem example using: diff --git a/examples/k8s/Chart.yaml b/examples/k8s/Chart.yaml index 5345e5217..390c7143d 100644 --- a/examples/k8s/Chart.yaml +++ b/examples/k8s/Chart.yaml @@ -19,6 +19,12 @@ dependencies: repository: file://../../charts/logprep version: 13.1.0 alias: connector + - name: logprep + repository: file://../../charts/logprep + version: 13.1.0 - name: kafka repository: https://charts.bitnami.com/bitnami version: 29.3.7 + - name: opensearch + repository: https://charts.bitnami.com/bitnami + version: 1.2.6 diff --git a/examples/k8s/templates/_helpers.tpl b/examples/k8s/templates/_helpers.tpl new file mode 100644 index 000000000..43a483388 --- /dev/null +++ b/examples/k8s/templates/_helpers.tpl @@ -0,0 +1,77 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "opensiem.name" -}} +{{- printf "%s" .Chart.Name | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "opensiem.fullname" -}} +{{- printf "%s-%s" .Release.Name .Chart.Name | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "opensiem.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "opensiem.release" -}} +{{- printf "%s" .Release.Name | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +{{- end }} + +{{/* +Common labels +*/}} +{{- define "opensiem.labels" -}} +helm.sh/chart: {{ include "opensiem.chart" . }} +{{ include "opensiem.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/application: {{ include "opensiem.name" . }} +{{- range $key, $value := .Values.extraLabels }} +{{ $key}}: {{ $value | quote }} +{{- end }} +{{- end }} + +{{/* +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "opensiem.selectorLabels" -}} +app.kubernetes.io/name: {{ include "opensiem.fullname" . }} +app.kubernetes.io/instance: {{ include "opensiem.release" . }} +{{- end }} + +{{/* +{{- end }} + +{{/* +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "opensiem.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "opensiem.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + diff --git a/examples/k8s/templates/gateway.yaml b/examples/k8s/templates/gateway.yaml new file mode 100644 index 000000000..f98fc7bc4 --- /dev/null +++ b/examples/k8s/templates/gateway.yaml @@ -0,0 +1,7 @@ +apiVersion: networking.istio.io/v1beta1 +kind: Gateway +metadata: + name: gateway +spec: + selector: + istio: ingressgateway diff --git a/examples/k8s/values.yaml b/examples/k8s/values.yaml index 6992184b8..7ead97e06 100644 --- a/examples/k8s/values.yaml +++ b/examples/k8s/values.yaml @@ -1,8 +1,7 @@ connector: - # The replica count + image: + tag: py3.11-13.0.0 replicas: 1 - - # The pod resources resources: limits: memory: "2Gi" @@ -10,22 +9,7 @@ connector: requests: memory: "2Gi" cpu: "250m" - - # Optional secrets that will be mounted into the pod - # Listed secrets are handled specially by the logprep deployment. - # Additional secrets will be mounted as usual. - # secrets: - # certificates: - # name: ca-cert # Name of the secret containing the ca certificate (or chain) in one data block - # credentials: - # name: logprep-credentials # Name of the secret containing the logprep credentials file - # imagePullSecret: - # name: logprep-image-pull-secret # Name of the secret containing the image pull secret secrets: {} - - # Logprep logging configuration. - # See: https://logprep.readthedocs.io/en/latest/user_manual/configuration/index.html#configuration-file-structure - # for available configuration options. logger: level: DEBUG input: @@ -46,7 +30,6 @@ connector: /lab/123/(ABC|DEF)/pl.*: plaintext /lab/123/ABC/auditlog: jsonl /health: plaintext - output: type: confluentkafka_output topic: consumer @@ -58,39 +41,6 @@ connector: compression.type: gzip statistics.interval.ms: "60000" - # Additional logprep configurations - # Note: The configuration keys 'logger', 'metrics', 'input' and 'output' will overwrite the - # corresponding keys in these configurations as they will be merged into - # the final configuration, after the configurations that are specified here. - # for configurations with data from an external API put the url in the name field and leave the data field undefined - # - # Example: - # - # configurations: - # - name: logprep-config - # data: - # pipeline: [] - # - name: https://rule-server.de - # - # See https://logprep.readthedocs.io/en/latest/user_manual/configuration/index.html#configuration-file-structure - # for available configuration options. - configurations: - - name: logprep-config - data: - pipeline: [] - - # Additional artifacts needed by the logprep deployment. These follow the same mechanic - # as the configurations. The artifacts will be mounted into the pod under the name given - # in the artifacts configuration - # Beware, that configMaps are limited to 1MiB in size. - # Example: - # artifacts: - # - name: admin-list - # data: | - # alfalfa - # admin - # admin2 - artifacts: [] ## for additional configurations see: `https://github.com/bitnami/charts/blob/main/bitnami/kafka/values.yaml` kafka: controller: @@ -102,3 +52,51 @@ kafka: topics: - consumer - producer + +logprep: + image: + tag: py3.11-13.0.0 + input: + type: confluentkafka_input + topic: consumer + kafka_config: + bootstrap.servers: opensiem-kafka:9092 + group.id: cgroup3 + enable.auto.commit: "true" + auto.commit.interval.ms: "10000" + enable.auto.offset.store: "false" + queued.min.messages: "100000" + queued.max.messages.kbytes: "65536" + statistics.interval.ms: "60000" + preprocessing: + version_info_target_field: Logprep_version_info + log_arrival_time_target_field: event.ingested + hmac: + target: + key: "thisisasecureandrandomkey" + output_field: Full_event + output: + type: opensearch_output + hosts: + - opensiem-opensearch:9200 + default_index: processed + error_index: errors + message_backlog_size: 10000 + timeout: 10000 + flush_timeout: 60 + max_retries: 3 + parallel_bulk: false + user: admin + secret: admin +## for additional configurations see: `https://github.com/bitnami/charts/blob/main/bitnami/opensearch/values.yaml` +opensearch: + dashboards: + enabled: true + ingest: + replicaCount: 1 + master: + replicaCount: 1 + data: + replicaCount: 1 + coordinating: + replicaCount: 1 From bfe9f35d27c0fdb9343b49b8b3f4b03621361d11 Mon Sep 17 00:00:00 2001 From: ekneg54 Date: Tue, 16 Jul 2024 15:26:03 +0200 Subject: [PATCH 06/14] add istio gateway and virtual service --- charts/logprep/templates/gateway.yaml | 30 +++++++ charts/logprep/templates/virtual-service.yaml | 36 +++++++++ charts/logprep/values.yaml | 37 ++++++++- tests/unit/charts/test_ingress_gateway.py | 28 +++++++ tests/unit/charts/test_virtual_service.py | 78 +++++++++++++++++++ 5 files changed, 208 insertions(+), 1 deletion(-) create mode 100644 charts/logprep/templates/gateway.yaml create mode 100644 charts/logprep/templates/virtual-service.yaml create mode 100644 tests/unit/charts/test_ingress_gateway.py create mode 100644 tests/unit/charts/test_virtual_service.py diff --git a/charts/logprep/templates/gateway.yaml b/charts/logprep/templates/gateway.yaml new file mode 100644 index 000000000..3cf1090e0 --- /dev/null +++ b/charts/logprep/templates/gateway.yaml @@ -0,0 +1,30 @@ +{{- if .Values.ingress.enabled -}} +--- +apiVersion: networking.istio.io/v1alpha3 +kind: Gateway +metadata: + name: {{ include "logprep.fullname" . }} + labels: + {{- include "logprep.labels" . | nindent 4 }} +spec: + selector: + istio: ingressgateway + servers: + - hosts: + - "*" + port: + name: http + number: 80 + protocol: HTTP + tls: + httpsRedirect: true + - hosts: + - "*" + port: + name: https + number: 443 + protocol: HTTPS + tls: + credentialName: {{ .Values.ingress.certificate.name }} + mode: SIMPLE +{{- end}} \ No newline at end of file diff --git a/charts/logprep/templates/virtual-service.yaml b/charts/logprep/templates/virtual-service.yaml new file mode 100644 index 000000000..1557ea250 --- /dev/null +++ b/charts/logprep/templates/virtual-service.yaml @@ -0,0 +1,36 @@ +{{- if .Values.ingress.enabled -}} +{{- if .Values.input -}} +{{- if eq .Values.input.type "http_input" -}} +--- +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: {{ include "logprep.fullname" . }} + labels: + {{- include "logprep.labels" . | nindent 4 }} +spec: + gateways: + - {{ include "logprep.fullname" . }}-gateway + hosts: + - {{ .Values.ingress.domain }} + http: + {{ $logprep_fullname := include "logprep.fullname" .}} + {{ $uvicorn_port := .Values.input.uvicorn_config.port }} + {{ $response_headers := .Values.ingress.response_headers }} + {{ range $key, $value := .Values.input.endpoints }} + - match: + - uri: + regex: {{ $key | quote }} + route: + - destination: + host: {{ $logprep_fullname }}-http-input + port: + number: {{ $uvicorn_port }} + headers: + response: + set: + {{- toYaml $response_headers | nindent 16 }} + {{ end }} +{{- end}} +{{- end}} +{{- end}} diff --git a/charts/logprep/values.yaml b/charts/logprep/values.yaml index 3a12a402a..7e51c4e75 100644 --- a/charts/logprep/values.yaml +++ b/charts/logprep/values.yaml @@ -121,7 +121,23 @@ logger: # Note: # For the `http_input` endpoints you have to add the endpoint `/health: plaintext` to ensure # readiness probes are working. -input: {} +input: + type: http_input + message_backlog_size: 150 + collect_meta: True + metafield_name: "@metadata" + uvicorn_config: + host: 0.0.0.0 + port: 9000 + workers: 2 + access_log: true + server_header: false + date_header: false + endpoints: + /auth-json: json + /json: json + /lab/123/(ABC|DEF)/pl.*: plaintext + /lab/123/ABC/auditlog: jsonl # The logprep output connector configuration # Note: This is only the default output. Additional outputs can be configured in @@ -166,3 +182,22 @@ configurations: # admin # admin2 artifacts: [] + +# The ingress configuration +# If enabled, an istio based ingress will be deployed. This option is only useful +# if the logprep configuration has a http_input configured. +ingress: + enabled: true + domain: "localhost" + certificate: + name: "istio-gateway-certificate" + response_headers: + Cache-Control: no-cache + Content-Security-Policy: default-src 'none'; script-src 'self'; connect-src 'self'; img-src 'self'; style-src 'self'; frame-ancestors 'self'; form-action 'self'; + Cross-Origin-Resource-Policy: same-site + Referrer-Policy: strict-origin-when-cross-origin + Strict-Transport-Security: max-age=31536000; includeSubdomains + Permissions-Policy: geolocation=(), camera=(), microphone=() + X-XSS-Protection: 1; mode=block + X-Content-Type-Options: nosniff + X-Frame-Options: DENY diff --git a/tests/unit/charts/test_ingress_gateway.py b/tests/unit/charts/test_ingress_gateway.py new file mode 100644 index 000000000..3765e842d --- /dev/null +++ b/tests/unit/charts/test_ingress_gateway.py @@ -0,0 +1,28 @@ +# pylint: disable=missing-docstring +# pylint: disable=attribute-defined-outside-init +# pylint: disable=protected-access + + +from tests.unit.charts.test_base import TestBaseChartTest + + +class TestIngressGateway(TestBaseChartTest): + + def test_ingress_gateway_is_rendered(self): + logprep_values = {"ingress": {"enabled": True}} + self.manifests = self.render_chart("logprep", logprep_values) + ingress_gateway = self.manifests.by_query( + "kind: Gateway AND apiVersion: networking.istio.io/v1alpha3" + ) + assert ingress_gateway + assert len(ingress_gateway) == 1 + ingress_gateway = ingress_gateway[0] + assert ingress_gateway["metadata"]["name"] == "logprep-logprep" + + def test_ingress_gateway_is_not_rendered(self): + logprep_values = {"ingress": {"enabled": False}} + self.manifests = self.render_chart("logprep", logprep_values) + ingress_gateway = self.manifests.by_query( + "kind: Gateway AND apiVersion: networking.istio.io/v1alpha3" + ) + assert not ingress_gateway diff --git a/tests/unit/charts/test_virtual_service.py b/tests/unit/charts/test_virtual_service.py new file mode 100644 index 000000000..a3952cb58 --- /dev/null +++ b/tests/unit/charts/test_virtual_service.py @@ -0,0 +1,78 @@ +# pylint: disable=missing-docstring +# pylint: disable=attribute-defined-outside-init +# pylint: disable=protected-access + + +from logprep.util.configuration import yaml +from tests.unit.charts.test_base import TestBaseChartTest + +http_input_config = { + "type": "http_input", + "message_backlog_size": 150, + "collect_meta": True, + "metafield_name": "@metadata", + "uvicorn_config": { + "host": "0.0.0.0", + "port": 9000, + "workers": 2, + "access_log": True, + "server_header": False, + "date_header": False, + }, + "endpoints": { + "/auth-json": "json", + "/json": "json", + "/lab/123/(ABC|DEF)/pl.*": "plaintext", + "/lab/123/ABC/auditlog": "jsonl", + }, +} + + +class TestIstioVirtualService(TestBaseChartTest): + + def test_virtual_service_is_rendered(self): + logprep_values = {"ingress": {"enabled": True}, "input": http_input_config} + self.manifests = self.render_chart("logprep", logprep_values) + virtual_service = self.manifests.by_query( + "kind: VirtualService AND apiVersion: networking.istio.io/v1alpha3" + ) + assert virtual_service + assert len(virtual_service) == 1 + virtual_service = virtual_service[0] + assert virtual_service["metadata"]["name"] == "logprep-logprep" + + def test_virtual_service_has_endpoint_routes(self): + logprep_values = {"ingress": {"enabled": True}, "input": http_input_config} + self.manifests = self.render_chart("logprep", logprep_values) + virtual_service = self.manifests.by_query( + "kind: VirtualService AND apiVersion: networking.istio.io/v1alpha3" + )[0] + defined_routes = [ + route["match"][0]["uri"]["regex"] for route in virtual_service["spec.http"] + ] + for endpoint in http_input_config["endpoints"]: + assert endpoint in defined_routes + + def test_virtual_service_routes_have_response_header(self): + logprep_values = {"ingress": {"enabled": True}, "input": http_input_config} + self.manifests = self.render_chart("logprep", logprep_values) + virtual_service = self.manifests.by_query( + "kind: VirtualService AND apiVersion: networking.istio.io/v1alpha3" + )[0] + response_headers_for_routes = [ + route["route"][0]["headers"]["response"]["set"] + for route in virtual_service["spec.http"] + ] + expected_headers = { + "Cache-Control": "no-cache", + "Content-Security-Policy": "default-src 'none'; script-src 'self'; connect-src 'self'; img-src 'self'; style-src 'self'; frame-ancestors 'self'; form-action 'self';", + "Cross-Origin-Resource-Policy": "same-site", + "Referrer-Policy": "strict-origin-when-cross-origin", + "Strict-Transport-Security": "max-age=31536000; includeSubdomains", + "Permissions-Policy": "geolocation=(), camera=(), microphone=()", + "X-XSS-Protection": "1; mode=block", + "X-Content-Type-Options": "nosniff", + "X-Frame-Options": "DENY", + } + for headers in response_headers_for_routes: + assert headers == expected_headers From 7f5d163b850caa75af3a974c1c3336a63813d4e9 Mon Sep 17 00:00:00 2001 From: ekneg54 Date: Tue, 16 Jul 2024 15:57:23 +0200 Subject: [PATCH 07/14] add add istio --- charts/logprep/values.yaml | 2 +- doc/source/examples/minikube.rst | 6 +----- examples/k8s/templates/gateway.yaml | 7 ------- examples/k8s/values.yaml | 6 ++++-- 4 files changed, 6 insertions(+), 15 deletions(-) delete mode 100644 examples/k8s/templates/gateway.yaml diff --git a/charts/logprep/values.yaml b/charts/logprep/values.yaml index 7e51c4e75..6038cf715 100644 --- a/charts/logprep/values.yaml +++ b/charts/logprep/values.yaml @@ -187,7 +187,7 @@ artifacts: [] # If enabled, an istio based ingress will be deployed. This option is only useful # if the logprep configuration has a http_input configured. ingress: - enabled: true + enabled: false domain: "localhost" certificate: name: "istio-gateway-certificate" diff --git a/doc/source/examples/minikube.rst b/doc/source/examples/minikube.rst index e1752188b..4b713acd5 100644 --- a/doc/source/examples/minikube.rst +++ b/doc/source/examples/minikube.rst @@ -74,14 +74,10 @@ Then you have to update and build the helm subcharts repository: Then install istio (for details see: `https://istio.io/latest/docs/setup/install/helm/`_. ): -.. code-block:: bash - :caption: Create the istio-system namespace - - kubectl create namespace istio-system - .. code-block:: bash :caption: Install istio + kubectl create namespace istio-system helm repo add istio https://istio-release.storage.googleapis.com/charts helm repo update helm install istio-base istio/base -n istio-system --set defaultRevision=opensiem --wait diff --git a/examples/k8s/templates/gateway.yaml b/examples/k8s/templates/gateway.yaml deleted file mode 100644 index f98fc7bc4..000000000 --- a/examples/k8s/templates/gateway.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: networking.istio.io/v1beta1 -kind: Gateway -metadata: - name: gateway -spec: - selector: - istio: ingressgateway diff --git a/examples/k8s/values.yaml b/examples/k8s/values.yaml index 7ead97e06..12ded9e58 100644 --- a/examples/k8s/values.yaml +++ b/examples/k8s/values.yaml @@ -1,6 +1,6 @@ connector: image: - tag: py3.11-13.0.0 + tag: py3.11-latest replicas: 1 resources: limits: @@ -40,6 +40,8 @@ connector: bootstrap.servers: opensiem-kafka:9092 compression.type: gzip statistics.interval.ms: "60000" + ingress: + enabled: true ## for additional configurations see: `https://github.com/bitnami/charts/blob/main/bitnami/kafka/values.yaml` kafka: @@ -55,7 +57,7 @@ kafka: logprep: image: - tag: py3.11-13.0.0 + tag: py3.11-latest input: type: confluentkafka_input topic: consumer From e8778f8056b7bbdb9f303342a634ff527bf0e7e4 Mon Sep 17 00:00:00 2001 From: ekneg54 Date: Tue, 16 Jul 2024 20:23:34 +0200 Subject: [PATCH 08/14] set chart --- doc/source/examples/minikube.rst | 18 +++++++++--------- examples/k8s/Chart.yaml | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/doc/source/examples/minikube.rst b/doc/source/examples/minikube.rst index 4b713acd5..a56572930 100644 --- a/doc/source/examples/minikube.rst +++ b/doc/source/examples/minikube.rst @@ -64,14 +64,6 @@ At first you have to install the prometheus PodMonitor CRD: kubectl apply -f https://raw.githubusercontent.com/prometheus-community/helm-charts/main/charts/kube-prometheus-stack/charts/crds/crds/crd-podmonitors.yaml -Then you have to update and build the helm subcharts repository: - -.. code-block:: bash - :caption: Add the bitnami helm repository - - helm dependencies update ./examples/k8s - helm dependencies build ./examples/k8s - Then install istio (for details see: `https://istio.io/latest/docs/setup/install/helm/`_. ): .. code-block:: bash @@ -106,7 +98,15 @@ Then install istio (for details see: `https://istio.io/latest/docs/setup/install NAME READY STATUS RESTARTS AGE istio-ingress-7f5f6f58b8-sv6gk 1/1 Running 0 16m -Next you can install the opensiem example using: +Then you have to update and build the helm subcharts repository: + +.. code-block:: bash + :caption: Add the bitnami helm repository + + helm dependencies update ./examples/k8s + helm dependencies build ./examples/k8s + +Next you are ready to install the opensiem example using: .. code-block:: bash :caption: Install opensiem diff --git a/examples/k8s/Chart.yaml b/examples/k8s/Chart.yaml index 390c7143d..810227c04 100644 --- a/examples/k8s/Chart.yaml +++ b/examples/k8s/Chart.yaml @@ -17,11 +17,11 @@ appVersion: "1.0.0" dependencies: - name: logprep repository: file://../../charts/logprep - version: 13.1.0 + version: 13.1.1 alias: connector - name: logprep repository: file://../../charts/logprep - version: 13.1.0 + version: 13.1.1 - name: kafka repository: https://charts.bitnami.com/bitnami version: 29.3.7 From b4ea5f0ebd61952e2f84526d9303358605d068f6 Mon Sep 17 00:00:00 2001 From: ekneg54 Date: Wed, 17 Jul 2024 11:37:30 +0200 Subject: [PATCH 09/14] use ingress in favor of istio --- charts/logprep/templates/gateway.yaml | 30 ------- charts/logprep/templates/virtual-service.yaml | 36 --------- charts/logprep/values.yaml | 19 ----- doc/source/examples/minikube.rst | 66 +++++++++------- examples/k8s/templates/ingress.yaml | 53 +++++++++++++ tests/unit/charts/test_ingress_gateway.py | 28 ------- tests/unit/charts/test_virtual_service.py | 78 ------------------- 7 files changed, 89 insertions(+), 221 deletions(-) delete mode 100644 charts/logprep/templates/gateway.yaml delete mode 100644 charts/logprep/templates/virtual-service.yaml create mode 100644 examples/k8s/templates/ingress.yaml delete mode 100644 tests/unit/charts/test_ingress_gateway.py delete mode 100644 tests/unit/charts/test_virtual_service.py diff --git a/charts/logprep/templates/gateway.yaml b/charts/logprep/templates/gateway.yaml deleted file mode 100644 index 3cf1090e0..000000000 --- a/charts/logprep/templates/gateway.yaml +++ /dev/null @@ -1,30 +0,0 @@ -{{- if .Values.ingress.enabled -}} ---- -apiVersion: networking.istio.io/v1alpha3 -kind: Gateway -metadata: - name: {{ include "logprep.fullname" . }} - labels: - {{- include "logprep.labels" . | nindent 4 }} -spec: - selector: - istio: ingressgateway - servers: - - hosts: - - "*" - port: - name: http - number: 80 - protocol: HTTP - tls: - httpsRedirect: true - - hosts: - - "*" - port: - name: https - number: 443 - protocol: HTTPS - tls: - credentialName: {{ .Values.ingress.certificate.name }} - mode: SIMPLE -{{- end}} \ No newline at end of file diff --git a/charts/logprep/templates/virtual-service.yaml b/charts/logprep/templates/virtual-service.yaml deleted file mode 100644 index 1557ea250..000000000 --- a/charts/logprep/templates/virtual-service.yaml +++ /dev/null @@ -1,36 +0,0 @@ -{{- if .Values.ingress.enabled -}} -{{- if .Values.input -}} -{{- if eq .Values.input.type "http_input" -}} ---- -apiVersion: networking.istio.io/v1alpha3 -kind: VirtualService -metadata: - name: {{ include "logprep.fullname" . }} - labels: - {{- include "logprep.labels" . | nindent 4 }} -spec: - gateways: - - {{ include "logprep.fullname" . }}-gateway - hosts: - - {{ .Values.ingress.domain }} - http: - {{ $logprep_fullname := include "logprep.fullname" .}} - {{ $uvicorn_port := .Values.input.uvicorn_config.port }} - {{ $response_headers := .Values.ingress.response_headers }} - {{ range $key, $value := .Values.input.endpoints }} - - match: - - uri: - regex: {{ $key | quote }} - route: - - destination: - host: {{ $logprep_fullname }}-http-input - port: - number: {{ $uvicorn_port }} - headers: - response: - set: - {{- toYaml $response_headers | nindent 16 }} - {{ end }} -{{- end}} -{{- end}} -{{- end}} diff --git a/charts/logprep/values.yaml b/charts/logprep/values.yaml index 6038cf715..fb7cfbaee 100644 --- a/charts/logprep/values.yaml +++ b/charts/logprep/values.yaml @@ -182,22 +182,3 @@ configurations: # admin # admin2 artifacts: [] - -# The ingress configuration -# If enabled, an istio based ingress will be deployed. This option is only useful -# if the logprep configuration has a http_input configured. -ingress: - enabled: false - domain: "localhost" - certificate: - name: "istio-gateway-certificate" - response_headers: - Cache-Control: no-cache - Content-Security-Policy: default-src 'none'; script-src 'self'; connect-src 'self'; img-src 'self'; style-src 'self'; frame-ancestors 'self'; form-action 'self'; - Cross-Origin-Resource-Policy: same-site - Referrer-Policy: strict-origin-when-cross-origin - Strict-Transport-Security: max-age=31536000; includeSubdomains - Permissions-Policy: geolocation=(), camera=(), microphone=() - X-XSS-Protection: 1; mode=block - X-Content-Type-Options: nosniff - X-Frame-Options: DENY diff --git a/doc/source/examples/minikube.rst b/doc/source/examples/minikube.rst index a56572930..b0fc8559e 100644 --- a/doc/source/examples/minikube.rst +++ b/doc/source/examples/minikube.rst @@ -51,6 +51,7 @@ with the following commands: minikube config set driver docker minikube config set cpus 16 minikube config set memory 16GB + minikube addons enable ingress minikube start Deploy the example @@ -64,51 +65,56 @@ At first you have to install the prometheus PodMonitor CRD: kubectl apply -f https://raw.githubusercontent.com/prometheus-community/helm-charts/main/charts/kube-prometheus-stack/charts/crds/crds/crd-podmonitors.yaml -Then install istio (for details see: `https://istio.io/latest/docs/setup/install/helm/`_. ): +Then you have to update and build the helm subcharts repository: .. code-block:: bash - :caption: Install istio + :caption: Add the bitnami helm repository - kubectl create namespace istio-system - helm repo add istio https://istio-release.storage.googleapis.com/charts - helm repo update - helm install istio-base istio/base -n istio-system --set defaultRevision=opensiem --wait - helm install istiod istio/istiod -n istio-system --wait + helm dependencies update ./examples/k8s + helm dependencies build ./examples/k8s +Next you are ready to install the opensiem example using: .. code-block:: bash - :caption: Install istio ingress gateway - - kubectl create namespace istio-ingress - helm install istio-ingress istio/gateway -n istio-ingress + :caption: Install opensiem -.. code-block:: bash - :caption: Verifiy the istio installation + helm install opensiem examples/k8s - ❯ helm ls -n istio-system - NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION - istio-base istio-system 1 2024-07-15 14:54:54.029747408 +0200 CEST deployed base-1.22.2 1.22.2 - istiod istio-system 1 2024-07-15 14:57:41.496783572 +0200 CEST deployed istiod-1.22.2 1.22.2 +Make the cluster locally resolvable: - ❯ kubectl get deployments -n istio-system --output wide - NAME READY UP-TO-DATE AVAILABLE AGE CONTAINERS IMAGES SELECTOR - istiod 1/1 1 1 24m discovery docker.io/istio/pilot:1.22.2 istio=pilot +.. code-block:: bash + :caption: add hosts entry to resolve the cluster - ❯ kubectl get pods -n istio-ingress - NAME READY STATUS RESTARTS AGE - istio-ingress-7f5f6f58b8-sv6gk 1/1 Running 0 16m + echo "$( minikube ip ) connector.opensiem dashboards.opensiem grafana.opensiem" | sudo tee -a /etc/hosts -Then you have to update and build the helm subcharts repository: +Test the defined ingresses: .. code-block:: bash - :caption: Add the bitnami helm repository + :caption: Test the opensiem example ingress - helm dependencies update ./examples/k8s - helm dependencies build ./examples/k8s + curl -v http://connector.opensiem/health + curl -v http://dashboards.opensiem -Next you are ready to install the opensiem example using: +Test the opensiem connector: .. code-block:: bash - :caption: Install opensiem + :caption: Test the opensiem example connector - helm install opensiem examples/k8s + ❯ logprep generate http --input-dir ./examples/exampledata/input_logdata/ --target-url http://connector.opensiem --events 100 --batch-size 10 + + 2024-07-17 11:15:35 301643 Generator INFO : Log level set to 'NOTSET' + 2024-07-17 11:15:35 301643 Generator INFO : Started Data Processing + 2024-07-17 11:15:35 301643 Input INFO : Reading input dataset and creating temporary event collections in: '/tmp/logprep_a51e1vh6' + 2024-07-17 11:15:35 301643 Input INFO : Preparing data took: 0.0042 seconds + 2024-07-17 11:15:35 301643 Input INFO : Cleaned up temp dir: '/tmp/logprep_a51e1vh6' + 2024-07-17 11:15:35 301643 Generator INFO : Completed with following statistics: { + "Number of failed events": 0, + "Number of successfull events": 100, + "Requests Connection Errors": 0, + "Requests Timeouts": 0, + "Requests http status 200": 10, + "Requests total": 10 + } + 2024-07-17 11:15:35 301643 Generator INFO : Execution time: 0.067013 seconds + +open your browser and go to `http://dashboards.opensiem`_ to see the generated data in the opensearch dashboards. diff --git a/examples/k8s/templates/ingress.yaml b/examples/k8s/templates/ingress.yaml new file mode 100644 index 000000000..551365712 --- /dev/null +++ b/examples/k8s/templates/ingress.yaml @@ -0,0 +1,53 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: connector-ingress +spec: + ingressClassName: nginx + rules: + - host: connector.opensiem + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: opensiem-connector-http-input + port: + number: 9000 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: dashboard-ingress +spec: + ingressClassName: nginx + rules: + - host: dashboards.opensiem + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: opensiem-opensearch-dashboards + port: + number: 5601 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: grafana-ingress +spec: + ingressClassName: nginx + rules: + - host: grafana.opensiem + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: opensiem-grafana + port: + number: 3000 diff --git a/tests/unit/charts/test_ingress_gateway.py b/tests/unit/charts/test_ingress_gateway.py deleted file mode 100644 index 3765e842d..000000000 --- a/tests/unit/charts/test_ingress_gateway.py +++ /dev/null @@ -1,28 +0,0 @@ -# pylint: disable=missing-docstring -# pylint: disable=attribute-defined-outside-init -# pylint: disable=protected-access - - -from tests.unit.charts.test_base import TestBaseChartTest - - -class TestIngressGateway(TestBaseChartTest): - - def test_ingress_gateway_is_rendered(self): - logprep_values = {"ingress": {"enabled": True}} - self.manifests = self.render_chart("logprep", logprep_values) - ingress_gateway = self.manifests.by_query( - "kind: Gateway AND apiVersion: networking.istio.io/v1alpha3" - ) - assert ingress_gateway - assert len(ingress_gateway) == 1 - ingress_gateway = ingress_gateway[0] - assert ingress_gateway["metadata"]["name"] == "logprep-logprep" - - def test_ingress_gateway_is_not_rendered(self): - logprep_values = {"ingress": {"enabled": False}} - self.manifests = self.render_chart("logprep", logprep_values) - ingress_gateway = self.manifests.by_query( - "kind: Gateway AND apiVersion: networking.istio.io/v1alpha3" - ) - assert not ingress_gateway diff --git a/tests/unit/charts/test_virtual_service.py b/tests/unit/charts/test_virtual_service.py deleted file mode 100644 index a3952cb58..000000000 --- a/tests/unit/charts/test_virtual_service.py +++ /dev/null @@ -1,78 +0,0 @@ -# pylint: disable=missing-docstring -# pylint: disable=attribute-defined-outside-init -# pylint: disable=protected-access - - -from logprep.util.configuration import yaml -from tests.unit.charts.test_base import TestBaseChartTest - -http_input_config = { - "type": "http_input", - "message_backlog_size": 150, - "collect_meta": True, - "metafield_name": "@metadata", - "uvicorn_config": { - "host": "0.0.0.0", - "port": 9000, - "workers": 2, - "access_log": True, - "server_header": False, - "date_header": False, - }, - "endpoints": { - "/auth-json": "json", - "/json": "json", - "/lab/123/(ABC|DEF)/pl.*": "plaintext", - "/lab/123/ABC/auditlog": "jsonl", - }, -} - - -class TestIstioVirtualService(TestBaseChartTest): - - def test_virtual_service_is_rendered(self): - logprep_values = {"ingress": {"enabled": True}, "input": http_input_config} - self.manifests = self.render_chart("logprep", logprep_values) - virtual_service = self.manifests.by_query( - "kind: VirtualService AND apiVersion: networking.istio.io/v1alpha3" - ) - assert virtual_service - assert len(virtual_service) == 1 - virtual_service = virtual_service[0] - assert virtual_service["metadata"]["name"] == "logprep-logprep" - - def test_virtual_service_has_endpoint_routes(self): - logprep_values = {"ingress": {"enabled": True}, "input": http_input_config} - self.manifests = self.render_chart("logprep", logprep_values) - virtual_service = self.manifests.by_query( - "kind: VirtualService AND apiVersion: networking.istio.io/v1alpha3" - )[0] - defined_routes = [ - route["match"][0]["uri"]["regex"] for route in virtual_service["spec.http"] - ] - for endpoint in http_input_config["endpoints"]: - assert endpoint in defined_routes - - def test_virtual_service_routes_have_response_header(self): - logprep_values = {"ingress": {"enabled": True}, "input": http_input_config} - self.manifests = self.render_chart("logprep", logprep_values) - virtual_service = self.manifests.by_query( - "kind: VirtualService AND apiVersion: networking.istio.io/v1alpha3" - )[0] - response_headers_for_routes = [ - route["route"][0]["headers"]["response"]["set"] - for route in virtual_service["spec.http"] - ] - expected_headers = { - "Cache-Control": "no-cache", - "Content-Security-Policy": "default-src 'none'; script-src 'self'; connect-src 'self'; img-src 'self'; style-src 'self'; frame-ancestors 'self'; form-action 'self';", - "Cross-Origin-Resource-Policy": "same-site", - "Referrer-Policy": "strict-origin-when-cross-origin", - "Strict-Transport-Security": "max-age=31536000; includeSubdomains", - "Permissions-Policy": "geolocation=(), camera=(), microphone=()", - "X-XSS-Protection": "1; mode=block", - "X-Content-Type-Options": "nosniff", - "X-Frame-Options": "DENY", - } - for headers in response_headers_for_routes: - assert headers == expected_headers From d712aedfe34a0b8331533d73e3410bcd314de569 Mon Sep 17 00:00:00 2001 From: ekneg54 Date: Wed, 17 Jul 2024 11:50:58 +0200 Subject: [PATCH 10/14] fix values --- charts/logprep/values.yaml | 18 +----------------- doc/source/examples/minikube.rst | 3 +++ 2 files changed, 4 insertions(+), 17 deletions(-) diff --git a/charts/logprep/values.yaml b/charts/logprep/values.yaml index fb7cfbaee..3a12a402a 100644 --- a/charts/logprep/values.yaml +++ b/charts/logprep/values.yaml @@ -121,23 +121,7 @@ logger: # Note: # For the `http_input` endpoints you have to add the endpoint `/health: plaintext` to ensure # readiness probes are working. -input: - type: http_input - message_backlog_size: 150 - collect_meta: True - metafield_name: "@metadata" - uvicorn_config: - host: 0.0.0.0 - port: 9000 - workers: 2 - access_log: true - server_header: false - date_header: false - endpoints: - /auth-json: json - /json: json - /lab/123/(ABC|DEF)/pl.*: plaintext - /lab/123/ABC/auditlog: jsonl +input: {} # The logprep output connector configuration # Note: This is only the default output. Additional outputs can be configured in diff --git a/doc/source/examples/minikube.rst b/doc/source/examples/minikube.rst index b0fc8559e..058eda7ac 100644 --- a/doc/source/examples/minikube.rst +++ b/doc/source/examples/minikube.rst @@ -57,6 +57,9 @@ with the following commands: Deploy the example ------------------ +The following steps install the actual opensiem example on the minikube cluster. +It will install + At first you have to install the prometheus PodMonitor CRD: .. code-block:: bash From 282fee510778d55500061bde726d1007655fe344 Mon Sep 17 00:00:00 2001 From: ekneg54 Date: Wed, 17 Jul 2024 12:35:19 +0200 Subject: [PATCH 11/14] working example --- doc/source/examples/minikube.rst | 5 +++++ examples/k8s/values.yaml | 16 +++++++++++----- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/doc/source/examples/minikube.rst b/doc/source/examples/minikube.rst index 058eda7ac..ec9bb7e95 100644 --- a/doc/source/examples/minikube.rst +++ b/doc/source/examples/minikube.rst @@ -45,6 +45,11 @@ with the following commands: sudo mv linux-amd64/helm /usr/local/bin/helm sudo chmod +x /usr/local/bin/helm +.. code-block:: bash + :caption: add helm repositories + + helm repo add bitnami https://charts.bitnami.com/bitnami + .. code-block:: bash :caption: Configure and start minikube diff --git a/examples/k8s/values.yaml b/examples/k8s/values.yaml index 12ded9e58..2753c4415 100644 --- a/examples/k8s/values.yaml +++ b/examples/k8s/values.yaml @@ -14,7 +14,7 @@ connector: level: DEBUG input: type: http_input - message_backlog_size: 150 + message_backlog_size: 15000 collect_meta: True metafield_name: "@metadata" uvicorn_config: @@ -45,15 +45,21 @@ connector: ## for additional configurations see: `https://github.com/bitnami/charts/blob/main/bitnami/kafka/values.yaml` kafka: + listeners: + client: + protocol: PLAINTEXT controller: - replicaCount: 1 + replicaCount: 3 metrics: jmx: enabled: true - provision: + provisioning: + enabled: true + replicationFactor: 3 + numPartitions: 10 topics: - - consumer - - producer + - name: consumer + - name: producer logprep: image: From 5a13ef760ca2c147778a2bd5ac3be100a119ec41 Mon Sep 17 00:00:00 2001 From: ekneg54 Date: Wed, 17 Jul 2024 12:39:45 +0200 Subject: [PATCH 12/14] update Changelog --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd3ffde10..a1aaf67de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,8 @@ * each processor returns an object including the processor name, generated extra_data, warnings and errors * the pipeline returns an object with the list of all processor result objects -* add deployment examples for compose and helm -* remove quickstart setup +* add kubernetes opensiem deployment example +* move quickstart setup to compose example ### Bugfix From 7dc97a9c570d3d3a4a2176976cfafe211ef64a6f Mon Sep 17 00:00:00 2001 From: djkhl Date: Thu, 18 Jul 2024 16:08:45 +0200 Subject: [PATCH 13/14] update chart version --- charts/logprep/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/logprep/Chart.yaml b/charts/logprep/Chart.yaml index 50f71b756..85dc04efa 100644 --- a/charts/logprep/Chart.yaml +++ b/charts/logprep/Chart.yaml @@ -6,7 +6,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "13.2.0" +version: "13.2.1" # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to From c7a68489c45309379b7894f37857916eaeb3c956 Mon Sep 17 00:00:00 2001 From: djkhl Date: Thu, 18 Jul 2024 16:37:46 +0200 Subject: [PATCH 14/14] update documentation for utilizing fda and ucl in compose example for little bit more consistency --- doc/source/examples/compose.rst | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/doc/source/examples/compose.rst b/doc/source/examples/compose.rst index 98c9769bc..f4c264aef 100644 --- a/doc/source/examples/compose.rst +++ b/doc/source/examples/compose.rst @@ -134,7 +134,12 @@ If you want to try out the FDA and UCL you first have to do some preparations. 0. Run the example compose setup with the :code:`oauth2` profile: - :code:`docker compose --profile oauth2 up -d`. + +.. code-block:: bash + + docker compose --profile oauth2 up -d. + + 1. Sign into the keycloak admin panel and create a logprep user in the :code:`logprep` realm. Make sure that the user is part of the :code:`logprep-admin` group and has a password. If you choose a password other than :code:`logprep` you have to update the credentials file @@ -148,19 +153,19 @@ If you want to try out the FDA and UCL you first have to do some preparations. your created logprep user and then configure required Use-Cases. At the current moment these configuration are not yet processed by logprep though, as the ucl only provides a mock endpoint which doesn't contain your Use-Case configurations. -4. Set the env :code:`LOGPREP_CREDENTIALS_FILE` to :code:`examples/exampledata/config/credentials.yml` +4. Set the env and run logprep + + .. code-block:: bash + + export LOGPREP_CREDENTIALS_FILE="examples/exampledata/config/credentials.yml" + logprep run examples/exampledata/config/pipeline.yml "http://localhost:3002/api/v1/pipelines?stage=prod&logclass=ExampleClass" "http://localhost:3001/api/v1/general-predetection" -Once you have set everything up you can run logprep with the following command. Just consider that the first :code:`pipeline.yml` argument is used to define a proper :code:`input` and :code:`output` as those are not part of the FDA/UCL output. Also, in the second argument you should ensure that the :code:`stage` and :code:`loglcass` are set properly. -.. code-block:: bash - - logprep run examples/exampledata/config/pipeline.yml "http://localhost:3002/api/v1/pipelines?stage=prod&logclass=ExampleClass" "http://localhost:3001/api/v1/general-predetection" - .. note:: If you did use the example compose setup before and run into problems it is advised to first pull all images again to update them to the latest version: - :code:`docker compose -f ./example/compose/docker-compose.yml pull`. + :code:`docker compose -f ./examples/compose/docker-compose.yml pull`.