Merge pull request #100 from strongdm/feat/add-entity-stream

Add entity stream

.github/workflows/tests.yml

@@ -21,3 +21,5 @@ jobs:
         run: bundle install
       - name: Run tests
         run: ruby test/run_test.rb
+      - name: Run RSpec Tests
+        run: bundle exec rspec test

Gemfile

@@ -22,3 +22,4 @@ gem 'fluent-plugin-elasticsearch'
 gem 'fluent-plugin-bigquery'
 gem 'fluent-plugin-prometheus'
 gem 'test-unit'
+gem 'rspec'

Gemfile.lock

@@ -215,7 +215,6 @@ GEM
       mime-types-data (~> 3.2015)
     mime-types-data (3.2022.0105)
     mini_mime (1.1.2)
-    mini_portile2 (2.8.0)
     minitest (5.15.0)
     mongo (2.6.4)
       bson (>= 4.3.0, < 5.0.0)
@@ -226,8 +225,7 @@ GEM
     net-http-persistent (3.1.0)
       connection_pool (~> 2.2)
     netrc (0.11.0)
-    nokogiri (1.13.6)
-      mini_portile2 (~> 2.8.0)
+    nokogiri (1.13.6-x86_64-linux)
       racc (~> 1.4)
     openid_connect (1.1.8)
       activemodel
@@ -351,6 +349,7 @@ DEPENDENCIES
   fluent-plugin-sanitizer
   fluent-plugin-splunk-hec
   fluent-plugin-sumologic_output
+  rspec
   test-unit
 
 BUNDLED WITH

conf-utils.rb

@@ -1,10 +1,7 @@
 
-SUPPORTED_STORES="stdout remote-syslog s3 cloudwatch splunk-hec datadog azure-loganalytics sumologic kafka mongo logz loki elasticsearch bigquery"
-AUDIT_ENTITY_TYPES = {
-  "resources" => "resource",
-  "users" => "user",
-  "roles" => "role",
-}
+require_relative './fluentd/scripts/dump_sdm_entities'
+
+SUPPORTED_STORES = "stdout remote-syslog s3 cloudwatch splunk-hec datadog azure-loganalytics sumologic kafka mongo logz loki elasticsearch bigquery"
 
 def extract_value(str)
   unless str
@@ -13,25 +10,16 @@ def extract_value(str)
   str.gsub(/ /, "").downcase
 end
 
-def extract_entity_interval(entity, default_interval)
-  treated_entity_list = ENV['LOG_EXPORT_CONTAINER_EXTRACT_AUDIT'].to_s.match /#{entity}\/+(\d+)/
-  if treated_entity_list != nil
-    interval = treated_entity_list[1]
-  else
-    interval = default_interval
+def extract_entity_interval(entity)
+  if entity == 'activities'
+    extract_interval = extract_activities_interval
+    return extract_interval ? "#{extract_interval}m" : ""
   end
+  entity_interval_match = ENV['LOG_EXPORT_CONTAINER_EXTRACT_AUDIT'].to_s.match /#{entity}\/(\d+)/
+  interval = entity_interval_match ? entity_interval_match[1] : 480
   "#{interval}m"
 end
 
-def extract_activity_interval
-  if ENV['LOG_EXPORT_CONTAINER_EXTRACT_AUDIT_ACTIVITIES_INTERVAL'] != nil
-    interval = "#{ENV['LOG_EXPORT_CONTAINER_EXTRACT_AUDIT_ACTIVITIES_INTERVAL']}m"
-  else
-    interval = extract_entity_interval("activities", "15")
-  end
-  interval
-end
-
 def monitoring_conf
   monitoring_enabled = extract_value(ENV['LOG_EXPORT_CONTAINER_ENABLE_MONITORING']) == "true"
   if monitoring_enabled
@@ -72,25 +60,17 @@ def decode_chunk_events_conf
   end
 end
 
-def input_extract_audit_activities_conf
+def input_extract_audit_entities_conf(entity)
   extract_activities = extract_value(ENV['LOG_EXPORT_CONTAINER_EXTRACT_AUDIT_ACTIVITIES'])
-  extracted_entities = extract_value(ENV['LOG_EXPORT_CONTAINER_EXTRACT_AUDIT'])
-  unless extract_activities == "true" || extracted_entities.match(/activities/)
+  extract_entities = extract_value(ENV['LOG_EXPORT_CONTAINER_EXTRACT_AUDIT'])
+  if entity == "activities" && extract_activities != "true" && !extract_entities.match(/activities/)
     return
-  end
-  read_file = File.read("#{ETC_DIR}/input-extract-audit-activities.conf")
-  read_file['$interval'] = extract_activity_interval
-  read_file
-end
-
-def input_extract_audit_entity_conf(entity)
-  extract_audit = extract_value(ENV['LOG_EXPORT_CONTAINER_EXTRACT_AUDIT'])
-  unless extract_audit.match(/#{entity}/)
+  elsif entity != "activities" && !extract_entities.match(/#{entity}/)
     return
   end
-  read_file = File.read("#{ETC_DIR}/input-extract-audit-entity.conf")
+  read_file = File.read("#{ETC_DIR}/input-extract-audit-entities.conf")
   read_file['$tag'] = AUDIT_ENTITY_TYPES[entity]
-  read_file['$interval'] = extract_entity_interval(entity, "480")
+  read_file['$interval'] = extract_entity_interval(entity)
   read_file.gsub!("$entity", entity)
   read_file
 end

create-conf.rb

@@ -6,11 +6,11 @@
 def create_file
   File.open("#{ETC_DIR}/fluent.conf", "w") do |f|
     f.write(input_conf)
-    f.write(input_extract_audit_activities_conf)
     f.write(monitoring_conf)
-    f.write(input_extract_audit_entity_conf("resources"))
-    f.write(input_extract_audit_entity_conf("users"))
-    f.write(input_extract_audit_entity_conf("roles"))
+    f.write(input_extract_audit_entities_conf("activities"))
+    f.write(input_extract_audit_entities_conf("resources"))
+    f.write(input_extract_audit_entities_conf("users"))
+    f.write(input_extract_audit_entities_conf("roles"))
     f.write(default_classify_conf)
     f.write(custom_classify_conf)
     f.write(File.read("#{ETC_DIR}/process.conf"))

docker-compose.yml

@@ -6,7 +6,7 @@ services:
       - "5140:5140"
       - "24321:24321"
     environment:
-      # IMPORTANT: Do not enclose values in double or single quotes 
+      # IMPORTANT: Do not enclose values in double or single quotes
 
       # Required variables
       - LOG_EXPORT_CONTAINER_INPUT=syslog-json # possible values = syslog-json, syslog-csv, tcp-json, tcp-csv, file-json and file-csv
@@ -26,3 +26,4 @@ services:
       # For LOG_EXPORT_CONTAINER_OUTPUT=loki refer to docs/CONFIGURE_LOKI.md
       # For LOG_EXPORT_CONTAINER_OUTPUT=elasticsearch refer to docs/CONFIGURE_ELASTICSEARCH.md
       # For LOG_EXPORT_CONTAINER_OUTPUT=bigquery refer to docs/CONFIGURE_BIGQUERY.md
+

docs/CONFIGURE_LOG_EXPORT_CONTAINER.md

@@ -75,7 +75,7 @@ When using `syslog-json` or `tcp-json` specify `LOG_EXPORT_CONTAINER_DECODE_CHUN
 
 ### strongDM Audit
 
-To use strongDM Audit specify `LOG_EXPORT_CONTAINER_EXTRACT_AUDIT=activities/15 resources/480 users/480 roles/480`, it'll store the logs from strongDM Audit in your specified output. You can configure this option with whatever features and log extraction interval you want. **It's not enabled by default**. Please refer to [CONFIGURE_SDM_AUDIT](inputs/CONFIGURE_SDM_AUDIT.md) for more information.
+When using strongDM Audit specify `LOG_EXPORT_CONTAINER_EXTRACT_AUDIT=activities/15 resources/480 users/480 roles/480`, it'll store the logs from strongDM Audit in your specified output. You can configure this option with whatever features and log extraction interval you want. **It's not enabled by default**. Please refer to [CONFIGURE_SDM_AUDIT](inputs/CONFIGURE_SDM_AUDIT.md) for more information.
 
 We moved the section describing the variable `LOG_EXPORT_CONTAINER_EXTRACT_AUDIT_ACTIVITIES` to [CONFIGURE_SDM_AUDIT](inputs/CONFIGURE_SDM_AUDIT.md) file. Please refer to it to know the behavior with this two variables.
 
@@ -94,3 +94,9 @@ By default, the container just classifies the different log traces (e.g. start,
 The current version of the container only supports rsyslog, please refer to the image below to observe a typical configuration:
 
 <img src="https://user-images.githubusercontent.com/313803/123248041-76aab480-d4b5-11eb-8070-9da9619f02f7.png" data-canonical-src="https://user-images.githubusercontent.com/313803/123248041-76aab480-d4b5-11eb-8070-9da9619f02f7.png" width="50%" height="50%" />
+
+## High Availability
+
+It's possible to set up a high availability environment using an AWS Load Balancer with more than one LEC instance. Please refer to this tutorial.
+
+https://user-images.githubusercontent.com/82273420/167867989-2eb64a2e-ce18-4b6b-998e-88e5a34a70e7.mp4

docs/deploy_log_export_container/CONFIGURE_LOCAL_ENV.md

@@ -31,10 +31,10 @@ bundle install
 ### Ruby tests
 ```
 ruby test/run_test.rb
+bundle exec rspec test
 ```
 
-### Python tests
+### Run locally
 ```
-pytest
 dev-tools/start-local.sh
 ```

docs/inputs/CONFIGURE_SDM_AUDIT.md

@@ -7,22 +7,45 @@ nav_order: 9
 
 # Configure Audit
 
+First, to make this work, you need to provide the following variable:
+
+- **SDM_ADMIN_TOKEN**. Admin Token created in SDM Web UI. You need to check the options `Activities`, `Datasources`, `Users`, `Roles` and `Gateways`
+to have permissions to extract all logs from the SDM CLI audit command.
+
+**NOTE**: if you intend to run LEC locally, you'll need to install the [SDM CLI](https://www.strongdm.com/docs/user-guide/client-installation).
+
+## Configure Periodic Audit Data Extraction
+
 The Log Export Container uses [fluentd input exec plugin](https://docs.fluentd.org/input/exec) to extract the logs from strongDM Audit command.
-To export the logs about activities, resources, users and roles coming from strongDM Audit command, you need to specify the value of the following variable with the name of the entity (activities, resources, users or roles) and the extract interval in minutes (you should follow the pattern shown below where we have `entity_name/extract_interval` space-separated):
+To export the logs about activities, resources, users and roles coming from strongDM Audit command, you need to specify the value of the following
+variable with the name of the entity (activities, resources, users or roles) and the extract interval in minutes (you should follow the syntax
+shown below where we have `entity_name/extract_interval` space-separated):
 
 ```
 LOG_EXPORT_CONTAINER_EXTRACT_AUDIT=activities/15 resources/480 users/480 roles/480
 ```
 
-It is worth noting that if you don't specify the interval value after each `/`, the default interval values for each entity will be as defined above.
-
-To make this work, you need to provide the following variable:
-
-- **SDM_ADMIN_TOKEN**. Admin Token created in SDM Web UI. You need to check the options `Activities`, `Datasources`, `Users`, `Roles` and `Gateways` to have permissions to extract all logs from the strongDM audit command.
+It is worth noting that if you do not specify the interval value after each `/`, the default interval values for each entity will be as defined above.
 
 If you want to specifically extract the activity logs you can also use the variables below:
 
 - `LOG_EXPORT_CONTAINER_EXTRACT_AUDIT_ACTIVITIES=true` Variable responsible for indicating whether activity logs will be extracted, default = false.
 - `LOG_EXPORT_CONTAINER_EXTRACT_AUDIT_ACTIVITIES_INTERVAL=15` Interval in minutes for running the extractor script, default = 15.
 
 However, be aware that if these variables are informed together with `LOG_EXPORT_CONTAINER_EXTRACT_AUDIT`, their content will have priority over `LOG_EXPORT_CONTAINER_EXTRACT_AUDIT`.
+
+**NOTE**: the variables `LOG_EXPORT_CONTAINER_EXTRACT_AUDIT_ACTIVITIES` and `LOG_EXPORT_CONTAINER_EXTRACT_AUDIT_ACTIVITIES_INTERVAL`
+will be deprecated. So we encourage to use the `LOG_EXPORT_CONTAINER_EXTRACT_AUDIT` variable instead.
+
+## Configure Stream
+
+If you want to see the Audit Data in real-time, use the following syntax (instead of defining the interval in minutes
+just set it as `stream`):
+
+```
+LOG_EXPORT_CONTAINER_EXTRACT_AUDIT=activities/stream
+```
+
+**NOTES**:
+- It's only possible to stream activities
+- It's only possible to stream when using a syslog (csv or json) input

fluentd/etc/classify-default-csv.conf

@@ -26,6 +26,11 @@
     pattern /event/
     tag     class.event
   </rule>
+  <rule>
+    key     8
+    pattern /activity/
+    tag     class.activity
+  </rule>
   <rule>
     key     2
     pattern /.*/

fluentd/etc/classify-syslog-csv.conf

@@ -73,3 +73,22 @@
   </record>
   remove_keys       1,2,3,4,5,6,7,8,9
 </filter>
+
+# line: {"1":"2022-06-16 13:34:56.502034 +0000 UTC","2":"xxx.xxx.xxx.xxx","3":"a-xxx","4":"Admin token","5":"user logged into the local client","6":"Admin token logged into the local client.","7":"activity","8":null,"9":null,"sourceAddress":"127.0.0.1","sourceHostname":"localhost"}
+<filter class.activity>
+  @type             record_transformer
+  enable_ruby       true
+  <record>
+    timestamp       ${record["1"].split('.').first}
+    ip_address      ${record["2"]}
+    actorUserID     ${record["3"]}
+    actorName       ${record["4"]}
+    activity        ${record["5"]}
+    description     ${record["6"]}
+    objects         ${record["7"]}
+    type            ${record["8"]}
+    sourceAddress   ${record["sourceAddress"]}
+    sourceHostname  ${record["sourceHostname"]}
+  </record>
+  remove_keys       1,2,3,4,5,6,7,8,9
+</filter>

fluentd/etc/input-extract-audit-entity.conf → fluentd/etc/input-extract-audit-entities.conf

@@ -5,5 +5,5 @@
   </parse>
   tag $tag
   run_interval $interval
-  command "ruby #{ENV['FLUENTD_DIR']}/scripts/dump_sdm_entity.rb $entity"
+  command "ruby #{ENV['FLUENTD_DIR']}/scripts/dump_sdm_entities.rb $entity"
 </source>