diff --git a/fluentd/scripts/dump_activities.rb b/fluentd/scripts/dump_activities.rb index a8122e8..19d1dfb 100644 --- a/fluentd/scripts/dump_activities.rb +++ b/fluentd/scripts/dump_activities.rb @@ -1,12 +1,19 @@ require 'json' require 'date' require_relative './parse_entities' +require_relative './dump_utils' def get_audit_activities interval_time = extract_activity_interval datetime_from = DateTime.now - (interval_time + 1.0)/(24*60) - datetime_to = DateTime.now - 1.0/(24*60) - output = `sdm audit activities -j -e --from "#{datetime_from.to_s}" --to "#{datetime_to.to_s}"` + error_file_path = "/var/log/sdm-audit-activities-errors.log" + output = backoff_retry(-> () { + datetime_to = DateTime.now - 1.0/(24*60) + `sdm audit activities -j -e --from "#{datetime_from.to_s}" --to "#{datetime_to.to_s}" 2> #{error_file_path}` + }, "activities", error_file_path) + unless output + return + end output.split("\n") end @@ -24,5 +31,8 @@ def print_activities(activities) end activities = get_audit_activities +unless activities + return +end parsed_activities = parse_activities(activities) print_activities(parsed_activities) diff --git a/fluentd/scripts/dump_sdm_entity.rb b/fluentd/scripts/dump_sdm_entity.rb index 41e25e4..ac64a48 100644 --- a/fluentd/scripts/dump_sdm_entity.rb +++ b/fluentd/scripts/dump_sdm_entity.rb @@ -1,5 +1,6 @@ require 'json' require 'date' +require_relative './dump_utils' ENTITY_TYPES = { "resources" => "resource", @@ -12,7 +13,13 @@ def entity_name_argument end def get_audit_rows - output = `sdm audit #{entity_name_argument} -j` + error_file_path = "/var/log/sdm-audit-#{entity_name_argument}-errors.log" + output = backoff_retry(-> () { + `sdm audit #{entity_name_argument} -j 2> #{error_file_path}` + }, entity_name_argument, error_file_path) + unless output + return + end output.split("\n") end diff --git a/fluentd/scripts/dump_utils.rb b/fluentd/scripts/dump_utils.rb new file mode 100644 index 0000000..6565433 --- /dev/null +++ b/fluentd/scripts/dump_utils.rb @@ -0,0 +1,29 @@ + +require 'json' + +DEFAULT_RETRY_INTERVAL = 2 +BACKOFF_MAX_RETRIES = 3 + +def backoff_retry(exec_fn, entity, error_file_path) + retries = 0 + begin + result = exec_fn.() + errors = "" + if File.exist?(error_file_path) + errors = File.read(error_file_path) + end + if errors != "" + raise StandardError.new errors.chomp + end + result + rescue StandardError => e + if retries < BACKOFF_MAX_RETRIES + puts "#{JSON.generate({"error_message" => "#{e.to_s} (retrying for the #{retries + 1} time)"})}" + sleep DEFAULT_RETRY_INTERVAL ** retries + retries += 1 + retry + else + puts "#{JSON.generate({"error_message" => "Aborting audit #{entity} extraction - the retrials count was exceeded"})}" + end + end +end diff --git a/test/plugin/test_input_filter_sdm_decode_chunk_events.rb b/test/plugin/test_input_filter_sdm_decode_chunk_events.rb index 70b84b5..fa36bf8 100644 --- a/test/plugin/test_input_filter_sdm_decode_chunk_events.rb +++ b/test/plugin/test_input_filter_sdm_decode_chunk_events.rb @@ -3,8 +3,8 @@ require 'test/unit' require 'fluent/test' require 'fluent/test/helpers' -require 'fluent/test/driver/output' require_relative '../helper' +require_relative '../stdout_utils' require_relative '../../fluentd/plugins/filter_sdm_decode_chunk_events' class TestSDMDecodeChunkEventsFilter < Test::Unit::TestCase @@ -166,13 +166,3 @@ def sample_decoded_chunk_log ] } end - -def omit_stdout(&block) - original_stdout = $stdout - $stdout = StringIO.new - begin - yield - ensure - $stdout = original_stdout - end -end diff --git a/test/scripts/test_dump_utils.rb b/test/scripts/test_dump_utils.rb new file mode 100644 index 0000000..2ed0f3b --- /dev/null +++ b/test/scripts/test_dump_utils.rb @@ -0,0 +1,91 @@ + +require 'test/unit' +require 'fluent/test' +require 'fluent/test/helpers' +require_relative '../../fluentd/scripts/dump_utils' +require_relative '../stdout_utils' + +FILE_PATH = "#{`pwd`.chomp}/sdm-audit-test-errors.log" + +class TestBackoffRetry < Test::Unit::TestCase + def cleanup + super + if File.exist?(FILE_PATH) + File.delete(FILE_PATH) + end + end + + def teardown + super + if File.exist?(FILE_PATH) + File.delete(FILE_PATH) + end + end + + def test_when_run_successfully + calls_count = 0 + result = nil + omit_stdout do + result = backoff_retry(-> () { + calls_count += 1 + mock_success + }, "test", FILE_PATH) + end + assert_true(result) + assert_true(File.exist?(FILE_PATH)) + assert_empty(File.read(FILE_PATH)) + assert_equal(1, calls_count) + end + + def test_when_fails_and_dont_exceed_retry + calls_count = 0 + result = nil + omit_stdout do + result = backoff_retry(-> () { + calls_count += 1 + mock_fail_once + }, "test", FILE_PATH) + end + assert_true(result) + assert_true(File.exist?(FILE_PATH)) + assert_empty(File.read(FILE_PATH)) + assert_equal(2, calls_count) + end + + def test_when_exceed_retry + calls_count = 0 + result = nil + omit_stdout do + result = backoff_retry(-> () { + calls_count += 1 + mock_fail + }, "test", FILE_PATH) + end + assert_nil(result) + assert_true(File.exist?(FILE_PATH)) + assert_not_empty(File.read(FILE_PATH)) + assert_equal(4, calls_count) + end + + private + def mock_success + File.new(FILE_PATH, "w+") + true + end + + def mock_fail_once + if File.exist?(FILE_PATH) + mock_success + else + mock_fail + nil + end + end + + def mock_fail + File.write(FILE_PATH, "w") do |file| + file.write("error") + end + nil + end +end diff --git a/test/stdout_utils.rb b/test/stdout_utils.rb new file mode 100644 index 0000000..7e776a9 --- /dev/null +++ b/test/stdout_utils.rb @@ -0,0 +1,12 @@ + +require 'fluent/test/driver/output' + +def omit_stdout + original_stdout = $stdout + $stdout = StringIO.new + begin + yield + ensure + $stdout = original_stdout + end +end