Merge pull request #114 from cloud-gov/s3-ingestor

finishing logsearch and adding more opensearch
cloud-gov · Nov 20, 2024 · 8be0163 · 8be0163
2 parents 5dfb557 + 13d2792
commit 8be0163
Show file tree

Hide file tree

Showing 4 changed files with 250 additions and 2 deletions.
diff --git a/opsfiles/add-prod-logsearch-ingestors.yml b/opsfiles/add-prod-logsearch-ingestors.yml
@@ -5,7 +5,7 @@
       bucket: ((logsearch_archive_bucket_name))
       region: ((vpc_region))
       type: syslog
-      prefix: 2024/01/14
+      prefix: 2024/10/07
     plugin: s3
 
 - type: replace

diff --git a/opsfiles/add-prod-opensearch-ingestors.yml b/opsfiles/add-prod-opensearch-ingestors.yml
@@ -149,6 +149,87 @@
       prefix: 2024/11/03
     plugin: s3
 
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_16?/jobs/name=ingestor_syslog/properties?/logstash_parser?/inputs?/0
+  value:
+    options:
+      bucket: ((opensearch_archive_bucket_name))
+      region: ((vpc_region))
+      type: syslog
+      prefix: 2024/11/02
+    plugin: s3
+
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_17?/jobs/name=ingestor_syslog/properties?/logstash_parser?/inputs?/0
+  value:
+    options:
+      bucket: ((opensearch_archive_bucket_name))
+      region: ((vpc_region))
+      type: syslog
+      prefix: 2024/11/01
+    plugin: s3
+
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_18?/jobs/name=ingestor_syslog/properties?/logstash_parser?/inputs?/0
+  value:
+    options:
+      bucket: ((opensearch_archive_bucket_name))
+      region: ((vpc_region))
+      type: syslog
+      prefix: 2024/10/31
+    plugin: s3
+
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_19?/jobs/name=ingestor_syslog/properties?/logstash_parser?/inputs?/0
+  value:
+    options:
+      bucket: ((opensearch_archive_bucket_name))
+      region: ((vpc_region))
+      type: syslog
+      prefix: 2024/10/30
+    plugin: s3
+
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_20?/jobs/name=ingestor_syslog/properties?/logstash_parser?/inputs?/0
+  value:
+    options:
+      bucket: ((opensearch_archive_bucket_name))
+      region: ((vpc_region))
+      type: syslog
+      prefix: 2024/10/29
+    plugin: s3
+
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_21?/jobs/name=ingestor_syslog/properties?/logstash_parser?/inputs?/0
+  value:
+    options:
+      bucket: ((opensearch_archive_bucket_name))
+      region: ((vpc_region))
+      type: syslog
+      prefix: 2024/10/28
+    plugin: s3
+
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_22?/jobs/name=ingestor_syslog/properties?/logstash_parser?/inputs?/0
+  value:
+    options:
+      bucket: ((opensearch_archive_bucket_name))
+      region: ((vpc_region))
+      type: syslog
+      prefix: 2024/10/27
+    plugin: s3
+
+
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_23?/jobs/name=ingestor_syslog/properties?/logstash_parser?/inputs?/0
+  value:
+    options:
+      bucket: ((opensearch_archive_bucket_name))
+      region: ((vpc_region))
+      type: syslog
+      prefix: 2024/10/26
+    plugin: s3
+
 # Ingestor_opensearch
 - type: replace
   path: /instance_groups/name=ingestor_opensearch_s3_1?/jobs/name=opensearch/properties?/opensearch?/http_host?
@@ -414,4 +495,140 @@
 
 - type: replace
   path: /instance_groups/name=ingestor_opensearch_s3_15?/jobs/name=opensearch/properties?/opensearch?/http?/ssl?
+  value: *http-tls-properties
+
+  # Ingestor_opensearch
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_16?/jobs/name=opensearch/properties?/opensearch?/http_host?
+  value: 127.0.0.1
+
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_16?/jobs/name=opensearch/properties/opensearch?/admin?
+  value: *admin-tls-properties
+
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_16?/jobs/name=opensearch/properties/opensearch?/node?/ssl?
+  value: *node-tls-properties
+
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_16?/jobs/name=opensearch/properties?/opensearch?/http?/ssl?
+  value: *http-tls-properties
+
+# Ingestor_opensearch
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_17?/jobs/name=opensearch/properties?/opensearch?/http_host?
+  value: 127.0.0.1
+
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_17?/jobs/name=opensearch/properties/opensearch?/admin?
+  value: *admin-tls-properties
+
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_17?/jobs/name=opensearch/properties/opensearch?/node?/ssl?
+  value: *node-tls-properties
+
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_17?/jobs/name=opensearch/properties?/opensearch?/http?/ssl?
+  value: *http-tls-properties
+
+# Ingestor_opensearch
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_18?/jobs/name=opensearch/properties?/opensearch?/http_host?
+  value: 127.0.0.1
+
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_18?/jobs/name=opensearch/properties/opensearch?/admin?
+  value: *admin-tls-properties
+
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_18?/jobs/name=opensearch/properties/opensearch?/node?/ssl?
+  value: *node-tls-properties
+
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_18?/jobs/name=opensearch/properties?/opensearch?/http?/ssl?
+  value: *http-tls-properties
+
+# Ingestor_opensearch
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_19?/jobs/name=opensearch/properties?/opensearch?/http_host?
+  value: 127.0.0.1
+
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_19?/jobs/name=opensearch/properties/opensearch?/admin?
+  value: *admin-tls-properties
+
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_19?/jobs/name=opensearch/properties/opensearch?/node?/ssl?
+  value: *node-tls-properties
+
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_19?/jobs/name=opensearch/properties?/opensearch?/http?/ssl?
+  value: *http-tls-properties
+
+# Ingestor_opensearch
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_20?/jobs/name=opensearch/properties?/opensearch?/http_host?
+  value: 127.0.0.1
+
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_20?/jobs/name=opensearch/properties/opensearch?/admin?
+  value: *admin-tls-properties
+
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_20?/jobs/name=opensearch/properties/opensearch?/node?/ssl?
+  value: *node-tls-properties
+
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_20?/jobs/name=opensearch/properties?/opensearch?/http?/ssl?
+  value: *http-tls-properties
+
+# Ingestor_opensearch
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_21?/jobs/name=opensearch/properties?/opensearch?/http_host?
+  value: 127.0.0.1
+
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_21?/jobs/name=opensearch/properties/opensearch?/admin?
+  value: *admin-tls-properties
+
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_21?/jobs/name=opensearch/properties/opensearch?/node?/ssl?
+  value: *node-tls-properties
+
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_21?/jobs/name=opensearch/properties?/opensearch?/http?/ssl?
+  value: *http-tls-properties
+
+# Ingestor_opensearch
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_22?/jobs/name=opensearch/properties?/opensearch?/http_host?
+  value: 127.0.0.1
+
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_22?/jobs/name=opensearch/properties/opensearch?/admin?
+  value: *admin-tls-properties
+
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_22?/jobs/name=opensearch/properties/opensearch?/node?/ssl?
+  value: *node-tls-properties
+
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_22?/jobs/name=opensearch/properties?/opensearch?/http?/ssl?
+  value: *http-tls-properties
+
+# Ingestor_opensearch
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_23?/jobs/name=opensearch/properties?/opensearch?/http_host?
+  value: 127.0.0.1
+
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_23?/jobs/name=opensearch/properties/opensearch?/admin?
+  value: *admin-tls-properties
+
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_23?/jobs/name=opensearch/properties/opensearch?/node?/ssl?
+  value: *node-tls-properties
+
+- type: replace
+  path: /instance_groups/name=ingestor_opensearch_s3_23?/jobs/name=opensearch/properties?/opensearch?/http?/ssl?
   value: *http-tls-properties
diff --git a/prod-logsearch-ingestors.yml b/prod-logsearch-ingestors.yml
@@ -91,28 +91,35 @@ instance_groups:
 
 - <<: *logsearch-ingestor-config
   name: ingestor_logsearch_7
+  instances: 0
 
 - <<: *logsearch-ingestor-config
   name: ingestor_logsearch_8
+  instances: 0
 
 - <<: *logsearch-ingestor-config
   name: ingestor_logsearch_9
+  instances: 0
 
 - <<: *logsearch-ingestor-config
   name: ingestor_logsearch_10
+  instances: 0
 
 - <<: *logsearch-ingestor-config
   name: ingestor_logsearch_11
+  instances: 0
 
 - <<: *logsearch-ingestor-config
   name: ingestor_logsearch_12
+  instances: 0
 
 - <<: *logsearch-ingestor-config
   name: ingestor_logsearch_13
   instances: 0
 
 - <<: *logsearch-ingestor-config
   name: ingestor_logsearch_14
+  instances: 1
 
 - <<: *logsearch-ingestor-config
   name: ingestor_logsearch_15

diff --git a/prod-opensearch-ingestors.yml b/prod-opensearch-ingestors.yml
@@ -113,4 +113,28 @@ instance_groups:
   name: ingestor_opensearch_s3_14
 
 - <<: *opensearch-ingestor-config
-  name: ingestor_opensearch_s3_15
+  name: ingestor_opensearch_s3_15
+
+- <<: *opensearch-ingestor-config
+  name: ingestor_opensearch_s3_16
+
+- <<: *opensearch-ingestor-config
+  name: ingestor_opensearch_s3_17
+
+- <<: *opensearch-ingestor-config
+  name: ingestor_opensearch_s3_18
+
+- <<: *opensearch-ingestor-config
+  name: ingestor_opensearch_s3_19
+
+- <<: *opensearch-ingestor-config
+  name: ingestor_opensearch_s3_20
+
+- <<: *opensearch-ingestor-config
+  name: ingestor_opensearch_s3_21
+
+- <<: *opensearch-ingestor-config
+  name: ingestor_opensearch_s3_22
+
+- <<: *opensearch-ingestor-config
+  name: ingestor_opensearch_s3_23