From 29007ea932f5572f7df6567970ffd4c9104fd9a5 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Mon, 2 Dec 2024 11:40:01 -0500 Subject: [PATCH 1/9] turning off until replicas is done --- opensearch-production.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-production.yml b/opensearch-production.yml index 31fa96d..4526a37 100644 --- a/opensearch-production.yml +++ b/opensearch-production.yml @@ -24,7 +24,7 @@ instance_groups: vm_type: r6i.large - name: ingestor - instances: 7 + instances: 0 vm_type: r6i.xlarge.logsearch.ingestor - name: maintenance From caf0bddc077c1764afb1602b558421b232753423 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Mon, 2 Dec 2024 18:09:34 -0500 Subject: [PATCH 2/9] removing cloudwatch ingestor --- opensearch-jobs.yml | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/opensearch-jobs.yml b/opensearch-jobs.yml index 378a113..ec4dab2 100644 --- a/opensearch-jobs.yml +++ b/opensearch-jobs.yml @@ -334,36 +334,6 @@ instance_groups: http_host: 127.0.0.1 jvm_options: - -Dlog4j2.formatMsgNoLookups=true - - consumes: *consumes-opensearch-manager - name: ingestor_cloudwatch - properties: - logstash: - jvm_options: - - -Dlog4j2.formatMsgNoLookups=true - queue: - max_bytes: 30gb - logstash_ingestor: - cloudwatch: - region: ((region)) - prefix: ((cloudwatch_prefix)) - syslog_tls: - port: 6972 - ssl_cert: ((ingestor_syslog_server_tls.certificate)) - ssl_key: ((ingestor_syslog_server_tls.private_key)) - logstash_parser: - opensearch: - data_hosts: - - localhost - index: ((alias)) - index_type: '%{@type}' - ssl: - ca: ((opensearch_node.ca)) - certificate: ((logstash.certificate)) - private_key: ((logstash.private_key)) - provides: - ingestor: - as: ingestor_cloudwtch - release: opensearch - name: ingestor_syslog consumes: *consumes-opensearch-manager properties: From 512c57789b7ecf3c46d6c42c8125eaf4c28f3c3a Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Mon, 2 Dec 2024 18:13:09 -0500 Subject: [PATCH 3/9] removing cloudwatch ingestor --- ci/pipeline.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 362f6c8..2c45bd9 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -503,13 +503,13 @@ jobs: - get: pipeline-tasks - get: deploy-logs-opensearch-config trigger: true - passed: [smoke-tests-staging] + #passed: [smoke-tests-staging] - get: opensearch-release trigger: true - passed: [smoke-tests-staging] + #passed: [smoke-tests-staging] - get: opensearch-stemcell-jammy trigger: true - passed: [smoke-tests-staging] + #passed: [smoke-tests-staging] - get: terraform-yaml resource: terraform-yaml-production trigger: true @@ -775,7 +775,7 @@ resources: source: commit_verification_keys: ((cloud-gov-pgp-keys)) uri: https://github.com/cloud-gov/cg-deploy-opensearch.git - branch: main + branch: partial - name: opensearch-stemcell-jammy source: From 8135f36bb23269d027955e3190bf3ed0b56daec2 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Mon, 2 Dec 2024 18:15:37 -0500 Subject: [PATCH 4/9] don't remove ingestors --- opensearch-production.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-production.yml b/opensearch-production.yml index 4526a37..31fa96d 100644 --- a/opensearch-production.yml +++ b/opensearch-production.yml @@ -24,7 +24,7 @@ instance_groups: vm_type: r6i.large - name: ingestor - instances: 0 + instances: 7 vm_type: r6i.xlarge.logsearch.ingestor - name: maintenance From 42d3159798a8c77691dce063323567208c371ee2 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Mon, 2 Dec 2024 18:43:30 -0500 Subject: [PATCH 5/9] restoring --- ci/pipeline.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 2c45bd9..b4f9b69 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -503,13 +503,13 @@ jobs: - get: pipeline-tasks - get: deploy-logs-opensearch-config trigger: true - #passed: [smoke-tests-staging] + passed: [smoke-tests-staging] - get: opensearch-release trigger: true - #passed: [smoke-tests-staging] + passed: [smoke-tests-staging] - get: opensearch-stemcell-jammy trigger: true - #passed: [smoke-tests-staging] + passed: [smoke-tests-staging] - get: terraform-yaml resource: terraform-yaml-production trigger: true From e935d19afd1700f66688a2f755668554340ea785 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Mon, 2 Dec 2024 18:44:42 -0500 Subject: [PATCH 6/9] testing cloudwatch_logs --- opensearch-jobs.yml | 56 +++++++++++++++++++++++++++++++++++++++ opensearch-production.yml | 4 +++ 2 files changed, 60 insertions(+) diff --git a/opensearch-jobs.yml b/opensearch-jobs.yml index ec4dab2..f9b58a8 100644 --- a/opensearch-jobs.yml +++ b/opensearch-jobs.yml @@ -376,4 +376,60 @@ instance_groups: - logs-opensearch-ingestor-profile - 15GB_ephemeral_disk networks: + - name: services + +- name: ingestor_cloudwatch + instances: 1 + jobs: + - name: bpm + release: bpm + - name: opensearch + release: opensearch + consumes: *consumes-opensearch-manager + properties: + opensearch: + heap_size: 1G + http_host: 127.0.0.1 + jvm_options: + - -Dlog4j2.formatMsgNoLookups=true + - consumes: *consumes-opensearch-manager + name: ingestor_cloudwatch + properties: + logstash: + jvm_options: + - -Dlog4j2.formatMsgNoLookups=true + queue: + max_bytes: 30gb + logstash_ingestor: + cloudwatch: + region: ((region)) + prefix: ((cloudwatch_prefix)) + syslog_tls: + port: 6972 + ssl_cert: ((ingestor_syslog_server_tls.certificate)) + ssl_key: ((ingestor_syslog_server_tls.private_key)) + logstash_parser: + opensearch: + data_hosts: + - localhost + index: ((alias)) + index_type: '%{@type}' + ssl: + ca: ((opensearch_node.ca)) + certificate: ((logstash.certificate)) + private_key: ((logstash.private_key)) + provides: + ingestor: + as: ingestor_cloudwtch + release: opensearch + - name: deployment_lookup_config + release: opensearch + persistent_disk_type: logs_opensearch_ingestor + stemcell: default + azs: [z1] + vm_type: t3.medium + vm_extensions: + - logs-opensearch-ingestor-profile + - 15GB_ephemeral_disk + networks: - name: services \ No newline at end of file diff --git a/opensearch-production.yml b/opensearch-production.yml index 31fa96d..7761bc7 100644 --- a/opensearch-production.yml +++ b/opensearch-production.yml @@ -27,6 +27,10 @@ instance_groups: instances: 7 vm_type: r6i.xlarge.logsearch.ingestor +- name: ingestor_cloudwatch + instances: 1 + vm_type: r6i.large.logsearch.ingestor + - name: maintenance vm_type: t3.large instances: 1 From b844aecd2ae898bb10367cfc4a99dafdd0a2f068 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Mon, 2 Dec 2024 19:07:19 -0500 Subject: [PATCH 7/9] testing cloudwatch_logs --- opsfiles/enable-node-tls.yml | 18 ++++++++++++++++++ opsfiles/enable-proxy-auth.yml | 5 ++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/opsfiles/enable-node-tls.yml b/opsfiles/enable-node-tls.yml index 9ac2a84..3f2bc5f 100644 --- a/opsfiles/enable-node-tls.yml +++ b/opsfiles/enable-node-tls.yml @@ -123,6 +123,24 @@ path: /instance_groups/name=ingestor/jobs/name=opensearch/properties?/opensearch?/http?/ssl? value: *http-tls-properties +# ingestor_cloudwatch +- type: replace + path: /instance_groups/name=ingestor_cloudwatch/jobs/name=opensearch/properties?/opensearch?/http_host? + value: 127.0.0.1 + +- type: replace + path: /instance_groups/name=ingestor_cloudwatch/jobs/name=opensearch/properties/opensearch?/admin? + value: *admin-tls-properties + +- type: replace + path: /instance_groups/name=ingestor_cloudwatch/jobs/name=opensearch/properties/opensearch?/node?/ssl? + value: *node-tls-properties + +- type: replace + path: /instance_groups/name=ingestor_cloudwatch/jobs/name=opensearch/properties?/opensearch?/http?/ssl? + value: *http-tls-properties + + # smoke_tests # - type: replace # path: /instance_groups/name=maintenance/jobs/name=smoke_tests/properties?/smoke_tests?/opensearch?/ssl? diff --git a/opsfiles/enable-proxy-auth.yml b/opsfiles/enable-proxy-auth.yml index 027dbd2..30b037c 100644 --- a/opsfiles/enable-proxy-auth.yml +++ b/opsfiles/enable-proxy-auth.yml @@ -33,7 +33,10 @@ - type: replace path: /instance_groups/name=ingestor/jobs/name=opensearch/properties?/opensearch?/enable_proxy_auth value: true - +# ingestor_cloudwatch +- type: replace + path: /instance_groups/name=ingestor_cloudwatch/jobs/name=opensearch/properties?/opensearch?/enable_proxy_auth + value: true # add variable for auth proxy certs - type: replace path: /variables/name=auth_proxy? From 5488d492b7521a4d9a3abaf3e3e66eb96ae017f7 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Mon, 2 Dec 2024 20:16:03 -0500 Subject: [PATCH 8/9] updating to m6i and changing to independent job --- opensearch-development.yml | 3 +++ opensearch-jobs.yml | 2 +- opensearch-production.yml | 1 - opensearch-staging.yml | 3 +++ 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/opensearch-development.yml b/opensearch-development.yml index 4b2600d..f096e80 100644 --- a/opensearch-development.yml +++ b/opensearch-development.yml @@ -24,6 +24,9 @@ instance_groups: instances: 1 vm_type: m6i.large +- name: ingestor_cloudwatch + instances: 1 + - name: maintenance vm_type: t3.large instances: 1 diff --git a/opensearch-jobs.yml b/opensearch-jobs.yml index f9b58a8..16afd55 100644 --- a/opensearch-jobs.yml +++ b/opensearch-jobs.yml @@ -427,7 +427,7 @@ instance_groups: persistent_disk_type: logs_opensearch_ingestor stemcell: default azs: [z1] - vm_type: t3.medium + vm_type: m6i.large vm_extensions: - logs-opensearch-ingestor-profile - 15GB_ephemeral_disk diff --git a/opensearch-production.yml b/opensearch-production.yml index 7761bc7..8e82693 100644 --- a/opensearch-production.yml +++ b/opensearch-production.yml @@ -29,7 +29,6 @@ instance_groups: - name: ingestor_cloudwatch instances: 1 - vm_type: r6i.large.logsearch.ingestor - name: maintenance vm_type: t3.large diff --git a/opensearch-staging.yml b/opensearch-staging.yml index fd5e2a9..01dca52 100644 --- a/opensearch-staging.yml +++ b/opensearch-staging.yml @@ -27,6 +27,9 @@ instance_groups: instances: 1 vm_type: m6i.large +- name: ingestor_cloudwatch + instances: 1 + - name: maintenance vm_type: t3.large instances: 1 From 2fb049492ab85b84065a97ae0fa15d8455fd5812 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Tue, 3 Dec 2024 09:49:57 -0500 Subject: [PATCH 9/9] updating branch to main --- ci/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/pipeline.yml b/ci/pipeline.yml index b4f9b69..362f6c8 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -775,7 +775,7 @@ resources: source: commit_verification_keys: ((cloud-gov-pgp-keys)) uri: https://github.com/cloud-gov/cg-deploy-opensearch.git - branch: partial + branch: main - name: opensearch-stemcell-jammy source: