From f92037535e53b8e2a1303d0d236d9cb8094c6c07 Mon Sep 17 00:00:00 2001 From: Riku Rouvila Date: Tue, 27 Feb 2024 14:51:24 +0200 Subject: [PATCH] revert some changes made to fix deployment issues comments about why certain changes are potentially needed can be found here https://github.com/opencrvs/opencrvs-farajaland/pull/897 --- infrastructure/deployment/deploy.sh | 12 +---------- infrastructure/docker-compose.deploy.yml | 21 ------------------- .../docker-compose.production-deploy.yml | 14 ++----------- .../docker-compose.staging-deploy.yml | 8 ------- infrastructure/logrotate.conf | 7 ------- .../monitoring/elastalert/elastalert.yaml | 3 ++- infrastructure/server-setup/backups.yml | 10 +++++++++ .../server-setup/tasks/application.yml | 8 ------- infrastructure/setup-deploy-config.sh | 14 ------------- 9 files changed, 15 insertions(+), 82 deletions(-) diff --git a/infrastructure/deployment/deploy.sh b/infrastructure/deployment/deploy.sh index b21c137c9..ea3e8e809 100755 --- a/infrastructure/deployment/deploy.sh +++ b/infrastructure/deployment/deploy.sh @@ -351,16 +351,7 @@ configured_ssh << EOF EOF # Setup configuration files and compose file for the deployment domain -configured_ssh " - HOST=$HOST - SMTP_HOST=$SMTP_HOST - SMTP_PORT=$SMTP_PORT - ALERT_EMAIL=$ALERT_EMAIL - SENDER_EMAIL_ADDRESS=$SENDER_EMAIL_ADDRESS - DOMAIN=$DOMAIN - MINIO_ROOT_USER=$MINIO_ROOT_USER - MINIO_ROOT_PASSWORD=$MINIO_ROOT_PASSWORD - /opt/opencrvs/infrastructure/setup-deploy-config.sh $HOST | tee -a $LOG_LOCATION/setup-deploy-config.log" +configured_ssh "/opt/opencrvs/infrastructure/setup-deploy-config.sh $HOST" rotate_secrets @@ -372,7 +363,6 @@ echo echo "Waiting 2 mins for mongo to deploy before working with data. Please note it can take up to 10 minutes for the entire stack to deploy in some scenarios." echo -sleep 120 # Required as Kibana cannot be immediately contacted echo "Setting up Kibana config & alerts" while true; do diff --git a/infrastructure/docker-compose.deploy.yml b/infrastructure/docker-compose.deploy.yml index 7e7b0e834..6ffd83733 100644 --- a/infrastructure/docker-compose.deploy.yml +++ b/infrastructure/docker-compose.deploy.yml @@ -140,8 +140,6 @@ services: 'file=@/config.ndjson' ] restart: on-failure - depends_on: - - kibana volumes: # Exceed Docker config file 500 kb file limit, thus a volume mount - '/opt/opencrvs/infrastructure/monitoring/kibana/config.ndjson:/config.ndjson' @@ -182,8 +180,6 @@ services: configs: - source: kibana.{{ts}} target: /usr/share/kibana/config/kibana.yml - depends_on: - - elasticsearch logging: driver: gelf options: @@ -234,8 +230,6 @@ services: replicas: 1 restart_policy: condition: none - depends_on: - - mongo1 environment: - REPLICAS=1 - MONGODB_ADMIN_USER=${MONGODB_ADMIN_USER} @@ -344,8 +338,6 @@ services: /bin/sh -c " /usr/bin/mc admin trace --path ocrvs/* minio " - depends_on: - - minio configs: - source: minio-mc-config.{{ts}} target: /root/.mc/config.json @@ -367,8 +359,6 @@ services: image: ubuntu:bionic entrypoint: ['bash', '/usr/app/setup.sh'] restart: on-failure - depends_on: - - elasticsearch environment: - ELASTICSEARCH_HOST=elasticsearch - ELASTIC_PASSWORD=${ELASTICSEARCH_SUPERUSER_PASSWORD} @@ -405,8 +395,6 @@ services: - '/opt/opencrvs/infrastructure/monitoring/elastalert/rules:/opt/elastalert/rules' networks: - overlay_net - depends_on: - - elasticsearch deploy: labels: - 'traefik.enable=false' @@ -423,8 +411,6 @@ services: logstash: image: logstash:7.17.0 command: logstash -f /etc/logstash/logstash.conf --verbose - depends_on: - - elasticsearch ports: - '12201:12201' - '12201:12201/udp' @@ -447,9 +433,6 @@ services: replicas: 1 apm-server: image: docker.elastic.co/apm/apm-server:7.15.2 - depends_on: - - elasticsearch - - kibana cap_add: ['CHOWN', 'DAC_OVERRIDE', 'SETGID', 'SETUID'] cap_drop: ['ALL'] restart: always @@ -906,8 +889,6 @@ services: environment: - mongodb__url=mongodb://hearth:${HEARTH_MONGODB_PASSWORD}@mongo1/hearth-dev?replicaSet=rs0 - logger__level=warn - depends_on: - - mongo1 deploy: labels: - 'traefik.enable=false' @@ -930,8 +911,6 @@ services: environment: - mongo_url=mongodb://openhim:${OPENHIM_MONGODB_PASSWORD}@mongo1/openhim-dev?replicaSet=rs0 - mongo_atnaUrl=mongodb://openhim:${OPENHIM_MONGODB_PASSWORD}@mongo1/openhim-dev?replicaSet=rs0 - depends_on: - - mongo1 deploy: labels: - 'traefik.enable=false' diff --git a/infrastructure/docker-compose.production-deploy.yml b/infrastructure/docker-compose.production-deploy.yml index bcbb7e900..e2286275a 100644 --- a/infrastructure/docker-compose.production-deploy.yml +++ b/infrastructure/docker-compose.production-deploy.yml @@ -145,9 +145,7 @@ services: hearth: environment: - mongodb__url=mongodb://hearth:${HEARTH_MONGODB_PASSWORD}@mongo1,mongo2/hearth-dev?replicaSet=rs0 - depends_on: - - mongo1 - - mongo2 + deploy: replicas: 2 @@ -159,17 +157,12 @@ services: - HEARTH_MONGO_URL=mongodb://hearth:${HEARTH_MONGODB_PASSWORD}@mongo1,mongo2/hearth-dev?replicaSet=rs0 - OPENHIM_MONGO_URL=mongodb://openhim:${OPENHIM_MONGODB_PASSWORD}@mongo1,mongo2/openhim-dev?replicaSet=rs0 - WAIT_HOSTS=mongo1:27017,mongo2:27017,influxdb:8086,minio:9000,elasticsearch:9200 - depends_on: - - mongo1 - - mongo2 openhim-core: environment: - mongo_url=mongodb://openhim:${OPENHIM_MONGODB_PASSWORD}@mongo1,mongo2/openhim-dev?replicaSet=rs0 - mongo_atnaUrl=mongodb://openhim:${OPENHIM_MONGODB_PASSWORD}@mongo1,mongo2/openhim-dev?replicaSet=rs0 - depends_on: - - mongo1 - - mongo2 + deploy: replicas: 2 @@ -207,9 +200,6 @@ services: - overlay_net mongo-on-update: - depends_on: - - mongo1 - - mongo2 environment: - REPLICAS=2 diff --git a/infrastructure/docker-compose.staging-deploy.yml b/infrastructure/docker-compose.staging-deploy.yml index c658af0b3..3e4a1f121 100644 --- a/infrastructure/docker-compose.staging-deploy.yml +++ b/infrastructure/docker-compose.staging-deploy.yml @@ -145,8 +145,6 @@ services: hearth: environment: - mongodb__url=mongodb://hearth:${HEARTH_MONGODB_PASSWORD}@mongo1/hearth-dev?replicaSet=rs0 - depends_on: - - mongo1 deploy: replicas: 1 @@ -158,15 +156,11 @@ services: - HEARTH_MONGO_URL=mongodb://hearth:${HEARTH_MONGODB_PASSWORD}@mongo1/hearth-dev?replicaSet=rs0 - OPENHIM_MONGO_URL=mongodb://openhim:${OPENHIM_MONGODB_PASSWORD}@mongo1/openhim-dev?replicaSet=rs0 - WAIT_HOSTS=mongo1:27017,influxdb:8086,minio:9000,elasticsearch:9200 - depends_on: - - mongo1 openhim-core: environment: - mongo_url=mongodb://openhim:${OPENHIM_MONGODB_PASSWORD}@mongo1/openhim-dev?replicaSet=rs0 - mongo_atnaUrl=mongodb://openhim:${OPENHIM_MONGODB_PASSWORD}@mongo1/openhim-dev?replicaSet=rs0 - depends_on: - - mongo1 deploy: replicas: 1 @@ -175,8 +169,6 @@ services: replicas: 1 mongo-on-update: - depends_on: - - mongo1 environment: - REPLICAS=1 diff --git a/infrastructure/logrotate.conf b/infrastructure/logrotate.conf index 91421677c..874c6af92 100644 --- a/infrastructure/logrotate.conf +++ b/infrastructure/logrotate.conf @@ -58,13 +58,6 @@ include /etc/logrotate.d rotate 1 } -/var/log/setup-deploy-config.log { - missingok - monthly - create 0660 root application - rotate 1 -} - /var/log/rotate-secrets.log { missingok monthly diff --git a/infrastructure/monitoring/elastalert/elastalert.yaml b/infrastructure/monitoring/elastalert/elastalert.yaml index b74db9981..1284704a6 100644 --- a/infrastructure/monitoring/elastalert/elastalert.yaml +++ b/infrastructure/monitoring/elastalert/elastalert.yaml @@ -16,7 +16,8 @@ buffer_time: es_host: elasticsearch es_port: 9200 - +es_username: '{{ES_USERNAME}}' +es_password: '{{ES_PASSWORD}}' writeback_index: elastalert_status alert_time_limit: diff --git a/infrastructure/server-setup/backups.yml b/infrastructure/server-setup/backups.yml index ca6b9d31f..12fd42540 100644 --- a/infrastructure/server-setup/backups.yml +++ b/infrastructure/server-setup/backups.yml @@ -87,6 +87,16 @@ vars: manager_hostname: "{{ groups['docker-manager-first'][0] }}" tasks: + - name: Ensure backup user is present + user: + name: '{{ external_backup_server_user }}' + state: present + create_home: true + home: '/home/{{ external_backup_server_user }}' + shell: /bin/bash + tags: + - backups + - set_fact: external_backup_server_user_home: '/home/{{ external_backup_server_user }}' tags: diff --git a/infrastructure/server-setup/tasks/application.yml b/infrastructure/server-setup/tasks/application.yml index f5ca9b2ed..4d70e66b4 100644 --- a/infrastructure/server-setup/tasks/application.yml +++ b/infrastructure/server-setup/tasks/application.yml @@ -33,14 +33,6 @@ groups: application append: yes -- name: Create deploy logfile - ansible.builtin.file: - path: /var/log/setup-deploy-config.log - owner: '{{ ansible_user }}' - group: 'application' - state: touch - mode: 'u+rwX,g+rwX,o-rwx' - - name: Create secret logfile ansible.builtin.file: path: /var/log/rotate-secrets.log diff --git a/infrastructure/setup-deploy-config.sh b/infrastructure/setup-deploy-config.sh index 6ce09263d..111956993 100755 --- a/infrastructure/setup-deploy-config.sh +++ b/infrastructure/setup-deploy-config.sh @@ -16,7 +16,6 @@ echo "Setting up deployment config for $HOST - `date --iso-8601=ns`" # Set hostname in openhim-console config sed -i "s/{{hostname}}/$HOST/g" /opt/opencrvs/infrastructure/openhim-console-config.deploy.json - # Set hostname in compose file for file in /opt/opencrvs/infrastructure/docker-compose*.yml; do sed -i "s/{{hostname}}/$HOST/g" "$file" @@ -26,19 +25,6 @@ done KIBANA_ENCRYPTION_KEY=`uuidgen` sed -i "s/{{KIBANA_ENCRYPTION_KEY}}/$KIBANA_ENCRYPTION_KEY/g" /opt/opencrvs/infrastructure/monitoring/kibana/kibana.yml -# Move metabase file -mv /opt/opencrvs/infrastructure/metabase.init.db.sql /data/metabase/metabase.init.db.sql - -# Replace environment variables from all alert definition files -for file in /opt/opencrvs/infrastructure/monitoring/elastalert/rules/*.yaml; do - sed -i -e "s%{{HOST}}%$1%" $file - sed -i -e "s%{{SMTP_HOST}}%$SMTP_HOST%" $file - sed -i -e "s%{{SMTP_PORT}}%$SMTP_PORT%" $file - sed -i -e "s%{{ALERT_EMAIL}}%$ALERT_EMAIL%" $file - sed -i -e "s%{{SENDER_EMAIL_ADDRESS}}%$SENDER_EMAIL_ADDRESS%" $file - sed -i -e "s%{{DOMAIN}}%$DOMAIN%" $file -done - sed -i -e "s%{{MINIO_ROOT_USER}}%$MINIO_ROOT_USER%" /opt/opencrvs/infrastructure/mc-config/config.json sed -i -e "s%{{MINIO_ROOT_PASSWORD}}%$MINIO_ROOT_PASSWORD%" /opt/opencrvs/infrastructure/mc-config/config.json