Merge branch 'release-v1.4.0' of github.com:opencrvs/opencrvs-farajal…

…and into re-release-v1.4.0

.github/workflows/deploy-prod.yml

@@ -7,13 +7,14 @@ on:
         type: choice
         description: Environment to deploy to
         required: true
-        default: 'production'
+        default: 'staging'
         options:
           - production
+          - staging
       core-image-tag:
         description: Core DockerHub image tag
         required: true
-        default: 'v1.4.1'
+        default: 'v1.4.0'
       countryconfig-image-tag:
         description: Your Country Config DockerHub image tag
         required: true

.github/workflows/deploy.yml

@@ -13,7 +13,7 @@ on:
       core-image-tag:
         description: Core DockerHub image tag
         required: true
-        default: 'v1.4.1'
+        default: 'v1.4.0'
       countryconfig-image-tag:
         description: Your Country Config DockerHub image tag
         required: true

.github/workflows/provision.yml

@@ -68,7 +68,6 @@ jobs:
           mongodb_admin_username: ${{ secrets.MONGODB_ADMIN_USER }}
           mongodb_admin_password: ${{ secrets.MONGODB_ADMIN_PASSWORD }}
           backup_encryption_passphrase: ${{ secrets.BACKUP_ENCRYPTION_PASSPHRASE }}
-          restore_backup_encryption_passphrase: ${{ secrets.RESTORE_BACKUP_ENCRYPTION_PASSPHRASE }}
           elasticsearch_superuser_password: ${{ secrets.ELASTICSEARCH_SUPERUSER_PASSWORD }}
           external_backup_server_ssh_port: 22
           external_backup_server_ip: ${{ secrets.BACKUP_HOST }}

CHANGELOG.md

@@ -1,6 +1,7 @@
 # Changelog
 
-## [1.4.1](https://github.com/opencrvs/opencrvs-farajaland/compare/v1.4.0...v1.4.1)
+
+## [1.4.0](https://github.com/opencrvs/opencrvs-farajaland/compare/v1.3.3...v1.4.0) (TBD)
 
 - Added examples for configuring HTTP-01, DNS-01, and manual HTTPS certificates. By default, development and QA environments use HTTP-01, while others use DNS-01.
 - All secrets & variables defined in Github Secrets are now passed automatically to the deployment script.
@@ -105,20 +106,6 @@ See [Releases](https://github.com/opencrvs/opencrvs-farajaland/releases) for rel
 
   ##### We will be deprecating the counterpart of the above mentioned handlebars that contains only the label of the specified location in a future version so we highly recommend that implementers update their certificates to use these new ones.
 
-- #### "Spouse" section in Farajaland death form
-
-  Spouse section is an optional section in death form. Going forward it will be included in Farajaland example configuration.
-
-- #### Type of ID dropdown
-  Farajaland forms will now include a dropdown to select the type of ID an individual is providing e.g. National ID, Driving License etc. instead of being restricted to only national ID number.
-- #### Number of dependents of deceased field
-  As an example of custom field, the deceased section in death form will now include the **numberOfDependants** field.
-- #### Reason for late registration field
-  The birth & death forms will include another custom field, **reasonForLateRegistration**, which makes use of "LATE_REGISTRATION_TARGET" configuration option in it's visibility conditional.
-
-## Bug fixes
-
-- Updated translations for form introduction page and sending for approval to reflect the default notification method being email.
-- Remove hard-coded conditionals from "occupation" field to make it usable in the deceased form
+### Bug fixes
 
 See [Releases](https://github.com/opencrvs/opencrvs-farajaland/releases) for release notes of older releases.

infrastructure/deployment/deploy.sh

@@ -352,6 +352,12 @@ EOF
 
 # Setup configuration files and compose file for the deployment domain
 configured_ssh "
+  HOST=$HOST
+  SMTP_HOST=$SMTP_HOST
+  SMTP_PORT=$SMTP_PORT
+  ALERT_EMAIL=$ALERT_EMAIL
+  SENDER_EMAIL_ADDRESS=$SENDER_EMAIL_ADDRESS
+  DOMAIN=$DOMAIN
   MINIO_ROOT_USER=$MINIO_ROOT_USER
   MINIO_ROOT_PASSWORD=$MINIO_ROOT_PASSWORD
   /opt/opencrvs/infrastructure/setup-deploy-config.sh $HOST | tee -a $LOG_LOCATION/setup-deploy-config.log"
@@ -366,6 +372,7 @@ echo
 echo "Waiting 2 mins for mongo to deploy before working with data. Please note it can take up to 10 minutes for the entire stack to deploy in some scenarios."
 echo
 
+sleep 120 # Required as Kibana cannot be immediately contacted
 echo "Setting up Kibana config & alerts"
 
 while true; do

infrastructure/docker-compose.deploy.yml

@@ -405,6 +405,8 @@ services:
       - '/opt/opencrvs/infrastructure/monitoring/elastalert/rules:/opt/elastalert/rules'
     networks:
       - overlay_net
+    depends_on:
+      - elasticsearch
     deploy:
       labels:
         - 'traefik.enable=false'

infrastructure/docker-compose.development-deploy.yml

@@ -15,6 +15,7 @@ services:
       - jwt-public-key.{{ts}}
     environment:
       - NODE_ENV=production
+      - QA_ENV=true
       - FHIR_URL=http://hearth:3447/fhir
       - AUTH_URL=http://auth:4040
       - APPLICATION_CONFIG_URL=http://config:2021
@@ -171,6 +172,7 @@ services:
       - MOSIP_DEATH_PROXY_CALLBACK_URL=${MOSIP_MEDIATOR_DEATH_PROXY_CALLBACK_URL}
       - MOSIP_GENERATE_AID_URL=${MOSIP_MEDIATOR_GENERATE_AID_URL}
 
+
   traefik:
     command:
       # Use HTTP-01 challenge as the web server is publicly available

infrastructure/monitoring/kibana/setup-config.sh

@@ -14,14 +14,14 @@ set -e
 docker_command="docker run --rm -v /opt/opencrvs/infrastructure/monitoring/kibana/config.ndjson:/config.ndjson --network=opencrvs_overlay_net curlimages/curl"
 
 # First delete all alerts. This is because the import doesn't remove alerts that are no longer in the config
-$docker_command --connect-timeout 60 -u elastic:${ELASTICSEARCH_SUPERUSER_PASSWORD} http://kibana:5601/api/alerting/rules/_find\?page\=1\&per_page\=100\&default_search_operator\=AND\&sort_field\=name\&sort_order\=asc | docker run --rm -i --network=opencrvs_overlay_net stedolan/jq -r '.data[].id' | while read -r id; do
-  $docker_command --connect-timeout 60 -X DELETE -H 'kbn-xsrf: true' -u elastic:${ELASTICSEARCH_SUPERUSER_PASSWORD} "http://kibana:5601/api/alerting/rule/$id"
+$docker_command --connect-timeout 60 -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD http://kibana:5601/api/alerting/rules/_find\?page\=1\&per_page\=100\&default_search_operator\=AND\&sort_field\=name\&sort_order\=asc | docker run --rm -i --network=opencrvs_overlay_net stedolan/jq -r '.data[].id' | while read -r id; do
+  $docker_command --connect-timeout 60 -X DELETE -H 'kbn-xsrf: true' -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD "http://kibana:5601/api/alerting/rule/$id"
 done
 
-$docker_command --connect-timeout 60 -u elastic:${ELASTICSEARCH_SUPERUSER_PASSWORD} -X POST http://kibana:5601/api/saved_objects/_import?overwrite=true -H 'kbn-xsrf: true' --form file=@/config.ndjson > /dev/null
+$docker_command --connect-timeout 60 -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD -X POST http://kibana:5601/api/saved_objects/_import?overwrite=true -H 'kbn-xsrf: true' --form file=@/config.ndjson > /dev/null
 
 # Re-enable all alerts. This is because after importing a config, all alerts are disabled by default
-$docker_command --connect-timeout 60 -u elastic:${ELASTICSEARCH_SUPERUSER_PASSWORD} http://kibana:5601/api/alerting/rules/_find\?page\=1\&per_page\=100\&default_search_operator\=AND\&sort_field\=name\&sort_order\=asc | docker run --rm -i --network=opencrvs_overlay_net stedolan/jq -r '.data[].id' | while read -r id; do
-  $docker_command --connect-timeout 60 -X POST -H 'kbn-xsrf: true' -u elastic:${ELASTICSEARCH_SUPERUSER_PASSWORD} "http://kibana:5601/api/alerting/rule/$id/_disable"
-  $docker_command --connect-timeout 60 -X POST -H 'kbn-xsrf: true' -u elastic:${ELASTICSEARCH_SUPERUSER_PASSWORD} "http://kibana:5601/api/alerting/rule/$id/_enable"
+$docker_command --connect-timeout 60 -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD http://kibana:5601/api/alerting/rules/_find\?page\=1\&per_page\=100\&default_search_operator\=AND\&sort_field\=name\&sort_order\=asc | docker run --rm -i --network=opencrvs_overlay_net stedolan/jq -r '.data[].id' | while read -r id; do
+  $docker_command --connect-timeout 60 -X POST -H 'kbn-xsrf: true' -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD "http://kibana:5601/api/alerting/rule/$id/_disable"
+  $docker_command --connect-timeout 60 -X POST -H 'kbn-xsrf: true' -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD "http://kibana:5601/api/alerting/rule/$id/_enable"
 done

infrastructure/server-setup/backups.yml

@@ -87,16 +87,6 @@
   vars:
     manager_hostname: "{{ groups['docker-manager-first'][0] }}"
   tasks:
-    - name: Ensure backup user is present
-      user:
-        name: '{{ external_backup_server_user }}'
-        state: present
-        create_home: true
-        home: '/home/{{ external_backup_server_user }}'
-        shell: /bin/bash
-      tags:
-        - backups
-
     - set_fact:
         external_backup_server_user_home: '/home/{{ external_backup_server_user }}'
       tags:

infrastructure/server-setup/playbook.yml

@@ -34,7 +34,9 @@
 
 - hosts: docker-manager-first, docker-workers
   become: yes
-  become_method: sudo
+  become_method: sudo  
+  vars:
+    crontab_user: root
   tasks:
     - include_tasks:
         file: tasks/application.yml

infrastructure/server-setup/qa.yml

@@ -43,6 +43,11 @@ all:
         state: present
         sudoer: false
         two_factor: false
+      - name: ashikul
+        ssh_keys:
+          - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDFr/v3hUGEbc2wQsDLCmqLrwiz964yVrnLZ6kafemjmX8aRGLp1CNFvrZ674SLnXidZGMkx9d5xVvv8IdFR3R50MqSqfolF43MV34/JVHjQHh9Vk4MJT/3GIaeNmr2GQ/38qAmt2BQn1ecnb7FjNO2bFvHokLhm2wCXt+A4avuTgJe0p4e6uu01IHeIzDb5sPzZ3ID0h6jJnjEDcET+Lf5NGpCjn7YKhLhBWSSl9cXQdOGLzNzg3aBk32kgJ1beP1funSeVd0jniJPZeZRC1G/kRdqBUOHKiENtwgquzZxXzdHkZV9+4mF7YGlx6LpQdNuDpW7JADtYNldtdbexdyfrgNoRzKwyMmaKNDbeHd1FsIHSDJmGm9hCoLTM2dEtsGzgghfe0tat8sOWmsj5v2en0V8rKV+w8OQEmHtaQkgMjqmZaAnd8uWiB2xIbrUuax5Pq8zkj37xnfbRxUPOEkMlOUbhh1wzGbqeUEB7nbv/vXZxwC0b7ryMk5egBP+0ZRONsdib9RkSTr3B9uSb7iTOQftdhy+CTqqOq+6s+TyC2qnu12B1WZb9sx9jQl0mBHd9gx/FgYDs8jfIr2vF4jRkejW/moaVqvCd/FLyS91eCMXQjIXdGKWKPUUL7GEBqdZRLnYSJOqgPp9sk1+NEvMabTXlWmoUjaShq8z+o7JsQ== [email protected]
+        state: present
+        sudoer: true
     # If the machine is repurposed to also be the backup host, we need to add more keys to the authorized_keys file so that
     # when the application servers get provisioned, the provision user of this machine can be used.
     additional_keys_for_provisioning_user:

infrastructure/server-setup/staging.yml

@@ -15,7 +15,6 @@ all:
       - 165.22.110.53
     enable_backups: false
     periodic_restore_from_backup: true
-    # restore_backup_encryption_passphrase: Defined in --extra-vars by the provisioning pipeline
     # external_backup_server_ssh_port: Defined in --extra-vars by the provisioning pipeline
     # external_backup_server_ip: Defined in --extra-vars by the provisioning pipeline
     users:
@@ -30,6 +29,16 @@ all:
           - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDECqHO65UpyrrO8uueD06RxGaVVq22f152Rf8qVQQAAIGAMu6gCs7ztlZ8a3yQgSEIjM/Jl1/RqIVs6CziTEef74nLFTZ5Ufz3CLRVgdebBeSBEmhTfTUV0HLkSyNzwKFpuzJxucGd72ulPvEp6eHvyJAPJz37YcU8cjaL1v05T6s2ee99li35GlDDtCzfjVV4ZPAg5JdfWuTj41RAVC0LQhk2/NB4qEu37UxGGjhRFSjBEsS5LxI9QfvgrsHpl/VOn+soH7ZkK7kS6qRgNP/uYsXRWXhHaamcl5OX68gJWTbrW6c7PCqlbCWGnsHJswCmqPIthwXXMfC7ULDNLSKG6mslAt5Dyc8/MCr3vTW7pDyr2d0FvvY86SMQUggxv3qF7TZewqfX1bhK0fMLarIxVMQ1RFo//wN9QGA+2we8rxd2Y1Kr1DBuJyuwXPfv+Exo8yNYQ+x/AYH5k6UVcSYuaB8eYmplG2KQCxt8RBFtoChrwOKNRWLqXdKyfpdp5XmnnWxPvR95gf3h3yLocVYkF0i0uvKKJ0vt8J0Ezfkdfow0B1kUg5bPXKJROX7PwbaCPdYcxyDaO6wwOigRnSmoFvkH1pLb4j1RQAXcX531CHgfN6Izi/h0mpMS4bnyIUcv2GQr+h4z4TxcCtj7qpH2y6yw7XG12jVh7TfeesXG2Q== [email protected]
         state: present
         sudoer: true
+      - name: tameem
+        ssh_keys:
+          - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGUprcQyUFYwRto0aRpgriR95C1pgNxrQ0lEWEe1D8he [email protected]
+        state: present
+        sudoer: true
+      - name: ashikul
+        ssh_keys:
+          - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDFr/v3hUGEbc2wQsDLCmqLrwiz964yVrnLZ6kafemjmX8aRGLp1CNFvrZ674SLnXidZGMkx9d5xVvv8IdFR3R50MqSqfolF43MV34/JVHjQHh9Vk4MJT/3GIaeNmr2GQ/38qAmt2BQn1ecnb7FjNO2bFvHokLhm2wCXt+A4avuTgJe0p4e6uu01IHeIzDb5sPzZ3ID0h6jJnjEDcET+Lf5NGpCjn7YKhLhBWSSl9cXQdOGLzNzg3aBk32kgJ1beP1funSeVd0jniJPZeZRC1G/kRdqBUOHKiENtwgquzZxXzdHkZV9+4mF7YGlx6LpQdNuDpW7JADtYNldtdbexdyfrgNoRzKwyMmaKNDbeHd1FsIHSDJmGm9hCoLTM2dEtsGzgghfe0tat8sOWmsj5v2en0V8rKV+w8OQEmHtaQkgMjqmZaAnd8uWiB2xIbrUuax5Pq8zkj37xnfbRxUPOEkMlOUbhh1wzGbqeUEB7nbv/vXZxwC0b7ryMk5egBP+0ZRONsdib9RkSTr3B9uSb7iTOQftdhy+CTqqOq+6s+TyC2qnu12B1WZb9sx9jQl0mBHd9gx/FgYDs8jfIr2vF4jRkejW/moaVqvCd/FLyS91eCMXQjIXdGKWKPUUL7GEBqdZRLnYSJOqgPp9sk1+NEvMabTXlWmoUjaShq8z+o7JsQ== [email protected]
+        state: present
+        sudoer: true
 docker-manager-first:
   hosts:
     farajaland-staging:

infrastructure/server-setup/tasks/backups/crontab.yml

@@ -16,19 +16,19 @@
     periodic_restore_from_backup: false
   when: periodic_restore_from_backup is not defined
 
-- name: Throw an error if periodic_restore_from_backup is true but restore_backup_encryption_passphrase is not defined
+- name: Throw an error if periodic_restore_from_backup is true but backup_encryption_passphrase is not defined
   fail:
-    msg: 'Error: restore_backup_encryption_passphrase is not defined. It usually means you haven't set RESTORE_BACKUP_ENCRYPTION_PASSPHRASE in your staging environments secrets'
-  when: periodic_restore_from_backup and restore_backup_encryption_passphrase is not defined
+    msg: 'Error: backup_encryption_passphrase is not defined. It usually means you have not set backup_encryption_passphrase in your staging environments secrets'
+  when: periodic_restore_from_backup and backup_encryption_passphrase is not defined
 
 - name: 'Setup crontab to download a backup periodically the opencrvs data'
   cron:
     user: '{{ crontab_user }}'
     name: 'download opencrvs backup'
     minute: '30'
     hour: '0'
-    job: 'cd / && bash /opt/opencrvs/infrastructure/backups/download.sh --passphrase={{ restore_backup_encryption_passphrase }} --ssh_user={{ external_backup_server_user }} --ssh_host={{ external_backup_server_ip }} --ssh_port={{ external_backup_server_ssh_port }} --remote_dir={{ external_backup_server_remote_directory }} >> /var/log/opencrvs-restore.log 2>&1'
-    state: "{{ 'present' if (external_backup_server_ip is defined and restore_backup_encryption_passphrase and periodic_restore_from_backup) else 'absent' }}"
+    job: 'cd / && bash /opt/opencrvs/infrastructure/backups/download.sh --passphrase={{ backup_encryption_passphrase }} --ssh_user={{ external_backup_server_user }} --ssh_host={{ external_backup_server_ip }} --ssh_port={{ external_backup_server_ssh_port }} --remote_dir={{ external_backup_server_remote_directory }} >> /var/log/opencrvs-restore.log 2>&1'
+    state: "{{ 'present' if (external_backup_server_ip is defined and backup_encryption_passphrase and periodic_restore_from_backup) else 'absent' }}"
 
 - name: 'Setup crontab to restore the opencrvs data'
   cron:
@@ -37,4 +37,4 @@
     minute: '0'
     hour: '1'
     job: 'cd / && bash /opt/opencrvs/infrastructure/backups/restore.sh --replicas=1 >> /var/log/opencrvs-restore.log 2>&1'
-    state: "{{ 'present' if (external_backup_server_ip is defined and restore_backup_encryption_passphrase and periodic_restore_from_backup) else 'absent' }}"
+    state: "{{ 'present' if (external_backup_server_ip is defined and backup_encryption_passphrase and periodic_restore_from_backup) else 'absent' }}"

infrastructure/server-setup/tasks/updates.yml

@@ -25,6 +25,9 @@
     content: |
       Unattended-Upgrade::Package-Blacklist {};
       Unattended-Upgrade::DevRelease "auto";
+      Unattended-Upgrade::Remove-Unused-Kernel-Packages "true";
+      Unattended-Upgrade::Remove-New-Unused-Dependencies "true";
+      Unattended-Upgrade::Remove-Unused-Dependencies "true";
       Unattended-Upgrade::Allowed-Origins {
           "${distro_id}:${distro_codename}-security";
       };

infrastructure/setup-deploy-config.sh

@@ -26,6 +26,19 @@ done
 KIBANA_ENCRYPTION_KEY=`uuidgen`
 sed -i "s/{{KIBANA_ENCRYPTION_KEY}}/$KIBANA_ENCRYPTION_KEY/g" /opt/opencrvs/infrastructure/monitoring/kibana/kibana.yml
 
+# Move metabase file
+mv /opt/opencrvs/infrastructure/metabase.init.db.sql /data/metabase/metabase.init.db.sql
+
+# Replace environment variables from all alert definition files
+for file in /opt/opencrvs/infrastructure/monitoring/elastalert/rules/*.yaml; do
+    sed -i -e "s%{{HOST}}%$1%" $file
+    sed -i -e "s%{{SMTP_HOST}}%$SMTP_HOST%" $file
+    sed -i -e "s%{{SMTP_PORT}}%$SMTP_PORT%" $file
+    sed -i -e "s%{{ALERT_EMAIL}}%$ALERT_EMAIL%" $file
+    sed -i -e "s%{{SENDER_EMAIL_ADDRESS}}%$SENDER_EMAIL_ADDRESS%" $file
+    sed -i -e "s%{{DOMAIN}}%$DOMAIN%" $file
+done
+
 sed -i -e "s%{{MINIO_ROOT_USER}}%$MINIO_ROOT_USER%" /opt/opencrvs/infrastructure/mc-config/config.json
 sed -i -e "s%{{MINIO_ROOT_PASSWORD}}%$MINIO_ROOT_PASSWORD%" /opt/opencrvs/infrastructure/mc-config/config.json
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opencrvs/countryconfig",
-  "version": "1.4.1",
+  "version": "1.4.0",
   "description": "OpenCRVS country configuration for reference data",
   "license": "MPL-2.0",
   "husky": {
@@ -88,7 +88,6 @@
     "cross-env": "^6.0.3",
     "csv2json": "^1.4.2",
     "date-fns": "^2.28.0",
-    "dotenv": "^6.1.0",
     "esbuild": "^0.18.9",
     "google-libphonenumber": "^3.2.32",
     "handlebars": "^4.7.7",