Restore on deploy

Makefile

@@ -11,9 +11,13 @@ help:
 	@echo "    make sort             run the linter."
 	@echo "    make test             run the tests."
 
-.PHONY: deploy
-deploy:
-	(cd deployment; ansible-playbook setup-server.yml -i hosts.dev --vault-password-file ~/.vault.txt)
+.PHONY: deploy-prod
+deploy-prod:
+	(cd deployment; ansible-playbook setup-prod.yml -i hosts.dev --vault-password-file ~/.vault.txt)
+
+.PHONY: deploy-dev
+deploy-dev:
+	(cd deployment; ansible-playbook setup-dev.yml -i hosts.dev --vault-password-file ~/.vault.txt)
 
 .PHONY: fix
 fix:

README.md

@@ -34,10 +34,15 @@ To import new CSVs via the web form, run the Queue worker:
 
 ## Deployment
 
-    1. update hosts.dev (and use keys natch)
-    1. set your branch in deployment/group_vars/all
-    1. create .vault.txt and put the vault password in there
-    1. `make deploy`
+    1. Update hosts.dev.
+    2. Set your branch in deployment/group_vars/all
+    3. `make deploy-prod` deploys prod, this means setting up back ups and restoring with the latest snapshot.
+
+    It must be run after 19:00 every day when the snap shot is taken. Otherwise the deployment will fail.
+
+    4. `make deploy-dev` deploys , this means no back ups and restoring from the first snap shot it can find from the last 4 days.
+    5. The ckan instance has a hard coded link to the NCDR box in the nginx config. Change this to point to your newly deployed instance.
+    6. Make sure that the NCDR instance is accessible, that the database is populated and that one can log in.
 
 To view the encrypted variables:
 

deployment/group_vars/all

@@ -45,5 +45,6 @@ AWS_SECRET_ACCESS_KEY: !vault |
           64653033316238373431396437616639346161393334326138383864613563303461646264623335
           3631663963393762306432623061336436623038376234666266
 AWS_BUCKET_NAME: "ncdr-v0.5"
+DEV_AWS_BUCKET_NAME: "ncdr-dev"
 
 BACKUPS_DIR: "/home/ubuntu/backups"

deployment/roles/aws/tasks/main.yml

@@ -0,0 +1,14 @@
+- name: Creates AWS dir
+  file:
+    path: /home/ubuntu/.aws
+    state: directory
+
+- name: create AWS credentials
+  template:
+    src: templates/aws_credentials.ini
+    dest: /home/ubuntu/.aws/credentials
+
+- name: create AWS config
+  template:
+    src: templates/aws_config.ini
+    dest: /home/ubuntu/.aws/config

deployment/roles/backup/tasks/main.yml

@@ -11,25 +11,11 @@
     path: "{{ BACKUPS_DIR }}"
     state: directory
 
-- name: Creates AWS dir
-  file:
-    path: /home/ubuntu/.aws
-    state: directory
-
-- name: create AWS credentials
-  template:
-    src: templates/aws_credentials.ini
-    dest: /home/ubuntu/.aws/credentials
-
-- name: create AWS config
-  template:
-    src: templates/aws_config.ini
-    dest: /home/ubuntu/.aws/config
 
 - name: fire the backup cron
   cron:
     name: "backup"
     minute: "0"
-    hour: "1"
+    hour: "19"
     user: "ubuntu"
     job: "{{ PROJECT_PATH }}/scripts/backup.sh"

deployment/roles/ncdr/tasks/main.yml

@@ -37,7 +37,6 @@
     path: "{{ LOG_DIR }}"
     state: directory
 
-
 - name: Creates config dir
   file:
     path: "{{ CONFIG_DIR }}"

deployment/roles/restore/tasks/main.yml

@@ -0,0 +1,34 @@
+- name: drop ncdr database
+  become: true
+  become_user: postgres
+  postgresql_db:
+    name: "{{ DB_NAME }}"
+    encoding: 'UTF-8'
+    owner: "{{ DB_USER }}"
+    template: template0
+    state: absent
+
+- name: create ncdr database
+  become: true
+  become_user: postgres
+  postgresql_db:
+    name: "{{ DB_NAME }}"
+    encoding: 'UTF-8'
+    owner: "{{ DB_USER }}"
+    template: template0
+
+
+- name: Restore the server from a bucket
+  command: "{{ VIRTUALENV_PATH }}/bin/python scripts/restore.py {{ DB_NAME }} {{ DB_USER }} {{ AWS_BUCKET_NAME }} {% if DEV %}{{ DEV }}{% endif %}"
+  args:
+      chdir: "{{ PROJECT_PATH }}"
+
+- name: Run Django database migrations
+  args:
+    chdir: "{{ PROJECT_PATH}}"
+  command: "envdir env_vars {{ VIRTUALENV_PATH }}/bin/python manage.py migrate"
+
+- name: Restart circus
+  become: yes
+  command: "circusctl restart"
+  ignore_errors: True

deployment/setup-dev.yml

@@ -0,0 +1,4 @@
+---
+- import_playbook: setup-server.yml
+  vars:
+    DEV: 1

deployment/setup-prod.yml

@@ -0,0 +1,4 @@
+---
+- import_playbook: setup-server.yml
+  vars:
+    DEV: 0

deployment/setup-server.yml

@@ -13,6 +13,12 @@
   roles:
     - django
 
+- name: Setup aws
+  user: ubuntu
+  hosts: webserver
+  roles:
+    - aws
+
 - name: Make the server capable of running a standard Django deployment
   become: yes
   user: ubuntu
@@ -30,4 +36,11 @@
   user: ubuntu
   hosts: webserver
   roles:
-    - backup
+    - restore
+
+- name: Set the project back ups
+  user: ubuntu
+  hosts: webserver
+  roles:
+    - role: backup
+      when: not DEV

deployment/templates/nginx_site.conf.jinja2

@@ -11,13 +11,17 @@ server {
     location / {
         proxy_pass http://127.0.0.1:4567;
 
-        proxy_redirect http://127.0.0.1:4567/ncdr/ http://$host/ncdr/;
-
         proxy_set_header Host $host;
-        proxy_set_header SCRIPT_NAME /ncdr;
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 
+        {% if not DEV %}
+
+        proxy_redirect http://127.0.0.1:4567/ncdr/ http://$host/ncdr/;
+        proxy_set_header SCRIPT_NAME /ncdr;
+        {% endif %}
+
+
         client_max_body_size 15m;
     }
 

deployment/templates/settings/AWS_STORAGE_BUCKET_NAME.jinja2

@@ -1 +1,5 @@
+{% if DEV %}
+{{ DEV_AWS_BUCKET_NAME }}
+{% else %}
 {{ AWS_BUCKET_NAME }}
+{% endif %}

deployment/templates/settings/STATIC_URL.jinja2

@@ -1 +1,5 @@
+{% if DEV %}
+/static/
+{% else %}
 /ncdr/static/
+{% endif %}

scripts/restore.py

@@ -0,0 +1,67 @@
+"""
+Restores the database from an S3 backup.
+If 3 arguments are passed in the backup it looks for has todays
+date.
+
+If 4 arguments are passed in it runs in dev mode and picks up
+the most recent backup from the last 4 days.
+"""
+import datetime
+import os
+import subprocess
+import sys
+
+import boto3
+import botocore
+
+
+def get_backup_name(some_datetime, db_name):
+    dt_str = some_datetime.strftime("%Y-%m-%d")
+    return f"{dt_str}-{db_name}.dump"
+
+
+def get_backup(bucket_name, db_name, look_back=False):
+    s3 = boto3.resource("s3")
+    if look_back:
+        date_times = [datetime.datetime.now() - datetime.timedelta(i) for i in range(5)]
+    else:
+        date_times = [datetime.datetime.now()]
+    file_names = [get_backup_name(dt, db_name) for dt in date_times]
+    success = False
+    for file_name in file_names:
+        try:
+            temp_file = f"/tmp/{file_name}"
+            s3.Bucket(bucket_name).download_file(f"backups/{file_name}", temp_file)
+        except botocore.exceptions.ClientError as e:
+            if e.response["Error"]["Code"] == "404":
+                print("unable to find {}".format(file_name))
+        else:
+            success = True
+            break
+    if not success:
+        raise ValueError("Unable to restore the database!")
+    return temp_file
+
+
+def load_file(db_name, db_user, file_name):
+    command = "psql -d {} -U {} -f {}".format(db_name, db_user, file_name)
+    subprocess.check_call(command, shell=True)
+
+
+def main(db_name, db_user, bucket_name, dev=False):
+    file_name = get_backup(bucket_name, db_name, look_back=dev)
+    load_file(db_name, db_user, file_name)
+    os.remove(file_name)
+
+
+if __name__ == "__main__":
+    try:
+        if len(sys.argv) == 4:
+            _, db_name, db_user, bucket_name = sys.argv
+            dev = False
+        else:
+            _, db_name, db_user, bucket_name, dev = sys.argv
+        main(db_name, db_user, bucket_name, dev)
+    except Exception as e:
+        print("errored with {}".format(str(e)))
+        raise