Merging v1.1.0 into master (#291)

Merge commit from `stable` with integration test support, documentation improvements, and a more efficient Docker image pull policy. Co-authored-by: willgraf <[email protected]> Co-authored-by: MekWarrior <[email protected]>
vanvalenlab · Mar 10, 2020 · 059281a · 059281a
1 parent 827ac74
commit 059281a
Show file tree

Hide file tree

Showing 36 changed files with 297 additions and 93 deletions.
diff --git a/.github/PULL_REQUEST.md b/.github/PULL_REQUEST.md
@@ -3,3 +3,7 @@
 
 ## Why
 * Provide the justifications for the changes.
+
+## Integration Tests
+* Run integration testing by including `[build-integration-tests]` in the commit message. (_Not required for PRs to `master`_).
+* Integration-tested commit hash: `#######`
diff --git a/.travis.yml b/.travis.yml
@@ -4,14 +4,118 @@ dist: xenial
 git:
   depth: false
 
-language: bash
+language: shell
+
+stages:
+  - unit_tests
+  - integration_tests
+  - deploy
 
 jobs:
+  allow_failures:
+    - script: make test/integration/gke/deploy/elk
+  fast_finish: true
   include:
-    - stage: test
-      script:
-      - echo "No tests to run!"
+    - stage: integration_tests
+      name: "Integration test - no ELK"
+      if: ( (type = pull_request AND branch = master) OR (commit_message =~ /\[build-integration-tests\]/) )
+      env:
+        - CHARTS_PATH=/home/travis/build/vanvalenlab/kiosk/conf/charts
+        - CLOUD_PROVIDER=gke 
+        - CLOUDSDK_BUCKET=deepcell-output-benchmarking
+        - CLOUDSDK_COMPUTE_REGION=us-west1
+        - CLOUDSDK_CONFIG=/home/travis/build/vanvalenlab/kiosk/.config/gcloud/
+        - CLOUDSDK_CORE_PROJECT=deepcell-209717
+        - CLOUDSDK_CORE_VERBOSITY=debug
+        - CONF_PATH_PREFIX=/home/travis/build/vanvalenlab/kiosk
+        - CONSUMER_MACHINE_TYPE=n1-highmem-2
+        - GCP_PREDICTION_GPU_TYPE=nvidia-tesla-t4
+        - GCP_TRAINING_GPU_TYPE=nvidia-tesla-v100
+        - GKE_MACHINE_TYPE=n1-standard-1
+        - GCP_SERVICE_ACCOUNT=continuous-integration-test@deepcell-209717.iam.gserviceaccount.com
+        - GPU_MACHINE_TYPE=n1-highmem-2
+        - GPU_MAX_DIVIDED_BY_FOUR=1
+        - GPU_MAX_DIVIDED_BY_THREE=1
+        - GPU_MAX_DIVIDED_BY_TWO=2
+        - GPU_MAX_TIMES_FIFTY=200 
+        - GPU_MAX_TIMES_FIVE=20
+        - GPU_MAX_TIMES_FOUR=16
+        - GPU_MAX_TIMES_FOURTY=160 
+        - GPU_MAX_TIMES_ONE_HUNDRED=400 
+        - GPU_MAX_TIMES_ONE_HUNDRED_FIFTY=600 
+        - GPU_MAX_TIMES_SEVENTY_FIVE=300 
+        - GPU_MAX_TIMES_TEN=40
+        - GPU_MAX_TIMES_THIRTY=120 
+        - GPU_MAX_TIMES_THREE=12
+        - GPU_MAX_TIMES_TWENTY=80
+        - GPU_MAX_TIMES_TWO=8
+        - GPU_MAX_TIMES_TWO_HUNDRED=800 
+        - GPU_NODE_MIN_SIZE=0
+        - GPU_NODE_MAX_SIZE=4
+        - GPU_PER_NODE=1
+        - KUBERNETES_VERSION=latest
+        - NODE_MIN_SIZE=1
+        - NODE_MAX_SIZE=60
+        - REGION_ZONES_WITH_GPUS=us-west1-a,us-west1-b
+      before_install:
+        - openssl aes-256-cbc -K $encrypted_84460f9bb44c_key -iv $encrypted_84460f9bb44c_iv -in .travis/gcloud_key_base64.enc -out gcloud_key_base64 -d
+        - cat gcloud_key_base64 | base64 --decode > gcloud_key.json
+        - export PATH=/home/travis/build/vanvalenlab/kiosk:$PATH
+      install: ./.travis/install_script.sh
+      script: make test/integration/gke/deploy
+    - stage: integration_tests
+      name: "Integration test - with ELK"
+      if: ( (type = pull_request AND branch = master) OR (commit_message =~ /\[build-integration-tests\]/) ) AND (commit_message =~ /\[test-elk\]/)
+      env:
+        - CHARTS_PATH=/home/travis/build/vanvalenlab/kiosk/conf/charts
+        - CLOUD_PROVIDER=gke 
+        - CLOUDSDK_BUCKET=deepcell-output-benchmarking
+        - CLOUDSDK_COMPUTE_REGION=us-west1
+        - CLOUDSDK_CONFIG=/home/travis/build/vanvalenlab/kiosk/.config/gcloud/
+        - CLOUDSDK_CORE_PROJECT=deepcell-209717
+        - CLOUDSDK_CORE_VERBOSITY=debug
+        - CONF_PATH_PREFIX=/home/travis/build/vanvalenlab/kiosk
+        - CONSUMER_MACHINE_TYPE=n1-highmem-2
+        - GCP_PREDICTION_GPU_TYPE=nvidia-tesla-t4
+        - GCP_TRAINING_GPU_TYPE=nvidia-tesla-v100
+        - GKE_MACHINE_TYPE=n1-standard-1
+        - GCP_SERVICE_ACCOUNT=continuous-integration-test@deepcell-209717.iam.gserviceaccount.com
+        - GPU_MACHINE_TYPE=n1-highmem-2
+        - GPU_MAX_DIVIDED_BY_FOUR=1
+        - GPU_MAX_DIVIDED_BY_THREE=1
+        - GPU_MAX_DIVIDED_BY_TWO=2
+        - GPU_MAX_TIMES_FIFTY=200 
+        - GPU_MAX_TIMES_FIVE=20
+        - GPU_MAX_TIMES_FOUR=16
+        - GPU_MAX_TIMES_FOURTY=160 
+        - GPU_MAX_TIMES_ONE_HUNDRED=400 
+        - GPU_MAX_TIMES_ONE_HUNDRED_FIFTY=600 
+        - GPU_MAX_TIMES_SEVENTY_FIVE=300 
+        - GPU_MAX_TIMES_TEN=40
+        - GPU_MAX_TIMES_THIRTY=120 
+        - GPU_MAX_TIMES_THREE=12
+        - GPU_MAX_TIMES_TWENTY=80
+        - GPU_MAX_TIMES_TWO=8
+        - GPU_MAX_TIMES_TWO_HUNDRED=800 
+        - GPU_NODE_MIN_SIZE=0
+        - GPU_NODE_MAX_SIZE=4
+        - GPU_PER_NODE=1
+        - KUBERNETES_VERSION=latest
+        - NODE_MIN_SIZE=1
+        - NODE_MAX_SIZE=60
+        - REGION_ZONES_WITH_GPUS=us-west1-a,us-west1-b
+      before_install:
+        - openssl aes-256-cbc -K $encrypted_84460f9bb44c_key -iv $encrypted_84460f9bb44c_iv -in .travis/gcloud_key_base64.enc -out gcloud_key_base64 -d
+        - cat gcloud_key_base64 | base64 --decode > gcloud_key.json
+        - export PATH=/home/travis/build/vanvalenlab/kiosk:/home/travis/build/vanvalenlab/kiosk/linux-amd64:$PATH
+      install: ./.travis/install_script.sh
+      script: make test/integration/gke/deploy/elk
+    - stage: unit_tests
+      name: "Unit Tests"
+      if: type = push
+      script: make test/unit
     - stage: deploy
+      name: "Docker Deployment"
       if: tag IS present
       script:
       - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin

diff --git a/.travis/gcloud_key_base64.enc b/.travis/gcloud_key_base64.enc
diff --git a/.travis/install_script.sh b/.travis/install_script.sh
@@ -0,0 +1,33 @@
+#/bin/bash
+
+# Install helmfile
+wget https://github.com/roboll/helmfile/releases/download/v0.82.0/helmfile_linux_amd64
+chmod 764 $CONF_PATH_PREFIX/helmfile_linux_amd64
+mv helmfile_linux_amd64 helmfile
+# Install gomplate
+wget https://github.com/hairyhenderson/gomplate/releases/download/v3.1.0/gomplate_linux-amd64-slim
+chmod 764 $CONF_PATH_PREFIX/gomplate_linux-amd64-slim
+mv gomplate_linux-amd64-slim gomplate
+# Install kubectl
+sudo apt-get update && sudo sudo apt-get install -y apt-transport-https
+curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
+echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee -a /etc/apt/sources.list.d/kubernetes.list
+sudo apt-get update
+sudo apt-get install -y kubectl
+# Install kubens
+wget https://raw.githubusercontent.com/ahmetb/kubectx/master/kubens
+chmod 764 $CONF_PATH_PREFIX/kubens
+# Install helm
+wget https://get.helm.sh/helm-v2.16.3-linux-amd64.tar.gz
+tar -xzvf helm-v2.16.3-linux-amd64.tar.gz
+chmod 764 $CONF_PATH_PREFIX/linux-amd64/helm
+mv $CONF_PATH_PREFIX/linux-amd64/helm $CONF_PATH_PREFIX/
+# Install gcloud
+sudo apt-get install apt-transport-https ca-certificates gnupg
+echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
+curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
+sudo apt-get update && sudo apt-get install google-cloud-sdk
+## prep directory structure for gcloud	
+mkdir -p $CONF_PATH_PREFIX/.config/gcloud
+sudo chmod 777 $CONF_PATH_PREFIX/.config
+sudo chmod 777 $CONF_PATH_PREFIX/.config/gcloud
diff --git a/Dockerfile b/Dockerfile
@@ -53,7 +53,10 @@ ENV CONSUMER_MACHINE_TYPE="n1-highmem-2"
 
 # Deployment config
 ENV CLOUD_PROVIDER=""
-ENV ELK_DEPLOYMENT_TOGGLE="OFF"
+ENV ELK_DEPLOYMENT_TOGGLE=""
+
+# Kiosk filesystem config
+ENV CONF_PATH_PREFIX=""
 
 # Filesystem entry for tfstate
 RUN s3 fstab '${KOPS_STATE_STORE}' '/' '/s3'

diff --git a/Makefile b/Makefile
@@ -7,6 +7,7 @@ export DOCKER_BUILD_FLAGS =
 export README_DEPS ?= docs/targets.md
 export INSTALL_PATH ?= /usr/local/bin
 
+
 -include $(shell curl -sSL -o .build-harness "https://git.io/build-harness"; echo .build-harness)
 
 ## Initialize build-harness, install deps, build docker container, install wrapper script and run shell
@@ -32,3 +33,28 @@ install:
 ## Start the geodesic shell by calling wrapper script
 run:
 	$(CLUSTER)
+
+## Target for testing cluster deployment
+test/integration/gke/deploy: export CLOUDSDK_CONTAINER_CLUSTER = deepcell-test-$(shell bash -c 'echo $$RANDOM')
+test/integration/gke/deploy:
+	# check environment variables
+	printenv
+	# check that necessary binaries are installed
+	helmfile --version
+	gomplate --version
+	kubectl version --client
+	helm version -c
+	gcloud version
+	# execute make targets 
+	# make init
+	cd ./conf && make test/create
+	cd ./conf && make test/destroy
+	# celebrate
+	echo "TESTED"
+
+test/integration/gke/deploy/elk: export ELK_DEPLOYMENT_TOGGLE = ON
+test/integration/gke/deploy/elk: \
+	test/integration/gke/deploy
+
+test/unit:
+	echo "No unit tests at this time."
diff --git a/README.md b/README.md
@@ -2,7 +2,7 @@
 
 [![Build Status](https://travis-ci.com/vanvalenlab/kiosk.svg?branch=master)](https://travis-ci.com/vanvalenlab/kiosk)
 [![Read the Docs](https://img.shields.io/readthedocs/kiosk?logo=Read%20the%20Docs)](https://deepcell-kiosk.readthedocs.io/en/master)
-[![Apache 2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
+[![Apache 2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/vanvalenlab/kiosk/blob/master/LICENSE)
 
 The DeepCell Kiosk is the entry point for users to spin up an end-to-end DeepCell environment in the cloud using [Kubernetes](https://kubernetes.io/). It is designed to allow researchers to easily deploy and scale a deep learning platform for biological image analysis. Once launched, users can drag-and-drop images to be processed in parallel using publicly available, or custom-built, TensorFlow models. To train custom models, please refer to [DeepCell-TF](https://github.com/vanvalenlab/deepcell-tf), which was designed to facilitate model development and export these models for use with the DeepCell Kiosk.
 
@@ -46,7 +46,7 @@ Tracked Image
 Start a terminal shell and install the DeepCell Kiosk wrapper script:
 
 ```bash
-docker run -e DOCKER_TAG=1.0.0 vanvalenlab/kiosk:1.0.0 | sudo bash
+docker run -e DOCKER_TAG=1.1.0 vanvalenlab/kiosk:1.1.0 | sudo bash
 ```
 
 To start the kiosk, just run `kiosk` from the terminal shell.
@@ -84,7 +84,7 @@ A [FAQ](http://www.deepcell.org.faq) page is also available.
 
 ## License
 
-This software is license under a modified [APACHE2](https://opensource.org/licenses/Apache-2.0). See [LICENSE](https://github.com/vanvalenlab/kiosk/blob/master/LICENSE) for full  details.
+This software is license under a modified [APACHE2](https://opensource.org/licenses/Apache-2.0). See [LICENSE](https://github.com/vanvalenlab/kiosk/blob/master/LICENSE) for full details.
 
 ## Trademarks
 

diff --git a/conf/Makefile b/conf/Makefile
@@ -1,6 +1,10 @@
 export CLOUD_PROVIDER ?= gke
+export TRANSLATE_COLON_NOTATION=false
 
-include /build-harness/templates/Makefile.build-harness
+# The if statement is meant to keep this line from executing during testing.
+ifeq ($(CONF_PATH_PREFIX), )
+	include /build-harness/templates/Makefile.build-harness
+endif
 
 include tasks/Makefile.*
 
@@ -18,3 +22,18 @@ destroy: \
   helmfile/destroy/all \
   $(CLOUD_PROVIDER)/destroy/all
 	@echo "Cluster destroyed"
+
+## Create test cluster
+test/create: \
+  $(CLOUD_PROVIDER)/test/create/all \
+  kubectl/create/prometheus-redis-exporter-script \
+  helmfile/create/all \
+  kubectl/display/ip \
+  kubectl/implement/autoscaling
+	@echo "Cluster created"
+
+## Destroy cluster
+test/destroy: \
+  helmfile/destroy/all \
+  $(CLOUD_PROVIDER)/test/destroy/all
+	@echo "Cluster destroyed"
diff --git a/conf/charts/autoscaler/values.yaml b/conf/charts/autoscaler/values.yaml
@@ -5,9 +5,9 @@
 replicas: 1
 
 image:
-  repository: "vanvalenlab/kiosk-autoscaler"
-  tag: "latest"
-  pullPolicy: "Always"
+  repository: vanvalenlab/kiosk-autoscaler
+  tag: latest
+  pullPolicy: IfNotPresent
 
 resources: {}
 

diff --git a/conf/charts/benchmarking/values.yaml b/conf/charts/benchmarking/values.yaml
@@ -5,9 +5,9 @@
 replicas: 1
 
 image:
-  repository: "vanvalenlab/kiosk-benchmarking"
-  tag: "latest"
-  pullPolicy: "Always"
+  repository: vanvalenlab/kiosk-benchmarking
+  tag: latest
+  pullPolicy: IfNotPresent
 
 resources: {}
 

diff --git a/conf/charts/bucket-monitor/values.yaml b/conf/charts/bucket-monitor/values.yaml
@@ -5,9 +5,9 @@
 replicas: 1
 
 image:
-  repository: "vanvalenlab/kiosk-bucket-monitor"
-  tag: "latest"
-  pullPolicy: "Always"
+  repository: vanvalenlab/kiosk-bucket-monitor
+  tag: latest
+  pullPolicy: IfNotPresent
 
 resources: {}
 

diff --git a/conf/charts/data-processing/values.yaml b/conf/charts/data-processing/values.yaml
@@ -5,9 +5,9 @@
 replicas: 1
 
 image:
-  repository: "vanvalenlab/kiosk-data-processing"
-  tag: "latest"
-  pullPolicy: "Always"
+  repository: vanvalenlab/kiosk-data-processing
+  tag: latest
+  pullPolicy: IfNotPresent
 
 resources: {}
 

diff --git a/conf/charts/frontend/values.yaml b/conf/charts/frontend/values.yaml
@@ -5,9 +5,9 @@
 replicas: 1
 
 image:
-  repository: "vanvalenlab/kiosk-frontend"
-  tag: "latest"
-  pullPolicy: "Always"
+  repository: vanvalenlab/kiosk-frontend
+  tag: latest
+  pullPolicy: IfNotPresent
 
 ingress:
   enabled: true

diff --git a/conf/charts/redis-consumer/values.yaml b/conf/charts/redis-consumer/values.yaml
@@ -5,9 +5,9 @@
 replicas: 1
 
 image:
-  repository: "vanvalenlab/kiosk-redis-consumer"
-  tag: "latest"
-  pullPolicy: "Always"
+  repository: vanvalenlab/kiosk-redis-consumer
+  tag: latest
+  pullPolicy: IfNotPresent
 
 resources: {}
 

diff --git a/conf/charts/redis-janitor/values.yaml b/conf/charts/redis-janitor/values.yaml
@@ -5,9 +5,9 @@
 replicas: 1
 
 image:
-  repository: "vanvalenlab/kiosk-redis-janitor"
-  tag: "latest"
-  pullPolicy: "Always"
+  repository: vanvalenlab/kiosk-redis-janitor
+  tag: latest
+  pullPolicy: IfNotPresent
 
 resources: {}
 

diff --git a/conf/charts/tensorboard/values.yaml b/conf/charts/tensorboard/values.yaml
@@ -5,9 +5,9 @@
 replicas: 1
 
 image:
-  repository: "vanvalenlab/kiosk-tensorboard"
-  tag: "latest"
-  pullPolicy: "Always"
+  repository: vanvalenlab/kiosk-tensorboard
+  tag: latest
+  pullPolicy: IfNotPresent
 
 resources: {}
 

diff --git a/conf/charts/tf-serving/values.yaml b/conf/charts/tf-serving/values.yaml
@@ -5,12 +5,12 @@
 replicas: 0
 
 image:
-  repository: "vanvalenlab/kiosk-tf-serving"
-  tag: "latest"
-  pullPolicy: "Always"
+  repository: vanvalenlab/kiosk-tf-serving
+  tag: latest
+  pullPolicy: IfNotPresent
 
 service:
-  type: "ClusterIP"
+  type: ClusterIP
 
   httpIngressEnabled: true
   internalHttpPort: 8501