From 13f5edbf21005269d3f76a8743880dfafcc2858e Mon Sep 17 00:00:00 2001 From: willgraf <7930703+willgraf@users.noreply.github.com> Date: Wed, 16 Dec 2020 09:18:08 -0800 Subject: [PATCH] Merge stable into master for release 1.4.0 (#410) * Support TLS traffic with cert-manager. (#357) * Fix frontend ingress issue when no hosts are provided. (#381) * Template frontend ingress annotations using `CERTIFICATE_MANAGER_ENABLED` (#383) * Create tf-serving configuration files using an initContainer. (#382) * Fix whitespace issue in tasks/Makefile.kubectl (#386) * Bump openvpn to 4.2.3 (#385) * Upgrade certificate manager to version 1.0.3 (#384) * Add screenshot of successfully created cluster to docs. (#388) * Set up an AlertManager with slack receiver support (#317) * Install procps to give access to sysctl. (#390) * Migrate CI/CD from TravisCI to GitHub Actions (#394) * Change the redis helm chart repo to bitnami (#393) * Upgrade tf-serving chart to 0.3.0 for application version 0.4.0 (#392) * Move the frontend HPA definition into the helm chart. (#395) * Move the tf-serving HPA into the helm chart. (#396) * Move redis-consumer HPA into the helm chart. (#397) * Remove deprecated and unused charts (#398) * Migrate stable helm chart repo to archived URL. (#399) * Destroy the secret and remove the key from the DNS solver SA in a new task: `gke/destroy/certificate-manager-secret` (fixes #391). * Use GCP_SERVICE_ACCOUNT for DNS resolution (#401) * Clean up docs and test them with new GitHub Action workflow (#402) * Add code-formatted filename to list of files to change (#403) * Update ELK stack helmfiles (#380) * Move the prometheus-redis-exporter script to a chart using incubator/raw. (#405) * Use `kubectl del pvc` instead of deleting all pds with the cluster name. (#406) * Update helmfile defaults for faster helm deployments. (#404) * Skip gke/destroy/node-pools during cluster teardown. (#407) * Update docs to reflect the pending 1.4.0 release. (#408) * Bump redis-consumer version to 0.8.3 (#409) * Run integration tests on all PRs to master OR if they have the commit message. (#411) * Remove helm defaults for ELK helmfiles (#413) Co-authored-by: Morgan Schwartz --- .github/workflows/docker-image.yaml | 48 ++ .github/workflows/integration-tests.yaml | 144 ++++++ .github/workflows/test-docs.yaml | 30 ++ .github/workflows/unit-tests.yaml | 39 ++ .gitignore | 5 +- .travis.yml | 88 ---- .travis/gcloud_key_base64.enc | Bin 3184 -> 0 bytes Dockerfile | 2 + README.md | 4 +- conf/ELK_helmfiles/0500.elasticsearch.yaml | 217 ++------- conf/ELK_helmfiles/0510.kibana.yaml | 109 ++--- conf/ELK_helmfiles/0520.logstash.yaml | 129 +----- conf/ELK_helmfiles/0530.filebeat.yaml | 90 +--- conf/Makefile | 17 +- conf/addons/hpa.yaml | 107 ----- conf/addons/redis-exporter-script.yaml | 65 --- conf/charts/data-processing/Chart.yaml | 18 - conf/charts/data-processing/LICENSE | 0 conf/charts/data-processing/README.md | 3 - conf/charts/data-processing/requirements.lock | 0 conf/charts/data-processing/requirements.yaml | 0 .../data-processing/templates/NOTES.txt | 0 .../data-processing/templates/_helpers.tpl | 33 -- .../data-processing/templates/deployment.yaml | 68 --- .../data-processing/templates/service.yaml | 33 -- conf/charts/data-processing/values.yaml | 39 -- conf/charts/frontend/Chart.yaml | 4 +- conf/charts/frontend/templates/hpa.yaml | 23 + conf/charts/frontend/templates/ingress.yaml | 28 +- conf/charts/frontend/templates/service.yaml | 5 +- conf/charts/frontend/values.yaml | 24 +- conf/charts/redis-consumer/Chart.yaml | 5 +- conf/charts/redis-consumer/requirements.lock | 3 - conf/charts/redis-consumer/requirements.yaml | 1 - conf/charts/redis-consumer/templates/hpa.yaml | 23 + conf/charts/redis-consumer/values.yaml | 16 +- conf/charts/tensorboard/Chart.yaml | 5 - conf/charts/tensorboard/LICENSE | 0 conf/charts/tensorboard/NOTES.txt | 19 - conf/charts/tensorboard/README.md | 0 .../charts/tensorboard/templates/_helpers.tpl | 40 -- .../tensorboard/templates/deployment.yaml | 46 -- .../charts/tensorboard/templates/ingress.yaml | 27 -- .../charts/tensorboard/templates/secrets.yaml | 11 - .../charts/tensorboard/templates/service.yaml | 27 -- conf/charts/tensorboard/values.yaml | 42 -- conf/charts/tf-serving/Chart.yaml | 5 +- .../tf-serving/templates/deployment.yaml | 32 ++ conf/charts/tf-serving/templates/hpa.yaml | 23 + conf/charts/tf-serving/values.yaml | 32 +- conf/charts/training/Chart.yaml | 20 - conf/charts/training/LICENSE | 0 conf/charts/training/README.md | 0 conf/charts/training/requirements.yaml | 0 conf/charts/training/templates/NOTES.txt | 0 conf/charts/training/templates/_helpers.tpl | 40 -- conf/charts/training/templates/job.yaml | 63 --- conf/charts/training/templates/secrets.yaml | 11 - conf/charts/training/templates/service.yaml | 27 -- conf/charts/training/values.yaml | 43 -- conf/helmfile.d/0000.ingress.yaml | 5 +- conf/helmfile.d/0010.cert-manager.yaml | 106 +++++ conf/helmfile.d/0100.redis.yaml | 110 ++--- .../0110.prometheus-redis-exporter.yaml | 148 ++++-- conf/helmfile.d/0200.bucket-monitor.yaml | 31 +- conf/helmfile.d/0210.autoscaler.yaml | 11 +- conf/helmfile.d/0220.redis-janitor.yaml | 13 +- .../0230.segmentation-consumer.yaml | 31 +- .../0240.segmentation-zip-consumer.yaml | 31 +- conf/helmfile.d/0250.tracking-consumer.yaml | 31 +- conf/helmfile.d/0300.frontend.yaml | 56 ++- conf/helmfile.d/0310.tf-serving.yaml | 55 ++- conf/helmfile.d/0400.benchmarking.yaml | 11 +- conf/helmfile.d/0600.prometheus-operator.yaml | 434 ++++++++++++++---- conf/helmfile.d/0610.prometheus-adapter.yaml | 15 +- conf/helmfile.d/9999.openvpn.yaml | 16 +- conf/tasks/Makefile.gke | 53 ++- conf/tasks/Makefile.helmfile | 5 +- conf/tasks/Makefile.kubectl | 18 +- docs/images/Kiosk-Complete-Blur.png | Bin 0 -> 92125 bytes docs/rtd-requirements.txt | 2 +- docs/source/CUSTOM-JOB.rst | 72 +-- docs/source/GETTING_STARTED.rst | 8 +- docs/source/TROUBLESHOOTING.rst | 2 +- docs/source/conf.py | 4 +- scripts/deploy-helmfiles.sh | 48 ++ scripts/gke-helmfile-deployment.sh | 17 - scripts/gke-pd-destruction.sh | 15 - 88 files changed, 1545 insertions(+), 1736 deletions(-) create mode 100644 .github/workflows/docker-image.yaml create mode 100644 .github/workflows/integration-tests.yaml create mode 100644 .github/workflows/test-docs.yaml create mode 100644 .github/workflows/unit-tests.yaml delete mode 100644 .travis.yml delete mode 100644 .travis/gcloud_key_base64.enc delete mode 100644 conf/addons/hpa.yaml delete mode 100644 conf/addons/redis-exporter-script.yaml delete mode 100644 conf/charts/data-processing/Chart.yaml delete mode 100644 conf/charts/data-processing/LICENSE delete mode 100644 conf/charts/data-processing/README.md delete mode 100644 conf/charts/data-processing/requirements.lock delete mode 100644 conf/charts/data-processing/requirements.yaml delete mode 100644 conf/charts/data-processing/templates/NOTES.txt delete mode 100644 conf/charts/data-processing/templates/_helpers.tpl delete mode 100644 conf/charts/data-processing/templates/deployment.yaml delete mode 100644 conf/charts/data-processing/templates/service.yaml delete mode 100644 conf/charts/data-processing/values.yaml create mode 100644 conf/charts/frontend/templates/hpa.yaml delete mode 100644 conf/charts/redis-consumer/requirements.lock create mode 100644 conf/charts/redis-consumer/templates/hpa.yaml delete mode 100644 conf/charts/tensorboard/Chart.yaml delete mode 100644 conf/charts/tensorboard/LICENSE delete mode 100644 conf/charts/tensorboard/NOTES.txt delete mode 100644 conf/charts/tensorboard/README.md delete mode 100644 conf/charts/tensorboard/templates/_helpers.tpl delete mode 100644 conf/charts/tensorboard/templates/deployment.yaml delete mode 100644 conf/charts/tensorboard/templates/ingress.yaml delete mode 100644 conf/charts/tensorboard/templates/secrets.yaml delete mode 100644 conf/charts/tensorboard/templates/service.yaml delete mode 100644 conf/charts/tensorboard/values.yaml create mode 100644 conf/charts/tf-serving/templates/hpa.yaml delete mode 100644 conf/charts/training/Chart.yaml delete mode 100644 conf/charts/training/LICENSE delete mode 100644 conf/charts/training/README.md delete mode 100644 conf/charts/training/requirements.yaml delete mode 100644 conf/charts/training/templates/NOTES.txt delete mode 100644 conf/charts/training/templates/_helpers.tpl delete mode 100644 conf/charts/training/templates/job.yaml delete mode 100644 conf/charts/training/templates/secrets.yaml delete mode 100644 conf/charts/training/templates/service.yaml delete mode 100644 conf/charts/training/values.yaml create mode 100644 conf/helmfile.d/0010.cert-manager.yaml create mode 100644 docs/images/Kiosk-Complete-Blur.png create mode 100755 scripts/deploy-helmfiles.sh delete mode 100755 scripts/gke-helmfile-deployment.sh delete mode 100755 scripts/gke-pd-destruction.sh diff --git a/.github/workflows/docker-image.yaml b/.github/workflows/docker-image.yaml new file mode 100644 index 00000000..43aae58b --- /dev/null +++ b/.github/workflows/docker-image.yaml @@ -0,0 +1,48 @@ +name: Build & Push Docker Image + +on: + release: + types: [published] + +jobs: + + docker: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Build and push Image + id: docker_build + uses: docker/build-push-action@v2 + with: + context: . + file: ./Dockerfile + push: true + tags: | + ${{ github.repository }}:latest + ${{ github.repository }}:${{ github.event.release.tag_name }} + + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml new file mode 100644 index 00000000..9282140e --- /dev/null +++ b/.github/workflows/integration-tests.yaml @@ -0,0 +1,144 @@ +name: integration-tests + +on: + pull_request: + branches: master + push: {} + +jobs: + integration-tests: + + runs-on: ubuntu-latest + + if: | + github.event_name == 'pull_request' || + ( + github.event_name == 'push' && + contains(github.event.head_commit.message, '[build-integration-tests]') + ) + + strategy: + matrix: + CLOUD_PROVIDER: + - gke + # - aws + + steps: + - uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - uses: google-github-actions/setup-gcloud@master + with: + version: '290.0.1' + project_id: ${{ secrets.CLOUDSDK_CORE_PROJECT }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + service_account_email: ${{ secrets.GCP_SERVICE_ACCOUNT }} + export_default_credentials: true + + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Build and Run Integration Tests + env: + GCP_SERVICE_ACCOUNT: ${{ secrets.GCP_SERVICE_ACCOUNT }} + CLOUDSDK_CORE_PROJECT: ${{ secrets.CLOUDSDK_CORE_PROJECT }} + CLOUDSDK_BUCKET: ${{ secrets.CLOUDSDK_BUCKET }} + CLOUDSDK_COMPUTE_REGION: us-west1 + REGION_ZONES_WITH_GPUS: us-west1-a,us-west1-b + IMAGE: ${{ github.repository }}:${{ github.sha }} + GCP_SA_KEY_PATH: /tmp/keys/gcloud_key.json + run: | + docker buildx build --load --tag ${{ env.IMAGE }} . + docker run -d -it \ + --volume $(readlink -f ${{ env.GOOGLE_APPLICATION_CREDENTIALS }}):${{ env.GCP_SA_KEY_PATH }}:ro \ + --env CLOUD_PROVIDER=${{ matrix.CLOUD_PROVIDER }} \ + --env GCP_SERVICE_ACCOUNT=${{ env.GCP_SERVICE_ACCOUNT }} \ + --env CLOUDSDK_CORE_PROJECT=${{ env.CLOUDSDK_CORE_PROJECT }} \ + --env CLOUDSDK_BUCKET=${{ env.CLOUDSDK_BUCKET }} \ + --env CLOUDSDK_COMPUTE_REGION=${{ env.CLOUDSDK_COMPUTE_REGION }} \ + --env REGION_ZONES_WITH_GPUS=${{ env.REGION_ZONES_WITH_GPUS }} \ + --env GOOGLE_APPLICATION_CREDENTIALS=${{ env.GCP_SA_KEY_PATH }} \ + --entrypoint=/bin/bash \ + --name kiosk \ + ${{ env.IMAGE }} + docker exec kiosk make test/integration/${{ matrix.CLOUD_PROVIDER }}/deploy + docker kill kiosk && docker rm kiosk + + elk-integration-tests: + + runs-on: ubuntu-latest + + if: | + github.event_name == 'pull_request' || + ( + github.event_name == 'push' && + contains(github.event.head_commit.message, '[build-integration-tests]') && + contains(github.event.head_commit.message, '[test-elk]') + ) + + strategy: + matrix: + CLOUD_PROVIDER: + - gke + # - aws + + steps: + - uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - uses: google-github-actions/setup-gcloud@master + with: + version: '290.0.1' + project_id: ${{ secrets.CLOUDSDK_CORE_PROJECT }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + service_account_email: ${{ secrets.GCP_SERVICE_ACCOUNT }} + export_default_credentials: true + + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Build and Run ELK Integration Tests + env: + GCP_SERVICE_ACCOUNT: ${{ secrets.GCP_SERVICE_ACCOUNT }} + CLOUDSDK_CORE_PROJECT: ${{ secrets.CLOUDSDK_CORE_PROJECT }} + CLOUDSDK_BUCKET: ${{ secrets.CLOUDSDK_BUCKET }} + CLOUDSDK_COMPUTE_REGION: us-west1 + REGION_ZONES_WITH_GPUS: us-west1-a,us-west1-b + IMAGE: ${{ github.repository }}:${{ github.sha }} + GCP_SA_KEY_PATH: /tmp/keys/gcloud_key.json + run: | + docker buildx build --load --tag ${{ env.IMAGE }} . + docker run -d -it \ + --volume $(readlink -f ${{ env.GOOGLE_APPLICATION_CREDENTIALS }}):${{ env.GCP_SA_KEY_PATH }}:ro \ + --env CLOUD_PROVIDER=${{ matrix.CLOUD_PROVIDER }} \ + --env GCP_SERVICE_ACCOUNT=${{ env.GCP_SERVICE_ACCOUNT }} \ + --env CLOUDSDK_CORE_PROJECT=${{ env.CLOUDSDK_CORE_PROJECT }} \ + --env CLOUDSDK_BUCKET=${{ env.CLOUDSDK_BUCKET }} \ + --env CLOUDSDK_COMPUTE_REGION=${{ env.CLOUDSDK_COMPUTE_REGION }} \ + --env REGION_ZONES_WITH_GPUS=${{ env.REGION_ZONES_WITH_GPUS }} \ + --env GOOGLE_APPLICATION_CREDENTIALS=${{ env.GCP_SA_KEY_PATH }} \ + --entrypoint=/bin/bash \ + --name kiosk \ + ${{ env.IMAGE }} + docker exec kiosk make test/integration/${{ matrix.CLOUD_PROVIDER }}/deploy/elk + docker kill kiosk && docker rm kiosk diff --git a/.github/workflows/test-docs.yaml b/.github/workflows/test-docs.yaml new file mode 100644 index 00000000..5fcd89e6 --- /dev/null +++ b/.github/workflows/test-docs.yaml @@ -0,0 +1,30 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Test Docs + +on: + - pull_request + +jobs: + + docs: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install -r docs/rtd-requirements.txt + sudo apt-get install pandoc -y + + - name: Test sphinx-build + run: sphinx-build -W -nT -b dummy ./docs/source build/html diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml new file mode 100644 index 00000000..dfb47291 --- /dev/null +++ b/.github/workflows/unit-tests.yaml @@ -0,0 +1,39 @@ +name: build + +on: [push, pull_request] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Build and Run Unit Tests + env: + CLOUD_PROVIDER: gke + IMAGE: ${{ github.repository }}:${{ github.sha }} + run: | + docker buildx build --load --tag ${{ env.IMAGE }} . + docker run -d -it \ + --env CLOUD_PROVIDER=${{ env.CLOUD_PROVIDER }} \ + --entrypoint=/bin/bash \ + --name kiosk \ + ${{ env.IMAGE }} + docker exec kiosk make test/unit + docker kill kiosk && docker rm kiosk diff --git a/.gitignore b/.gitignore index 12f613a3..93d6446d 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,10 @@ build-harness/ # Documentation +__pycache__/ +build/ docs/build/ docs/source/_sidebar.rst.inc -.vscode/* \ No newline at end of file +# IDE Configuration +.vscode/ diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 0615e5ae..00000000 --- a/.travis.yml +++ /dev/null @@ -1,88 +0,0 @@ -sudo: true -dist: xenial - -git: - depth: false - -language: shell - -services: - - docker - -stages: - - unit_tests - - integration_tests - - deploy - -jobs: - allow_failures: - - name: "Integration test - ELK Enabled" - fast_finish: true - include: - - stage: unit_tests - name: "Unit Tests" - install: - - docker build -t kiosk . - - docker run -d -it --env CLOUD_PROVIDER=gke --entrypoint=/bin/bash --name kiosk kiosk - script: - - docker exec kiosk make test/unit - - docker kill kiosk && docker rm kiosk - - - stage: integration_tests - name: "Integration Tests - ELK Disabled" - if: ((type = pull_request AND branch = master) OR (commit_message =~ /\[build-integration-tests\]/)) - before_install: - - openssl aes-256-cbc -K $encrypted_84460f9bb44c_key -iv $encrypted_84460f9bb44c_iv -in .travis/gcloud_key_base64.enc -out gcloud_key_base64 -d - - cat gcloud_key_base64 | base64 --decode > gcloud_key.json - install: - - docker build -t kiosk . - - > - docker run -d -it - --volume $(readlink -f gcloud_key.json):/tmp/keys/gcloud_key.json:ro - --env CLOUD_PROVIDER=gke - --env GCP_SERVICE_ACCOUNT=$GCP_SERVICE_ACCOUNT - --env CLOUDSDK_CORE_PROJECT=$CLOUDSDK_CORE_PROJECT - --env CLOUDSDK_BUCKET=$CLOUDSDK_BUCKET - --env CLOUDSDK_COMPUTE_REGION=us-west1 - --env REGION_ZONES_WITH_GPUS=us-west1-a,us-west1-b - --env GOOGLE_APPLICATION_CREDENTIALS=/tmp/keys/gcloud_key.json - --entrypoint=/bin/bash - --name kiosk kiosk - script: - - docker exec kiosk make test/integration/gke/deploy - - docker kill kiosk && docker rm kiosk - - - stage: integration_tests - name: "Integration test - ELK Enabled" - if: ((type = pull_request AND branch = master) OR (commit_message =~ /\[build-integration-tests\]/)) AND (commit_message =~ /\[test-elk\]/) - before_install: - - openssl aes-256-cbc -K $encrypted_84460f9bb44c_key -iv $encrypted_84460f9bb44c_iv -in .travis/gcloud_key_base64.enc -out gcloud_key_base64 -d - - cat gcloud_key_base64 | base64 --decode > gcloud_key.json - install: - - docker build -t kiosk . - - > - docker run -d -it - --volume $(readlink -f gcloud_key.json):/tmp/keys/gcloud_key.json:ro - --env CLOUD_PROVIDER=gke - --env GCP_SERVICE_ACCOUNT=$GCP_SERVICE_ACCOUNT - --env CLOUDSDK_CORE_PROJECT=$CLOUDSDK_CORE_PROJECT - --env CLOUDSDK_BUCKET=$CLOUDSDK_BUCKET - --env CLOUDSDK_COMPUTE_REGION=us-west1 - --env REGION_ZONES_WITH_GPUS=us-west1-a,us-west1-b - --env GOOGLE_APPLICATION_CREDENTIALS=/tmp/keys/gcloud_key.json - --entrypoint=/bin/bash - --name kiosk kiosk - script: - - docker exec kiosk make test/integration/gke/deploy/elk - - docker kill kiosk && docker rm kiosk - - - stage: deploy - name: "Docker Deployment" - if: tag IS present - script: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - - docker build -t "$TRAVIS_REPO_SLUG" . - - docker images - - docker tag "$TRAVIS_REPO_SLUG" "$TRAVIS_REPO_SLUG":latest - - docker tag "$TRAVIS_REPO_SLUG" "$TRAVIS_REPO_SLUG":"${TRAVIS_TAG}" - - docker push "$TRAVIS_REPO_SLUG" diff --git a/.travis/gcloud_key_base64.enc b/.travis/gcloud_key_base64.enc deleted file mode 100644 index edf8369f0db7a358ad9ba51ac60894744b2cafc7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3184 zcmV-$43G2tJGNoToA|JP<8QMM0emb2iM;M1!8623g(#z2w&3IG`t4D9TxP&uuc|1* zz)DV_$ORS-rFr?R&laQB(L=%{n1EC@PIz4f&Myzo=1v2!%kmQhQ9Q-=WP_N!U4MJm zQ8dpU$l)O6Qx@(TcMX#82y4&;6}9_`SgqMfjgKEKw8(8N(){uf3-$13g@X+C1k6w7}Cc*6*|DvQpB6pwZki-zJI(0@9_( znV!=BSG2`_%;b`io0e7@Wtm5R?!AlEVqN2@6<`h-s#GO26W~!@CA~Nmr&-*Z@U5JP z-EDOwBZ&<`n&5j3QJ!^YG5|(pWYN>4G=8kF7A-KFI-^X%RzN{_G8-kb)wJk_7J>L^FJvR zRwSezNu5xtZ?jGu@Z(U*E4AV*(8v8Jv$=Uy3)#CW z4jE#D=CY-XOF&Qq%(wb{kI9#oW-It{nDc@ALPqU0c2w*3$iVk}6$p&b@Ofyewcc;{ z2YLg18QVs?XqWpqk^M2?|^8?$5jJ41kY|K2raQz@|VoP%UOgDmp*+zQJM zT$hRy%Q}7B*Xeu7eM@$U`G?qy>G}g3OaM2ZRI>VBCHp~UnUwki6z)7yXZkN&tEW`H z%8`odk3+#GWN$Bz61iFO-)c;?j-0`U+T2TqQuMI9Kt3FDYSSpPvkfDgooHi+yhK@E z2+zWYw7QbQ2;3=au7^?TW;?cX?IE;xPpZe34}Sg@uTlW}8hSBcPUPjZzLM!0mx+uB zFXzpLg==HF)j36l**35I_r_HSTb&zeZ9=IDlwkN~)r(wxM1gVq_Fv2j55u2T5U|dA zw?uN`o8Wl>aURs$O~n<<#5a+=pK2i6h(R$U?i4n5mWKOPlx&SGe_N7W(6z)6^JkPz z(nPUHqzwK*DbNW(MqiidGG(8=$Bh7x10koA2;t}yy7oIngXc=eHud++KA2j>>K3$*Ggw+K7T|;a{*DcywGB> z`?|U$S5Vr?a6HQHyj;OUk)^FRe2uOT zPv=zXTa~Tu(ggG25Umvd3VjQDM4e95n`7O8OS2@}13v6@t;COLeVQ%qd4Jqn%=Ct& zHi%=+u*5Fe*|!t&))yfugNah%z2S3~Qdzk=M{l`s))L+hGpEdb?uN#TaILFTuAE1% zYurq3%5UmiUw(4VT4F>hW11Xem!Ot3Of(t*;GCI|c$%?ARk6Hl(5QpkVXqOCD61Qa zvqfpKqL?@v>oKP3@=B|jPtBW9^u5Kgm3W0__f})25wjYZaiSLD@IQSYfn!;fmxz=p zM#0WnudxM_HZ*OoQel4v2aOXo*p^Ie6AmbtVe)AqQ63+>i0>_zQd@J;jSHy9qnVR9 zS2L>x87unB$cr2;&~lP20te}u=OIYgj{MX*FdDT{U1>2)jG!6*gj1&y-(JJ0U+G{@ zSlT%7M)@?noWTRXg%`rTxV9feQPvS6El^ z2>%qUnFF3{E{Fw3aVgk5GJPkZ)$!a2C`Z{(da(iSnn?NHO>oiF(ba8~E5+nMLj(uH z!^&=Qt!!rTD8(@ht`ZJqg&F9-@o0UW(~ZLfCBu}oxQ@i)hg@9sPY9`)3;nooN=3RQ zdgNdwlL3pKp(ZK*(2NG-QipUj5aeYfE8QihxL-W*=8Q&FhV*u=iWn#*1o*Nb!i3|B z2N6g;~0%%vDc@==>*lFuOBtD$NjKYMXWPE0HyaMc?YvHL3X^)gjLhT>W{IXdOLf+ zkcSYuR+;eBLK%K72flRR5MlP4LG)pnGPzhE*-@E^XxB&fgOfX?G0Q9vEWX&L3bA#y zlf}8XvGhb0E|pYU*S&!u%FY3W>+U>nj|NmLmJ%akRME7(};~JHBUMrQ$UhB;sEnMIbWu~ zMatZ6)hNLSeEpyo(cub;{P#oX20afMtin?LU-<=N$})RXU+wljKrHMK#i(yA@qWJN z{P6aEe@-+0)?5vh?Qc4D55@sRwL0uLE_D~uJvt5Xp?vfixml!l+c)*yy3k?NuZ*1! z2Fs%Q@zWdOCz-c9ZGdg6rwto{1N+H#i6t}Cz;M-ki18zl$AE^)$p?k zD}tSpo{`wR{HB6WUI;PxR8PYq;bFhq+*k$PQ|l9_saz}~Q#0PLLM4cj2)x%E$Dw1{ zu8VhWZQaa{gNp13t?@r#BLuIisGe&jo0TcaY<-KdDp%HfwkvfASP+Crb7#2Bq6ldEd@fBd z&&R%BdN!>~9Obsx$tpWkuV#D@!a)h0~|# zw%*az;CZ-?g0jX8K6_hVKj#!`<$64{OE!u}cXpL$jdRIh3Y9Ww&5z+e4XY&WWH6uM z&qq4S;3Hr>=S!ZkP8Q8di&Ioz-6;5TJG?U}JGTyjf)M)UX?c3OpofA`qLv8>!=SYU zEA8+LXpqSb^7*L{`zhMTEf#GPkLfmP(%Gw*U#eNs++HsXrzzIHFYuhAp)D~tuGj4H zz9g?c=^FRcTv+XwTL_n_>|m*B2+k7;Z)RmA}6Rx6#Vw6mn(cv#dE=U{v+8 zqPr8OLYC`2Wo+q+$4IrV6C5?zO=;G9!3KXxwl?ahUWcH|@k8lWB9e0;@jkTuqdeAA z&XeV7f-Hz5*IsDqes3a&7{(_~1)ofAVw*@ zSokG^Jrmmmh?m6hmzSzXb{>oenaz{FZs?)~QZb9b8$8{((ze={Cw=@V(S>*!sh6}; zMLX7uPEJok^Pb3PG!Mlc+Vj5dX+LW0j zR24A#R-|J5jfpzpE>d1XsSWIJtY7AQCsM5?)06dWsR!>r/dev/null + @echo "DeepCell Kiosk has been created." ## Destroy cluster destroy: \ helmfile/destroy/all \ + kubectl/destroy/pvc \ $(CLOUD_PROVIDER)/destroy/all - @echo "DeepCell Kiosk has been destroyed successfully." + @echo "DeepCell Kiosk has been destroyed." # Unit tests test/unit: @@ -34,20 +34,21 @@ test/unit: ## Create test cluster test/create: \ $(CLOUD_PROVIDER)/test/create/all \ - kubectl/create/prometheus-redis-exporter-script \ helmfile/create/all \ - kubectl/display/ip \ - kubectl/implement/autoscaling + kubectl/display/ip + @kubens deepcell @echo "Cluster created" ## Destroy cluster test/destroy: \ helmfile/destroy/all \ + kubectl/destroy/pvc \ $(CLOUD_PROVIDER)/test/destroy/all @echo "Cluster destroyed" ## Target for testing cluster deployment test/integration/gke/deploy: export CLOUDSDK_CONTAINER_CLUSTER = deepcell-test-$(shell bash -c 'echo $$((1 + $$RANDOM % 1000))') +test/integration/gke/deploy: export CERTIFICATE_MANAGER_ENABLED = true test/integration/gke/deploy: make test/create # TODO: add more testing workflows diff --git a/conf/addons/hpa.yaml b/conf/addons/hpa.yaml deleted file mode 100644 index 18f7353e..00000000 --- a/conf/addons/hpa.yaml +++ /dev/null @@ -1,107 +0,0 @@ -{{ $max_gpus := conv.ToInt (getenv "GPU_NODE_MAX_SIZE" | default 1) }} ---- -apiVersion: autoscaling/v2beta1 -kind: HorizontalPodAutoscaler -metadata: - name: tf-serving - namespace: deepcell -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: tf-serving - minReplicas: 1 - maxReplicas: {{ $max_gpus }} - metrics: - - type: Object - object: - metricName: tf_serving_gpu_usage - target: - apiVersion: v1 - kind: Namespace - name: tf_serving_gpu_usage - targetValue: 70 ---- -apiVersion: autoscaling/v2beta1 -kind: HorizontalPodAutoscaler -metadata: - name: frontend - namespace: deepcell -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: frontend - minReplicas: 1 - maxReplicas: {{ mul $max_gpus 10 }} - metrics: - - type: Resource - resource: - name: cpu - targetAverageUtilization: 80 ---- -apiVersion: autoscaling/v2beta1 -kind: HorizontalPodAutoscaler -metadata: - name: segmentation-consumer - namespace: deepcell -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: segmentation-consumer - minReplicas: 1 - maxReplicas: {{ mul $max_gpus 150 }} - metrics: - - type: Object - object: - metricName: segmentation_consumer_key_ratio - target: - apiVersion: v1 - kind: Namespace - name: segmentation_consumer_key_ratio - targetValue: .15 ---- -apiVersion: autoscaling/v2beta1 -kind: HorizontalPodAutoscaler -metadata: - name: segmentation-zip-consumer - namespace: deepcell -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: segmentation-zip-consumer - minReplicas: 1 - maxReplicas: {{ mul $max_gpus 100 }} - metrics: - - type: Object - object: - metricName: segmentation_zip_consumer_key_ratio - target: - apiVersion: v1 - kind: Namespace - name: segmentation_zip_consumer_key_ratio - targetValue: 2 ---- -apiVersion: autoscaling/v2beta1 -kind: HorizontalPodAutoscaler -metadata: - name: tracking-consumer - namespace: deepcell -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: tracking-consumer - minReplicas: 1 - maxReplicas: {{ mul $max_gpus 50 }} - metrics: - - type: Object - object: - metricName: tracking_consumer_key_ratio - target: - apiVersion: v1 - kind: Namespace - name: tracking_consumer_key_ratio - targetValue: 1 diff --git a/conf/addons/redis-exporter-script.yaml b/conf/addons/redis-exporter-script.yaml deleted file mode 100644 index 1d6e1dee..00000000 --- a/conf/addons/redis-exporter-script.yaml +++ /dev/null @@ -1,65 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: prometheus-redis-exporter-script - namespace: monitoring -data: - script: | - -- Based on https://github.com/soveran/rediscan.lua by GitHub user Soveran. - - local function get_queue_count(queue) - -- Find number of keys in the queue - local queue_size = redis.call("LLEN", queue) - - -- Get all processing queues - local queue_regex = "processing-" .. queue .. ":*" - - local count = 0 - - local cursor = "0" - local done = false - - repeat - - local result = redis.call("SCAN", cursor, "MATCH", queue_regex, "COUNT", 1000) - cursor = result[1] - - for i, key in ipairs(result[2]) do - -- How many keys are in each queue (should be 1) - local keys_in_queue = redis.call("LLEN", key) - count = count + keys_in_queue - end - - if cursor == "0" then - done = true - end - - until done - - return count + queue_size - end - - -- Final table to output - local results = {} - - -- All Queues to Monitor: - local queues = {} - - queues[#queues+1] = "segmentation" - queues[#queues+1] = "tracking" - - for _,queue in ipairs(queues) do - local zip_queue = queue .. "-zip" - - local queue_count = get_queue_count(queue) - local zip_queue_count = get_queue_count(zip_queue) - - table.insert(results, queue) - table.insert(results, tostring(queue_count)) - - table.insert(results, zip_queue) - table.insert(results, tostring(zip_queue_count)) - - end - - return results diff --git a/conf/charts/data-processing/Chart.yaml b/conf/charts/data-processing/Chart.yaml deleted file mode 100644 index 1d11c391..00000000 --- a/conf/charts/data-processing/Chart.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: v1 -name: data-processing -version: 0.1.0 -#kubeVersion: ^1.9.8 -description: A gRPC API for data pre- and post-processing. -keywords: - - kiosk-data-processing -home: https://www.github.com/vanvalenlab/kiosk-data-processing -sources: - - https://hub.docker.com/vanvalenlab/kiosk-data-processing -maintainers: - - name: Dylan Bannon - email: bbannon@caltech.edu - url: vanvalen.caltech.edu -engine: gotpl -appVersion: 0.1.0 -deprecated: false -tillerVersion: ^2.9.1 diff --git a/conf/charts/data-processing/LICENSE b/conf/charts/data-processing/LICENSE deleted file mode 100644 index e69de29b..00000000 diff --git a/conf/charts/data-processing/README.md b/conf/charts/data-processing/README.md deleted file mode 100644 index e038f7b2..00000000 --- a/conf/charts/data-processing/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# kiosk-data-processing - -This is a gRPC API for pre- and post-processing data. diff --git a/conf/charts/data-processing/requirements.lock b/conf/charts/data-processing/requirements.lock deleted file mode 100644 index e69de29b..00000000 diff --git a/conf/charts/data-processing/requirements.yaml b/conf/charts/data-processing/requirements.yaml deleted file mode 100644 index e69de29b..00000000 diff --git a/conf/charts/data-processing/templates/NOTES.txt b/conf/charts/data-processing/templates/NOTES.txt deleted file mode 100644 index e69de29b..00000000 diff --git a/conf/charts/data-processing/templates/_helpers.tpl b/conf/charts/data-processing/templates/_helpers.tpl deleted file mode 100644 index 6efc7301..00000000 --- a/conf/charts/data-processing/templates/_helpers.tpl +++ /dev/null @@ -1,33 +0,0 @@ - -{{/* vim: set filetype=mustache: */}} -{{/* -Expand the name of the chart. -*/}} -{{- define "name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} -{{- end -}} diff --git a/conf/charts/data-processing/templates/deployment.yaml b/conf/charts/data-processing/templates/deployment.yaml deleted file mode 100644 index 7be9c7ee..00000000 --- a/conf/charts/data-processing/templates/deployment.yaml +++ /dev/null @@ -1,68 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ template "fullname" . }} - labels: - app: {{ template "name" . }} - chart: {{ template "chart" . }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -spec: - selector: - matchLabels: - app: {{ template "name" . }} - release: {{ .Release.Name }} - replicas: {{ .Values.replicas }} - template: - metadata: - labels: - app: {{ template "name" . }} - release: {{ .Release.Name }} - annotations: -{{ toYaml .Values.annotations | indent 8 }} - spec: - {{- with .Values.nodeSelector }} - nodeSelector: -{{ toYaml . | indent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: -{{ toYaml . | indent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: -{{ toYaml . | indent 8 }} - {{- end }} - containers: - - name: {{ .Chart.Name }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - name: http - containerPort: {{ .Values.service.internalHttpPort }} - protocol: TCP - - name: https - containerPort: {{ .Values.service.internalHttpsPort }} - protocol: TCP - - name: prometheus - containerPort: {{ .Values.service.prometheusHttpPort }} - protocol: TCP - resources: -{{ toYaml .Values.resources | indent 10 }} - env: -{{- range $name, $value := .Values.env }} -{{- if not ( empty $value) }} - - name: {{ $name }} - value: {{ $value | quote }} -{{- end }} -{{- end }} -{{ $secrets_name := .Values.secrets_name }} -{{- range $name, $value := .Values.secrets }} -{{- if not ( empty $value) }} - - name: {{ $name }} - valueFrom: - secretKeyRef: - name: {{ $secrets_name }} - key: {{ $name }} -{{- end }} -{{- end }} diff --git a/conf/charts/data-processing/templates/service.yaml b/conf/charts/data-processing/templates/service.yaml deleted file mode 100644 index 4f7d9738..00000000 --- a/conf/charts/data-processing/templates/service.yaml +++ /dev/null @@ -1,33 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - name: {{ template "fullname" . }} - labels: - app: {{ template "name" . }} -{{- if .Values.service.annotations }} - annotations: -{{ toYaml .Values.service.annotations | indent 4 }} -{{- end }} -spec: - type: {{ .Values.service.type }} - ports: - {{- if .Values.service.prometheusEnabled }} - - name: prometheus - targetPort: {{ .Values.service.prometheusHttpPort }} - port: {{ .Values.service.prometheusHttpPort }} - protocol: TCP - {{- end }} - {{- if .Values.service.httpIngressEnabled }} - - name: http - targetPort: {{ .Values.service.httpTargetPort }} - port: {{ .Values.service.externalHttpPort }} - protocol: TCP - {{- end }} - {{- if .Values.service.httpsIngressEnabled }} - - name: https - targetPort: {{ .Values.service.httpsTargetPort }} - port: {{ .Values.service.externalHttpsPort }} - protocol: TCP - {{- end }} - selector: - app: {{ template "fullname" . }} diff --git a/conf/charts/data-processing/values.yaml b/conf/charts/data-processing/values.yaml deleted file mode 100644 index e7f73dd0..00000000 --- a/conf/charts/data-processing/values.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# Default values for data-processing. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -replicas: 1 - -image: - repository: vanvalenlab/kiosk-data-processing - tag: latest - pullPolicy: IfNotPresent - -resources: {} - -tolerations: {} - -affinity: {} - -nodeSelector: {} - -service: - type: "ClusterIP" - - httpIngressEnabled: true - internalHttpPort: 8080 - httpTargetPort: 8080 - externalHttpPort: 8080 - - httpsIngressEnabled: false - internalHttpsPort: 8443 - httpsTargetPort: 8443 - externalHttpsPort: 8443 - - prometheusEnabled: true - prometheusHttpPort: 8000 - -env: - LISTEN_PORT: 8080 - PROMETHEUS_PORT: 8000 - PROMETHEUS_ENABLED: "true" diff --git a/conf/charts/frontend/Chart.yaml b/conf/charts/frontend/Chart.yaml index 1fe58e6d..c5e7fb4f 100644 --- a/conf/charts/frontend/Chart.yaml +++ b/conf/charts/frontend/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: frontend -version: 0.1.0 +version: 0.2.1 #kubeVersion: ^1.9.8 description: This project provides the frontend interface for the Tensorflow-serving backend of Deepcell. keywords: @@ -13,6 +13,6 @@ maintainers: email: bbannon@caltech.edu url: vanvalen.caltech.edu engine: gotpl -appVersion: 0.1.0 +appVersion: 0.6.0 deprecated: false tillerVersion: ^2.9.1 diff --git a/conf/charts/frontend/templates/hpa.yaml b/conf/charts/frontend/templates/hpa.yaml new file mode 100644 index 00000000..26eb7921 --- /dev/null +++ b/conf/charts/frontend/templates/hpa.yaml @@ -0,0 +1,23 @@ +{{- if (.Values.hpa.enabled) }} +--- +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ template "fullname" . }} + labels: + app: {{ template "name" . }} + chart: {{ template "chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ template "fullname" . }} + minReplicas: {{ .Values.hpa.minReplicas }} + maxReplicas: {{ .Values.hpa.maxReplicas }} + metrics: +{{ toYaml .Values.hpa.metrics | indent 4 }} +{{- end }} diff --git a/conf/charts/frontend/templates/ingress.yaml b/conf/charts/frontend/templates/ingress.yaml index 8abf9f92..17cb455d 100644 --- a/conf/charts/frontend/templates/ingress.yaml +++ b/conf/charts/frontend/templates/ingress.yaml @@ -4,24 +4,42 @@ apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: - name: {{ $fullName }} + name: {{ template "fullname" . }} labels: app: {{ template "name" . }} chart: {{ template "chart" . }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} annotations: -{{- range $name, $value := .Values.ingress.annotations }} -{{- if not ( empty $value) }} - {{ $name }}: {{ $value | quote }} -{{- end }} +{{- if .Values.service.annotations }} + annotations: +{{ toYaml .Values.service.annotations | indent 4 }} {{- end }} spec: rules: - http: + paths: + - path: {{ .Values.ingress.path }} + backend: + serviceName: {{ template "fullname" . }} + servicePort: http + {{- range .Values.ingress.hosts }} + - host: {{ . }} + http: paths: - path: {{ $ingressPath }} backend: serviceName: {{ $fullName }} servicePort: http + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end -}} {{- end }} diff --git a/conf/charts/frontend/templates/service.yaml b/conf/charts/frontend/templates/service.yaml index 2b3d9d85..15c871f8 100644 --- a/conf/charts/frontend/templates/service.yaml +++ b/conf/charts/frontend/templates/service.yaml @@ -3,7 +3,10 @@ apiVersion: v1 metadata: name: {{ template "fullname" . }} labels: - app: {{ template "fullname" . }} + app: {{ template "name" . }} + chart: {{ template "chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} {{- if .Values.service.annotations }} annotations: {{ toYaml .Values.service.annotations | indent 4 }} diff --git a/conf/charts/frontend/values.yaml b/conf/charts/frontend/values.yaml index 5e364cd2..5389859e 100644 --- a/conf/charts/frontend/values.yaml +++ b/conf/charts/frontend/values.yaml @@ -6,7 +6,7 @@ replicas: 1 image: repository: vanvalenlab/kiosk-frontend - tag: latest + tag: 0.6.0 pullPolicy: IfNotPresent ingress: @@ -17,6 +17,12 @@ ingress: #nginx.ingress.kubernetes.io/rewrite-target: / #nginx.ingress.kubernetes.io/ssl-redirect: "false" #nginx.ingress.kubernetes.io/proxy-body-size: "25m" + hosts: [] + # - example.com + tls: [] + # - hosts: + # - example.com + # secretName: tls-cert service: type: "ClusterIP" @@ -31,6 +37,12 @@ service: httpsTargetPort: 8443 externalHttpsPort: 8443 +hpa: + enabled: false + minReplicas: 1 + maxReplicas: 2 + metrics: {} + resources: {} env: @@ -40,11 +52,9 @@ env: REDIS_PORT: "6379" MODEL_PREFIX: "models/" AWS_REGION: "us-east-1" - CLOUD_PROVIDER: "aws" secrets: - AWS_ACCESS_KEY_ID: "change_me" - AWS_SECRET_ACCESS_KEY: "change_me" - AWS_S3_BUCKET: "change_me" - GCLOUD_STORAGE_BUCKET: "change_me" - GCLOUD_PROJECT_ID: "change_me" + AWS_ACCESS_KEY_ID: "" + AWS_SECRET_ACCESS_KEY: "" + GCLOUD_PROJECT_ID: "" + STORAGE_BUCKET: "s3://example-bucket" diff --git a/conf/charts/redis-consumer/Chart.yaml b/conf/charts/redis-consumer/Chart.yaml index 63cea649..9c4c17cc 100644 --- a/conf/charts/redis-consumer/Chart.yaml +++ b/conf/charts/redis-consumer/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: redis-consumer -version: 0.1.0 +version: 0.2.1 #kubeVersion: ^1.9.8 description: This project consumes and processes redis event, placing the final result back to redis. keywords: @@ -13,7 +13,6 @@ maintainers: email: bbannon@caltech.edu url: vanvalen.caltech.edu engine: gotpl -appVersion: 0.1.0 +appVersion: 0.8.3 deprecated: false tillerVersion: ^2.9.1 - diff --git a/conf/charts/redis-consumer/requirements.lock b/conf/charts/redis-consumer/requirements.lock deleted file mode 100644 index a9121a7a..00000000 --- a/conf/charts/redis-consumer/requirements.lock +++ /dev/null @@ -1,3 +0,0 @@ -dependencies: [] -digest: sha256:a4028ef6b5df0ba52501af1116d232a75056093f30ddf3b725e34fff764048c2 -generated: 2018-08-30T19:10:26.219214026Z diff --git a/conf/charts/redis-consumer/requirements.yaml b/conf/charts/redis-consumer/requirements.yaml index 8b137891..e69de29b 100644 --- a/conf/charts/redis-consumer/requirements.yaml +++ b/conf/charts/redis-consumer/requirements.yaml @@ -1 +0,0 @@ - diff --git a/conf/charts/redis-consumer/templates/hpa.yaml b/conf/charts/redis-consumer/templates/hpa.yaml new file mode 100644 index 00000000..26eb7921 --- /dev/null +++ b/conf/charts/redis-consumer/templates/hpa.yaml @@ -0,0 +1,23 @@ +{{- if (.Values.hpa.enabled) }} +--- +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ template "fullname" . }} + labels: + app: {{ template "name" . }} + chart: {{ template "chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ template "fullname" . }} + minReplicas: {{ .Values.hpa.minReplicas }} + maxReplicas: {{ .Values.hpa.maxReplicas }} + metrics: +{{ toYaml .Values.hpa.metrics | indent 4 }} +{{- end }} diff --git a/conf/charts/redis-consumer/values.yaml b/conf/charts/redis-consumer/values.yaml index 8390aa72..09f268c2 100644 --- a/conf/charts/redis-consumer/values.yaml +++ b/conf/charts/redis-consumer/values.yaml @@ -6,7 +6,7 @@ replicas: 1 image: repository: vanvalenlab/kiosk-redis-consumer - tag: latest + tag: 0.8.3 pullPolicy: IfNotPresent resources: {} @@ -27,6 +27,12 @@ service: httpsIngressEnabled: false +hpa: + enabled: false + minReplicas: 1 + maxReplicas: 2 + metrics: {} + env: DEBUG: "True" INTERVAL: 10 @@ -77,7 +83,7 @@ env: TRACKING_POSTPROCESS_FUNCTION: "deep_watershed" secrets: - AWS_ACCESS_KEY_ID: "change_me" - AWS_SECRET_ACCESS_KEY: "change_me" - AWS_S3_BUCKET: "change_me" - GKE_BUCKET: "change_me" + AWS_ACCESS_KEY_ID: "" + AWS_SECRET_ACCESS_KEY: "" + AWS_S3_BUCKET: "s3://example-bucket" + GKE_BUCKET: "gs://example-bucket" diff --git a/conf/charts/tensorboard/Chart.yaml b/conf/charts/tensorboard/Chart.yaml deleted file mode 100644 index 89817463..00000000 --- a/conf/charts/tensorboard/Chart.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -appVersion: "1.0" -description: A Helm chart for Kubernetes -name: tensorboard -version: 0.1.0 diff --git a/conf/charts/tensorboard/LICENSE b/conf/charts/tensorboard/LICENSE deleted file mode 100644 index e69de29b..00000000 diff --git a/conf/charts/tensorboard/NOTES.txt b/conf/charts/tensorboard/NOTES.txt deleted file mode 100644 index b08b83b5..00000000 --- a/conf/charts/tensorboard/NOTES.txt +++ /dev/null @@ -1,19 +0,0 @@ -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range .Values.ingress.hosts }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ . }}{{ $.Values.ingress.path }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "tensorboard.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get svc -w {{ include "tensorboard.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "tensorboard.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "tensorboard.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl port-forward $POD_NAME 8080:80 -{{- end }} diff --git a/conf/charts/tensorboard/README.md b/conf/charts/tensorboard/README.md deleted file mode 100644 index e69de29b..00000000 diff --git a/conf/charts/tensorboard/templates/_helpers.tpl b/conf/charts/tensorboard/templates/_helpers.tpl deleted file mode 100644 index 1b60a1c0..00000000 --- a/conf/charts/tensorboard/templates/_helpers.tpl +++ /dev/null @@ -1,40 +0,0 @@ - -{{/* vim: set filetype=mustache: */}} -{{/* -Expand the name of the chart. -*/}} -{{- define "name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Generate chart secret name -*/}} -{{- define "secretName" -}} -{{ default (include "fullname" .) .Values.existingSecret }} -{{- end -}} diff --git a/conf/charts/tensorboard/templates/deployment.yaml b/conf/charts/tensorboard/templates/deployment.yaml deleted file mode 100644 index 557e090d..00000000 --- a/conf/charts/tensorboard/templates/deployment.yaml +++ /dev/null @@ -1,46 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ template "fullname" . }} - labels: - app: {{ template "name" . }} - chart: {{ template "chart" . }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -spec: - selector: - matchLabels: - app: {{ template "name" . }} - release: {{ .Release.Name }} - replicas: {{ .Values.replicas }} - template: - metadata: - labels: - app: {{ template "name" . }} - release: {{ .Release.Name }} - spec: - containers: - - name: {{ .Chart.Name }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - containerPort: {{ .Values.service.httpTargetPort }} - resources: -{{ toYaml .Values.resources | indent 10 }} - env: -{{- range $name, $value := .Values.env }} -{{- if not ( empty $value) }} - - name: {{ $name }} - value: {{ $value | quote }} -{{- end }} -{{- end }} -{{ $dot := . }} -{{- range $name, $value := .Values.secrets }} -{{- if not ( empty $value) }} - - name: {{ $name }} - valueFrom: - secretKeyRef: - name: {{ template "secretName" $dot }} - key: {{ $name }} -{{- end }} -{{- end }} diff --git a/conf/charts/tensorboard/templates/ingress.yaml b/conf/charts/tensorboard/templates/ingress.yaml deleted file mode 100644 index 8abf9f92..00000000 --- a/conf/charts/tensorboard/templates/ingress.yaml +++ /dev/null @@ -1,27 +0,0 @@ -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "fullname" . -}} -{{- $ingressPath := .Values.ingress.path -}} -apiVersion: networking.k8s.io/v1beta1 -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - app: {{ template "name" . }} - chart: {{ template "chart" . }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} - annotations: -{{- range $name, $value := .Values.ingress.annotations }} -{{- if not ( empty $value) }} - {{ $name }}: {{ $value | quote }} -{{- end }} -{{- end }} -spec: - rules: - - http: - paths: - - path: {{ $ingressPath }} - backend: - serviceName: {{ $fullName }} - servicePort: http -{{- end }} diff --git a/conf/charts/tensorboard/templates/secrets.yaml b/conf/charts/tensorboard/templates/secrets.yaml deleted file mode 100644 index 28daba85..00000000 --- a/conf/charts/tensorboard/templates/secrets.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: {{ template "secretName" . }} -type: Opaque -data: -{{- range $name, $value := .Values.secrets }} -{{- if not (empty $value) }} - {{ $name }}: {{ $value | b64enc }} -{{- end }} -{{- end }} diff --git a/conf/charts/tensorboard/templates/service.yaml b/conf/charts/tensorboard/templates/service.yaml deleted file mode 100644 index 2b3d9d85..00000000 --- a/conf/charts/tensorboard/templates/service.yaml +++ /dev/null @@ -1,27 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - name: {{ template "fullname" . }} - labels: - app: {{ template "fullname" . }} -{{- if .Values.service.annotations }} - annotations: -{{ toYaml .Values.service.annotations | indent 4 }} -{{- end }} -spec: - type: {{ .Values.service.type }} - ports: - {{- if .Values.service.httpIngressEnabled }} - - name: http - targetPort: {{ .Values.service.httpTargetPort }} - port: {{ .Values.service.externalHttpPort }} - protocol: TCP - {{- end }} - {{- if .Values.service.httpsIngressEnabled }} - - name: https - targetPort: {{ .Values.service.httpsTargetPort }} - port: {{ .Values.service.externalHttpsPort }} - protocol: TCP - {{- end }} - selector: - app: {{ template "fullname" . }} diff --git a/conf/charts/tensorboard/values.yaml b/conf/charts/tensorboard/values.yaml deleted file mode 100644 index a2a8f720..00000000 --- a/conf/charts/tensorboard/values.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Default values for tensorboard. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -replicas: 1 - -image: - repository: vanvalenlab/kiosk-tensorboard - tag: latest - pullPolicy: IfNotPresent - -resources: {} - -service: - type: "ClusterIP" - - httpIngressEnabled: true - internalHttpPort: 6006 - httpTargetPort: 6006 - externalHttpPort: 6006 - - httpsIngressEnabled: false - -ingress: - enabled: true - path: /tensorboard - annotations: {} - # kubernetes.io/ingress.class: "nginx" - # nginx.ingress.kubernetes.io/rewrite-target: / - # nginx.ingress.kubernetes.io/ssl-redirect: "false" - # nginx.ingress.kubernetes.io/proxy-body-size: "25m" - -env: - LOG_PREFIX: "tensorboard_logs" - CLOUD_PROVIDER: "aws" - GKE_COMPUTE_ZONE: "us-west1-b" - -secrets: - AWS_ACCESS_KEY_ID: "change_me" - AWS_SECRET_ACCESS_KEY: "change_me" - AWS_S3_BUCKET: "change_me" - GKE_BUCKET: "change_me" diff --git a/conf/charts/tf-serving/Chart.yaml b/conf/charts/tf-serving/Chart.yaml index bebd4012..3d0569e9 100755 --- a/conf/charts/tf-serving/Chart.yaml +++ b/conf/charts/tf-serving/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: tf-serving -version: 0.1.0 +version: 0.4.0 #kubeVersion: ^1.9.8 description: This helm chart provides the Tensorflow-serving backend functionality of the Deepcell application. keywords: @@ -13,7 +13,6 @@ maintainers: email: bbannon@caltech.edu url: vanvalen.caltech.edu engine: gotpl -appVersion: 0.1.0 +appVersion: 0.4.0 deprecated: false tillerVersion: ^2.9.1 - diff --git a/conf/charts/tf-serving/templates/deployment.yaml b/conf/charts/tf-serving/templates/deployment.yaml index 9dc097fe..bfb653d1 100755 --- a/conf/charts/tf-serving/templates/deployment.yaml +++ b/conf/charts/tf-serving/templates/deployment.yaml @@ -34,10 +34,42 @@ spec: tolerations: {{ toYaml . | indent 8 }} {{- end }} + dnsPolicy: Default + volumes: + - name: {{ .Values.configWriter.mountedVolume.name }} + emptyDir: {} + # These containers are run during pod initialization + initContainers: + - name: install + image: "{{ .Values.configWriter.image.repository }}:{{ .Values.configWriter.image.tag }}" + imagePullPolicy: {{ .Values.configWriter.image.pullPolicy }} + volumeMounts: + - name: {{ .Values.configWriter.mountedVolume.name }} + mountPath: {{ .Values.configWriter.mountedVolume.path }} + env: +{{- range $name, $value := .Values.env }} +{{- if not ( empty $value) }} + - name: {{ $name }} + value: {{ $value | quote }} +{{- end }} +{{- end }} +{{ $dot := . }} +{{- range $name, $value := .Values.secrets }} +{{- if not ( empty $value) }} + - name: {{ $name }} + valueFrom: + secretKeyRef: + name: {{ template "secretName" $dot }} + key: {{ $name }} +{{- end }} +{{- end }} containers: - name: {{ .Chart.Name }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} + volumeMounts: + - name: {{ .Values.configWriter.mountedVolume.name }} + mountPath: {{ .Values.configWriter.mountedVolume.path }} resources: {{ toYaml .Values.resources | indent 10 }} ports: diff --git a/conf/charts/tf-serving/templates/hpa.yaml b/conf/charts/tf-serving/templates/hpa.yaml new file mode 100644 index 00000000..26eb7921 --- /dev/null +++ b/conf/charts/tf-serving/templates/hpa.yaml @@ -0,0 +1,23 @@ +{{- if (.Values.hpa.enabled) }} +--- +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ template "fullname" . }} + labels: + app: {{ template "name" . }} + chart: {{ template "chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ template "fullname" . }} + minReplicas: {{ .Values.hpa.minReplicas }} + maxReplicas: {{ .Values.hpa.maxReplicas }} + metrics: +{{ toYaml .Values.hpa.metrics | indent 4 }} +{{- end }} diff --git a/conf/charts/tf-serving/values.yaml b/conf/charts/tf-serving/values.yaml index 86124ce9..4762e389 100755 --- a/conf/charts/tf-serving/values.yaml +++ b/conf/charts/tf-serving/values.yaml @@ -6,9 +6,19 @@ replicas: 0 image: repository: vanvalenlab/kiosk-tf-serving - tag: latest + tag: 0.4.0 pullPolicy: IfNotPresent +configWriter: + mountedVolume: + name: configdir + path: /config + + image: + repository: vanvalenlab/kiosk-tf-serving-config-writer + tag: 0.4.0 + pullPolicy: IfNotPresent + service: type: ClusterIP @@ -45,27 +55,31 @@ tolerations: {} affinity: {} +hpa: + enabled: false + minReplicas: 0 + maxReplicas: 1 + metrics: {} + env: PORT: 8500 REST_API_PORT: 8501 REST_API_TIMEOUT: 30000 - MODEL_CONFIG_FILE: /kiosk/tf-serving/models.conf - BATCHING_CONFIG_FILE: /kiosk/tf-serving/batching_config.txt + MODEL_CONFIG_FILE: /config/models.conf + BATCHING_CONFIG_FILE: /config/batching_config.txt ENABLE_BATCHING: "true" MAX_BATCH_SIZE: 1 BATCH_TIMEOUT_MICROS: 0 MAX_ENQUEUED_BATCHES: 512 GRPC_CHANNEL_ARGS: "" MODEL_PREFIX: models - CLOUD_PROVIDER: '{{ env "CLOUD_PROVIDER" | default "aws" }}' TF_CPP_MIN_LOG_LEVEL: 0 TF_SESSION_PARALLELISM: 0 - MONITORING_CONFIG_FILE: /kiosk/tf-serving/monitoring_config.txt + MONITORING_CONFIG_FILE: /config/monitoring_config.txt PROMETHEUS_MONITORING_ENABLED: "true" PROMETHEUS_MONITORING_PATH: /monitoring/prometheus/metrics secrets: - AWS_ACCESS_KEY_ID: "change_me" - AWS_SECRET_ACCESS_KEY: "change_me" - AWS_S3_BUCKET: "change_me" - GCLOUD_STORAGE_BUCKET: "change_me" + AWS_ACCESS_KEY_ID: "" + AWS_SECRET_ACCESS_KEY: "" + STORAGE_BUCKET: gs://deepcell-models diff --git a/conf/charts/training/Chart.yaml b/conf/charts/training/Chart.yaml deleted file mode 100644 index 2db2c1b0..00000000 --- a/conf/charts/training/Chart.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: v1 -name: training -version: 0.1.0 -#kubeVersion: ^1.9.8 -description: This helm chart runs a training job based on deepcell generated notebooks. -keywords: - - tensorflow - - deepcell-tf - - kiosk-training -home: https://www.github.com/vanvalenlab/kiosk-training -sources: - - https://hub.docker.com/vanvalenlab/kiosk-training -maintainers: - - name: Dylan Bannon - email: bbannon@caltech.edu - url: vanvalen.caltech.edu -engine: gotpl -appVersion: 0.1.0 -deprecated: false -tillerVersion: ^2.9.1 diff --git a/conf/charts/training/LICENSE b/conf/charts/training/LICENSE deleted file mode 100644 index e69de29b..00000000 diff --git a/conf/charts/training/README.md b/conf/charts/training/README.md deleted file mode 100644 index e69de29b..00000000 diff --git a/conf/charts/training/requirements.yaml b/conf/charts/training/requirements.yaml deleted file mode 100644 index e69de29b..00000000 diff --git a/conf/charts/training/templates/NOTES.txt b/conf/charts/training/templates/NOTES.txt deleted file mode 100644 index e69de29b..00000000 diff --git a/conf/charts/training/templates/_helpers.tpl b/conf/charts/training/templates/_helpers.tpl deleted file mode 100644 index 1b60a1c0..00000000 --- a/conf/charts/training/templates/_helpers.tpl +++ /dev/null @@ -1,40 +0,0 @@ - -{{/* vim: set filetype=mustache: */}} -{{/* -Expand the name of the chart. -*/}} -{{- define "name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Generate chart secret name -*/}} -{{- define "secretName" -}} -{{ default (include "fullname" .) .Values.existingSecret }} -{{- end -}} diff --git a/conf/charts/training/templates/job.yaml b/conf/charts/training/templates/job.yaml deleted file mode 100644 index b1663e75..00000000 --- a/conf/charts/training/templates/job.yaml +++ /dev/null @@ -1,63 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ template "fullname" . }} - labels: - app: {{ template "name" . }} - chart: {{ template "chart" . }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -spec: - parallelism: {{ .Values.parallelism }} - completions: 2147483647 - # selector: - # matchLabels: - # app: {{ template "name" . }} - # release: {{ .Release.Name }} - template: - metadata: - labels: - app: {{ template "name" . }} - release: {{ .Release.Name }} - spec: - restartPolicy: Never - nodeSelector: -{{- range $name, $value := .Values.nodeSelector }} -{{- if not ( empty $value) }} - {{ $name }}: {{ $value | quote }} -{{- end }} -{{- end }} - tolerations: - - key: "nvidia.com/gpu" - operator: "Exists" - effect: "NoSchedule" - - key: "training_gpu" - operator: "Equal" - value: "yes" - effect: "NoSchedule" - containers: - - name: {{ .Chart.Name }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - resources: - limits: - nvidia.com/gpu: 1 - ports: - - containerPort: {{ .Values.service.httpTargetPort }} - env: -{{- range $name, $value := .Values.env }} -{{- if not ( empty $value) }} - - name: {{ $name }} - value: {{ $value | quote }} -{{- end }} -{{- end }} -{{ $dot := . }} -{{- range $name, $value := .Values.secrets }} -{{- if not ( empty $value) }} - - name: {{ $name }} - valueFrom: - secretKeyRef: - name: {{ template "secretName" $dot }} - key: {{ $name }} -{{- end }} -{{- end }} diff --git a/conf/charts/training/templates/secrets.yaml b/conf/charts/training/templates/secrets.yaml deleted file mode 100644 index 28daba85..00000000 --- a/conf/charts/training/templates/secrets.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: {{ template "secretName" . }} -type: Opaque -data: -{{- range $name, $value := .Values.secrets }} -{{- if not (empty $value) }} - {{ $name }}: {{ $value | b64enc }} -{{- end }} -{{- end }} diff --git a/conf/charts/training/templates/service.yaml b/conf/charts/training/templates/service.yaml deleted file mode 100644 index 2b3d9d85..00000000 --- a/conf/charts/training/templates/service.yaml +++ /dev/null @@ -1,27 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - name: {{ template "fullname" . }} - labels: - app: {{ template "fullname" . }} -{{- if .Values.service.annotations }} - annotations: -{{ toYaml .Values.service.annotations | indent 4 }} -{{- end }} -spec: - type: {{ .Values.service.type }} - ports: - {{- if .Values.service.httpIngressEnabled }} - - name: http - targetPort: {{ .Values.service.httpTargetPort }} - port: {{ .Values.service.externalHttpPort }} - protocol: TCP - {{- end }} - {{- if .Values.service.httpsIngressEnabled }} - - name: https - targetPort: {{ .Values.service.httpsTargetPort }} - port: {{ .Values.service.externalHttpsPort }} - protocol: TCP - {{- end }} - selector: - app: {{ template "fullname" . }} diff --git a/conf/charts/training/values.yaml b/conf/charts/training/values.yaml deleted file mode 100644 index d1581323..00000000 --- a/conf/charts/training/values.yaml +++ /dev/null @@ -1,43 +0,0 @@ -# Default values for training. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -parallelism: 0 - -image: - repository: vanvalenlab/kiosk-training - tag: latest - pullPolicy: IfNotPresent - -resources: {} - -service: - type: ClusterIP - - httpIngressEnabled: true - internalHttpPort: 7241 - httpTargetPort: 7241 - externalHttpPort: 7241 - - httpsIngressEnabled: false - -nodeSelector: - # cloud.google.com/gke-accelerator: "nvidia-tesla-k80" - # beta.kubernetes.io/instance-type: "p2.xlarge" - -# secrets.yaml and hemlfile.d/training.yaml -env: - CLOUD_PROVIDER: "aws" - DEBUG: "true" - EXPORT_PREFIX: "models" - LOG_PREFIX: "tensorboard_logs" - REDIS_HOST: "redis-master" - REDIS_PORT: "6379" - STATUS: "new" - GKE_COMPUTE_ZONE: "us-west1-b" - -secrets: - AWS_ACCESS_KEY_ID: "change_me" - AWS_SECRET_ACCESS_KEY: "change_me" - AWS_S3_BUCKET: "change_me" - GCLOUD_STORAGE_BUCKET: "change_me" diff --git a/conf/helmfile.d/0000.ingress.yaml b/conf/helmfile.d/0000.ingress.yaml index 6282dfc5..a534d944 100644 --- a/conf/helmfile.d/0000.ingress.yaml +++ b/conf/helmfile.d/0000.ingress.yaml @@ -1,7 +1,7 @@ repositories: # Stable repo of official helm charts - name: stable - url: https://kubernetes-charts.storage.googleapis.com + url: https://charts.helm.sh/stable releases: @@ -25,6 +25,9 @@ releases: chart: stable/nginx-ingress version: 1.40.2 wait: true + timeout: 300 + atomic: true + cleanupOnFail: true values: - controller: ### Optional: NGINX_INGRESS_CONTROLLER_REPLICA_COUNT; e.g. 2 diff --git a/conf/helmfile.d/0010.cert-manager.yaml b/conf/helmfile.d/0010.cert-manager.yaml new file mode 100644 index 00000000..e3dcadba --- /dev/null +++ b/conf/helmfile.d/0010.cert-manager.yaml @@ -0,0 +1,106 @@ +repositories: + # Add the Jetstack Helm repository + - name: jetstack + url: https://charts.jetstack.io + # Kubernetes incubator repo of helm charts + - name: incubator + url: https://charts.helm.sh/incubator + +releases: + +################################################################################ +## Certificate Manager ######################################################### +################################################################################ + +# +# References: +# - https://github.com/cloudposse/charts/blob/master/incubator/nginx-ingress/values.yaml +# +- name: cert-manager + namespace: cert-manager + labels: + chart: cert-manager + repo: jetstack + component: cert-manager + namespace: cert-manager + vendor: kubernetes + chart: jetstack/cert-manager + version: 1.1.0 + wait: true + timeout: 300 + atomic: true + cleanupOnFail: true + hooks: + - events: ["postsync"] + # Give cert-manager time to initialize itself + showlogs: true + command: /bin/sleep + args: + - 15 + values: + - installCRDs: true + + # serviceAccount: + # annotations: + # iam.gke.io/gcp-service-account: {{ env "GCP_SERVICE_ACCOUNT" | default "cloud-dns-sa-not-found"}} + + ingressShim: + # Use "letsencrypt-prod" for production. + defaultIssuerName: {{ env "CERTIFICATE_MANAGER_CLUSTER_ISSUER" | default "letsecrypt-staging" }} + defaultIssuerKind: ClusterIssuer + defaultIssuerGroup: cert-manager.io + +################################################################################ +## Certificate Issuers ######################################################### +################################################################################ + +# +# Certificate Issuers +- name: cert-manager-issuers + needs: + - cert-manager/cert-manager + namespace: cert-manager + chart: incubator/raw + wait: true + timeout: 300 + atomic: true + cleanupOnFail: true + disableValidation: true + values: + - resources: + + - apiVersion: cert-manager.io/v1 + kind: ClusterIssuer + metadata: + name: letsencrypt-staging + spec: + acme: + server: https://acme-staging-v02.api.letsencrypt.org/directory + privateKeySecretRef: + name: letsencrypt-staging + solvers: + - selector: {} + dns01: + cloudDNS: + project: {{ env "CLOUDSDK_CORE_PROJECT" | default "project-not-found" }} + serviceAccountSecretRef: + name: clouddns-dns01-solver-svc-acct + key: key.json + + - apiVersion: cert-manager.io/v1 + kind: ClusterIssuer + metadata: + name: letsencrypt-prod + spec: + acme: + server: https://acme-v02.api.letsencrypt.org/directory + privateKeySecretRef: + name: letsencrypt-prod + solvers: + - selector: {} + dns01: + cloudDNS: + project: {{ env "CLOUDSDK_CORE_PROJECT" | default "project-not-found" }} + serviceAccountSecretRef: + name: clouddns-dns01-solver-svc-acct + key: key.json diff --git a/conf/helmfile.d/0100.redis.yaml b/conf/helmfile.d/0100.redis.yaml index e466f285..03dea12b 100644 --- a/conf/helmfile.d/0100.redis.yaml +++ b/conf/helmfile.d/0100.redis.yaml @@ -1,59 +1,61 @@ -helmDefaults: - wait: true - timeout: 600 - force: true +repositories: + - name: bitnami + url: https://charts.bitnami.com/bitnami + +releases: ################################################################################ ## Redis Data Store ############################################################ ################################################################################ -repositories: - # Stable repo of official helm charts - - name: stable - url: https://kubernetes-charts.storage.googleapis.com - -releases: - # Redis - - name: redis - labels: - chart: redis - component: database - chart: stable/redis - version: 10.5.7 - namespace: deepcell - values: - - resources: - requests: - cpu: 200m - memory: 256Mi - - usePassword: false - - cluster: - enabled: true - slaveCount: 3 - - sentinel: - enabled: true - - configmap: |- - # Enable AOF https://redis.io/topics/persistence#append-only-file - appendonly yes - # Disable RDB persistence, AOF persistence already enabled. - save "" - # Prevent fsync() from being called in the main process while a - # BGSAVE or BGREWRITEAOF is in progress. - # If you have latency problems turn this to "yes". Otherwise leave it as - # "no" that is the safest pick from the point of view of durability. - no-appendfsync-on-rewrite yes - - sysctlImage: - enabled: true - mountHostSys: true - command: - - /bin/sh - - -c - - |- - install_packages systemd - sysctl -w net.core.somaxconn=10000 - echo never > /host-sys/kernel/mm/transparent_hugepage/enabled +# +# References: +# - https://github.com/bitnami/charts/blob/master/bitnami/redis/values.yaml +# +- name: redis + labels: + chart: redis + component: database + chart: bitnami/redis + version: 10.5.7 + namespace: deepcell + wait: true + atomic: true + timeout: 600 + cleanupOnFail: true + values: + - resources: + requests: + cpu: 200m + memory: 256Mi + + usePassword: false + + cluster: + enabled: true + slaveCount: 3 + + sentinel: + enabled: true + + configmap: |- + # Enable AOF https://redis.io/topics/persistence#append-only-file + appendonly yes + # Disable RDB persistence, AOF persistence already enabled. + save "" + # Prevent fsync() from being called in the main process while a + # BGSAVE or BGREWRITEAOF is in progress. + # If you have latency problems turn this to "yes". Otherwise leave it as + # "no" that is the safest pick from the point of view of durability. + no-appendfsync-on-rewrite yes + + sysctlImage: + enabled: true + mountHostSys: true + command: + - /bin/sh + - -c + - |- + install_packages systemd procps + sysctl -w net.core.somaxconn=10000 + echo never > /host-sys/kernel/mm/transparent_hugepage/enabled diff --git a/conf/helmfile.d/0110.prometheus-redis-exporter.yaml b/conf/helmfile.d/0110.prometheus-redis-exporter.yaml index 415c6bcb..7e4e4603 100644 --- a/conf/helmfile.d/0110.prometheus-redis-exporter.yaml +++ b/conf/helmfile.d/0110.prometheus-redis-exporter.yaml @@ -1,40 +1,132 @@ -helmDefaults: - wait: true - timeout: 600 - force: true +repositories: + # Stable repo of official helm charts + - name: stable + url: https://charts.helm.sh/stable + # Kubernetes incubator repo of helm charts + - name: incubator + url: https://charts.helm.sh/incubator + +releases: ################################################################################ -## Redis Monitoring ############################################################ +## Exporter Script ConfigMap ################################################### ################################################################################ -repositories: - # Stable repo of official helm charts - - name: stable - url: https://kubernetes-charts.storage.googleapis.com +# +# ConfigMap for the custom script +- name: prometheus-redis-exporter-script + namespace: monitoring + chart: incubator/raw + wait: true + timeout: 180 + atomic: true + cleanupOnFail: true + disableValidation: true + values: + - resources: -releases: - - name: prometheus-redis-exporter - labels: - chart: prometheus-redis-exporter - component: monitoring - namespace: monitoring - chart: stable/prometheus-redis-exporter - version: 3.3.3 - namespace: monitoring - values: - - script: - configmap: prometheus-redis-exporter-script - keyname: script + - apiVersion: v1 + kind: ConfigMap + metadata: + name: prometheus-redis-exporter-script + data: + script: | + -- Based on https://github.com/soveran/rediscan.lua by GitHub user Soveran. + + local function get_queue_count(queue) + -- Find number of keys in the queue + local queue_size = redis.call("LLEN", queue) + + -- Get all processing queues + local queue_regex = "processing-" .. queue .. ":*" + + local count = 0 + + local cursor = "0" + local done = false + + repeat + + local result = redis.call("SCAN", cursor, "MATCH", queue_regex, "COUNT", 1000) + cursor = result[1] + + for i, key in ipairs(result[2]) do + -- How many keys are in each queue (should be 1) + local keys_in_queue = redis.call("LLEN", key) + count = count + keys_in_queue + end + + if cursor == "0" then + done = true + end - service: - annotations: - prometheus.io/path: "/metrics" - prometheus.io/port: "9121" - prometheus.io/scrape: "true" + until done + return count + queue_size + end + + -- Final table to output + local results = {} + + -- All Queues to Monitor: + local queues = {} + + queues[#queues+1] = "segmentation" + queues[#queues+1] = "tracking" + + for _,queue in ipairs(queues) do + local zip_queue = queue .. "-zip" + + local queue_count = get_queue_count(queue) + local zip_queue_count = get_queue_count(zip_queue) + + table.insert(results, queue) + table.insert(results, tostring(queue_count)) + + table.insert(results, zip_queue) + table.insert(results, tostring(zip_queue_count)) + + end + + return results + + +################################################################################ +## Prometheus Redis Exporter ################################################### +################################################################################ + +# +# References: +# - https://github.com/helm/charts/blob/master/stable/prometheus-redis-exporter/values.yaml +# +- name: prometheus-redis-exporter + chart: stable/prometheus-redis-exporter + namespace: monitoring + labels: + chart: prometheus-redis-exporter + component: monitoring + namespace: monitoring + needs: + - monitoring/prometheus-redis-exporter-script + version: 3.5.0 + wait: true + timeout: 120 + atomic: true + cleanupOnFail: true + values: + - script: + configmap: prometheus-redis-exporter-script + keyname: script + + service: annotations: prometheus.io/path: "/metrics" prometheus.io/port: "9121" prometheus.io/scrape: "true" - redisAddress: redis://redis.deepcell:6379 + annotations: + prometheus.io/path: "/metrics" + prometheus.io/port: "9121" + prometheus.io/scrape: "true" + + redisAddress: redis://redis.deepcell:6379 diff --git a/conf/helmfile.d/0200.bucket-monitor.yaml b/conf/helmfile.d/0200.bucket-monitor.yaml index 4a9949be..ce38dee1 100644 --- a/conf/helmfile.d/0200.bucket-monitor.yaml +++ b/conf/helmfile.d/0200.bucket-monitor.yaml @@ -1,8 +1,3 @@ -helmDefaults: - wait: true - timeout: 600 - force: true - releases: ################################################################################ @@ -11,24 +6,28 @@ releases: # # References: -# - [web address of Helm chart's YAML file] +# - https://github.com/vanvalenlab/kiosk-console/blob/master/conf/charts/bucket-monitor/values.yaml # -- name: "bucket-monitor" - namespace: "deepcell" +- name: bucket-monitor + namespace: deepcell labels: - chart: "bucket-monitor" - component: "bucket-monitor" - namespace: "deepcell" - vendor: "vanvalenlab" - default: "true" + chart: bucket-monitor + component: bucket-monitor + namespace: deepcell + vendor: vanvalenlab + default: true chart: '{{ env "CHARTS_PATH" | default "/conf/charts" }}/bucket-monitor' - version: "0.1.0" + version: 0.1.0 + wait: true + timeout: 300 + atomic: true + cleanupOnFail: true values: - replicas: 1 image: - repository: "vanvalenlab/kiosk-bucket-monitor" - tag: "0.3.0" + repository: vanvalenlab/kiosk-bucket-monitor + tag: 0.3.0 resources: requests: diff --git a/conf/helmfile.d/0210.autoscaler.yaml b/conf/helmfile.d/0210.autoscaler.yaml index 071ed5fc..a9d7abf8 100644 --- a/conf/helmfile.d/0210.autoscaler.yaml +++ b/conf/helmfile.d/0210.autoscaler.yaml @@ -1,8 +1,3 @@ -helmDefaults: - wait: true - timeout: 600 - force: true - releases: ################################################################################ @@ -11,7 +6,7 @@ releases: # # References: -# - [web address of Helm chart's YAML file] +# - https://github.com/vanvalenlab/kiosk-console/blob/master/conf/charts/autoscaler/values.yaml # - name: autoscaler namespace: deepcell @@ -23,6 +18,10 @@ releases: default: true chart: '{{ env "CHARTS_PATH" | default "/conf/charts" }}/autoscaler' version: 0.1.0 + wait: true + timeout: 300 + atomic: true + cleanupOnFail: true values: - replicas: 1 diff --git a/conf/helmfile.d/0220.redis-janitor.yaml b/conf/helmfile.d/0220.redis-janitor.yaml index 0b3f3b5c..a27c3583 100644 --- a/conf/helmfile.d/0220.redis-janitor.yaml +++ b/conf/helmfile.d/0220.redis-janitor.yaml @@ -1,17 +1,12 @@ -helmDefaults: - wait: true - timeout: 600 - force: true - releases: ################################################################################ -## Redis-Janitor ############################################################### +## Redis Janitor ############################################################### ################################################################################ # # References: -# - [web address of Helm chart's YAML file] +# - https://github.com/vanvalenlab/kiosk-console/blob/master/conf/charts/redis-janitor/values.yaml # - name: redis-janitor namespace: deepcell @@ -23,6 +18,10 @@ releases: default: true chart: '{{ env "CHARTS_PATH" | default "/conf/charts" }}/redis-janitor' version: 0.1.0 + wait: true + timeout: 300 + atomic: true + cleanupOnFail: true values: - replicas: 1 diff --git a/conf/helmfile.d/0230.segmentation-consumer.yaml b/conf/helmfile.d/0230.segmentation-consumer.yaml index 6ae641c6..6718520e 100644 --- a/conf/helmfile.d/0230.segmentation-consumer.yaml +++ b/conf/helmfile.d/0230.segmentation-consumer.yaml @@ -1,17 +1,12 @@ -helmDefaults: - wait: true - timeout: 600 - force: true - releases: ################################################################################ -## Redis-Consumer ############################################################## +## Segmentation Consumer ####################################################### ################################################################################ # # References: -# - https://github.com/vanvalenlab/kiosk-console/tree/master/conf/charts/redis-consumer +# - https://github.com/vanvalenlab/kiosk-console/blob/master/conf/charts/redis-consumer/values.yaml # - name: segmentation-consumer namespace: deepcell @@ -22,13 +17,17 @@ releases: vendor: vanvalenlab default: true chart: '{{ env "CHARTS_PATH" | default "/conf/charts" }}/redis-consumer' - version: 0.1.0 + version: 0.2.1 + wait: true + timeout: 300 + atomic: true + cleanupOnFail: true values: - replicas: 1 image: repository: vanvalenlab/kiosk-redis-consumer - tag: 0.6.1 + tag: 0.8.3 nameOverride: segmentation-consumer @@ -48,6 +47,20 @@ releases: nodeSelector: consumer: "yes" + hpa: + enabled: true + minReplicas: 1 + maxReplicas: {{ mul (int (env "GPU_NODE_MAX_SIZE" | default 1)) 150 }} + metrics: + - type: Object + object: + metricName: segmentation_consumer_key_ratio + target: + apiVersion: v1 + kind: Namespace + name: segmentation_consumer_key_ratio + targetValue: .15 + env: DEBUG: "true" INTERVAL: 1 diff --git a/conf/helmfile.d/0240.segmentation-zip-consumer.yaml b/conf/helmfile.d/0240.segmentation-zip-consumer.yaml index 4535f366..20ca3e6d 100644 --- a/conf/helmfile.d/0240.segmentation-zip-consumer.yaml +++ b/conf/helmfile.d/0240.segmentation-zip-consumer.yaml @@ -1,17 +1,12 @@ -helmDefaults: - wait: true - timeout: 600 - force: true - releases: ################################################################################ -## Zip-Consumer ################################################################ +## Segmentation Zip Consumer ################################################### ################################################################################ # # References: -# - https://github.com/vanvalenlab/kiosk-console/tree/master/conf/charts/redis-consumer +# - https://github.com/vanvalenlab/kiosk-console/blob/master/conf/charts/redis-consumer/values.yaml # - name: segmentation-zip-consumer namespace: deepcell @@ -22,13 +17,17 @@ releases: vendor: vanvalenlab default: true chart: '{{ env "CHARTS_PATH" | default "/conf/charts" }}/redis-consumer' - version: 0.1.0 + version: 0.2.1 + wait: true + timeout: 300 + atomic: true + cleanupOnFail: true values: - replicas: 1 image: repository: vanvalenlab/kiosk-redis-consumer - tag: 0.6.1 + tag: 0.8.3 nameOverride: segmentation-zip-consumer @@ -48,6 +47,20 @@ releases: nodeSelector: consumer: "yes" + hpa: + enabled: true + minReplicas: 1 + maxReplicas: {{ mul (int (env "GPU_NODE_MAX_SIZE" | default 1)) 100 }} + metrics: + - type: Object + object: + metricName: segmentation_zip_consumer_key_ratio + target: + apiVersion: v1 + kind: Namespace + name: segmentation_zip_consumer_key_ratio + targetValue: 2 + env: QUEUE: "segmentation" REDIS_HOST: "redis" diff --git a/conf/helmfile.d/0250.tracking-consumer.yaml b/conf/helmfile.d/0250.tracking-consumer.yaml index a1a89571..373a3ba5 100644 --- a/conf/helmfile.d/0250.tracking-consumer.yaml +++ b/conf/helmfile.d/0250.tracking-consumer.yaml @@ -1,17 +1,12 @@ -helmDefaults: - wait: true - timeout: 600 - force: true - releases: ################################################################################ -## Tracking-Consumer ################################################################ +## Tracking Consumer ########################################################### ################################################################################ # # References: -# - https://github.com/vanvalenlab/kiosk-console/tree/master/conf/charts/redis-consumer +# - https://github.com/vanvalenlab/kiosk-console/blob/master/conf/charts/redis-consumer/values.yaml # - name: tracking-consumer namespace: deepcell @@ -22,13 +17,17 @@ releases: vendor: vanvalenlab default: true chart: '{{ env "CHARTS_PATH" | default "/conf/charts" }}/redis-consumer' - version: 0.1.0 + version: 0.2.1 + wait: true + timeout: 300 + atomic: true + cleanupOnFail: true values: - replicas: 1 image: repository: vanvalenlab/kiosk-redis-consumer - tag: 0.6.1 + tag: 0.8.3 nameOverride: tracking-consumer @@ -48,6 +47,20 @@ releases: nodeSelector: consumer: "yes" + hpa: + enabled: true + minReplicas: 1 + maxReplicas: {{ mul (int (env "GPU_NODE_MAX_SIZE" | default 1)) 50 }} + metrics: + - type: Object + object: + metricName: tracking_consumer_key_ratio + target: + apiVersion: v1 + kind: Namespace + name: tracking_consumer_key_ratio + targetValue: 1 + env: DEBUG: "true" INTERVAL: 1 diff --git a/conf/helmfile.d/0300.frontend.yaml b/conf/helmfile.d/0300.frontend.yaml index 4aa9a3a5..db00bf65 100644 --- a/conf/helmfile.d/0300.frontend.yaml +++ b/conf/helmfile.d/0300.frontend.yaml @@ -1,8 +1,3 @@ -helmDefaults: - wait: true - timeout: 600 - force: true - releases: ################################################################################ @@ -11,7 +6,7 @@ releases: # # References: -# - [web address of Helm chart's YAML file] +# - https://github.com/vanvalenlab/kiosk-console/blob/master/conf/charts/frontend/values.yaml # - name: frontend namespace: deepcell @@ -22,13 +17,17 @@ releases: vendor: vanvalenlab default: true chart: '{{ env "CHARTS_PATH" | default "/conf/charts" }}/frontend' - version: 0.1.0 + version: 0.2.1 + wait: true + timeout: 300 + atomic: true + cleanupOnFail: true values: - replicas: 1 image: repository: vanvalenlab/kiosk-frontend - tag: 0.4.2 + tag: 0.6.0 resources: requests: @@ -46,7 +45,7 @@ releases: httpTargetPort: 8080 externalHttpPort: 8080 - httpsIngressEnabled: false + httpsIngressEnabled: true internalHttpsPort: 8443 httpsTargetPort: 8443 externalHttpsPort: 8443 @@ -56,9 +55,36 @@ releases: path: / annotations: kubernetes.io/ingress.class: "nginx" - nginx.ingress.kubernetes.io/rewrite-target: / nginx.ingress.kubernetes.io/ssl-redirect: "false" nginx.ingress.kubernetes.io/proxy-body-size: "1g" + {{ if ne (env "CERTIFICATE_MANAGER_ENABLED" | default "") "" }} + kubernetes.io/tls-acme: "true" + nginx.ingress.kubernetes.io/auth-tls-secret: "deepcell/tls-cert" + # Use "letsencrypt-prod" for production. + cert-manager.io/cluster-issuer: {{ env "CERTIFICATE_MANAGER_CLUSTER_ISSUER" | default "letsecrypt-staging" }} + {{ end }} + + # Use $DNS_DOMAIN_NAME in production + {{ if ne (env "CERTIFICATE_MANAGER_ENABLED" | default "") "" }} + hosts: + - {{ env "DNS_DOMAIN_NAME" | default "deepcell.org" }} + - www.{{ env "DNS_DOMAIN_NAME" | default "deepcell.org" }} + tls: + - hosts: + - {{ env "DNS_DOMAIN_NAME" | default "deepcell.org" }} + - www.{{ env "DNS_DOMAIN_NAME" | default "deepcell.org" }} + secretName: tls-cert + {{ end }} + + hpa: + enabled: true + minReplicas: 1 + maxReplicas: {{ mul (int (env "GPU_NODE_MAX_SIZE" | default 1)) 10 }} + metrics: + - type: Resource + resource: + name: cpu + targetAverageUtilization: 80 env: PORT: 8080 @@ -67,13 +93,15 @@ releases: REDIS_SENTINEL: "true" MODEL_PREFIX: "models/" AWS_REGION: '{{ env "AWS_REGION" | default "us-east-1" }}' - CLOUD_PROVIDER: '{{ env "CLOUD_PROVIDER" | default "aws" }}' UPLOAD_PREFIX: "uploads/" JOB_TYPES: "segmentation,tracking" secrets: AWS_ACCESS_KEY_ID: '{{ env "AWS_ACCESS_KEY_ID" | default "NA" }}' AWS_SECRET_ACCESS_KEY: '{{ env "AWS_SECRET_ACCESS_KEY" | default "NA" }}' - AWS_S3_BUCKET: '{{ env "AWS_S3_BUCKET" | default "NA" }}' - GCLOUD_STORAGE_BUCKET: '{{ env "CLOUDSDK_BUCKET" | default "NA" }}' - GCLOUD_PROJECT_ID: '{{ env "PROJECT" | default "NA" }}' + GCLOUD_PROJECT_ID: '{{ env "CLOUDSDK_CORE_PROJECT" | default "NA" }}' +{{ if eq (env "CLOUD_PROVIDER" | default "aws") "aws" }} + STORAGE_BUCKET: 's3://{{ env "AWS_S3_BUCKET" | default "NA" }}' +{{ else }} + STORAGE_BUCKET: 'gs://{{ env "CLOUDSDK_BUCKET" | default "NA" }}' +{{ end }} diff --git a/conf/helmfile.d/0310.tf-serving.yaml b/conf/helmfile.d/0310.tf-serving.yaml index 77ed967d..344957a9 100644 --- a/conf/helmfile.d/0310.tf-serving.yaml +++ b/conf/helmfile.d/0310.tf-serving.yaml @@ -1,8 +1,3 @@ -helmDefaults: - wait: true - timeout: 600 - force: true - releases: ################################################################################ @@ -11,7 +6,7 @@ releases: # # References: -# - [web address of Helm chart's YAML file] +# - https://github.com/vanvalenlab/kiosk-console/blob/master/conf/charts/tf-serving/values.yaml # - name: tf-serving namespace: deepcell @@ -22,13 +17,27 @@ releases: vendor: vanvalenlab default: true chart: '{{ env "CHARTS_PATH" | default "/conf/charts" }}/tf-serving' - version: 0.1.0 + version: 0.4.0 + wait: true + timeout: 300 + atomic: true + cleanupOnFail: true values: - replicas: 0 image: repository: vanvalenlab/kiosk-tf-serving - tag: 0.2.1 + tag: 0.4.0 + + configWriter: + mountedVolume: + name: configdir + path: /config + + image: + repository: vanvalenlab/kiosk-tf-serving-config-writer + tag: 0.4.0 + pullPolicy: IfNotPresent resources: requests: @@ -67,6 +76,20 @@ releases: prometheus.io/port: "8501" prometheus.io/scrape: "true" + hpa: + enabled: true + minReplicas: 1 + maxReplicas: {{ int (env "GPU_NODE_MAX_SIZE" | default 1) }} + metrics: + - type: Object + object: + metricName: tf_serving_gpu_usage + target: + apiVersion: v1 + kind: Namespace + name: tf_serving_gpu_usage + targetValue: 70 + annotations: prometheus.io/path: /monitoring/prometheus/metrics prometheus.io/port: "8501" @@ -84,24 +107,26 @@ releases: PORT: 8500 REST_API_PORT: 8501 REST_API_TIMEOUT: 30000 - MODEL_CONFIG_FILE: /kiosk/tf-serving/models.conf - BATCHING_CONFIG_FILE: /kiosk/tf-serving/batching_config.txt + MODEL_CONFIG_FILE: /config/models.conf + BATCHING_CONFIG_FILE: /config/batching_config.txt ENABLE_BATCHING: "true" MAX_BATCH_SIZE: 128 BATCH_TIMEOUT_MICROS: 0 MAX_ENQUEUED_BATCHES: 512 GRPC_CHANNEL_ARGS: "" MODEL_PREFIX: models - CLOUD_PROVIDER: '{{ env "CLOUD_PROVIDER" | default "aws" }}' TF_CPP_MIN_LOG_LEVEL: 0 TF_SESSION_PARALLELISM: 0 - MONITORING_CONFIG_FILE: /kiosk/tf-serving/monitoring_config.txt + MONITORING_CONFIG_FILE: /config/monitoring_config.txt PROMETHEUS_MONITORING_ENABLED: "true" PROMETHEUS_MONITORING_PATH: /monitoring/prometheus/metrics secrets: AWS_ACCESS_KEY_ID: '{{ env "AWS_ACCESS_KEY_ID" | default "NA" }}' AWS_SECRET_ACCESS_KEY: '{{ env "AWS_SECRET_ACCESS_KEY" | default "NA" }}' - AWS_S3_BUCKET: '{{ env "AWS_S3_BUCKET" | default "NA" }}' - # GCLOUD_STORAGE_BUCKET: '{{ env "CLOUDSDK_BUCKET" | default "NA" }}' - GCLOUD_STORAGE_BUCKET: deepcell-models + # Uncomment to override the default models. +{{ if eq (env "CLOUD_PROVIDER" | default "aws") "aws" }} + # STORAGE_BUCKET: 's3://{{ env "AWS_S3_BUCKET" | default "NA" }}' +{{ else }} + # STORAGE_BUCKET: 'gs://{{ env "CLOUDSDK_BUCKET" | default "NA" }}' +{{ end }} diff --git a/conf/helmfile.d/0400.benchmarking.yaml b/conf/helmfile.d/0400.benchmarking.yaml index f56d8c6c..f1866bd3 100644 --- a/conf/helmfile.d/0400.benchmarking.yaml +++ b/conf/helmfile.d/0400.benchmarking.yaml @@ -1,8 +1,3 @@ -helmDefaults: - wait: true - timeout: 600 - force: true - releases: ################################################################################ @@ -11,7 +6,7 @@ releases: # # References: -# - [web address of Helm chart's YAML file] +# - https://github.com/vanvalenlab/kiosk-console/blob/master/conf/charts/benchmarking/values.yaml # - name: benchmarking namespace: deepcell @@ -23,6 +18,10 @@ releases: default: true chart: '{{ env "CHARTS_PATH" | default "/conf/charts" }}/benchmarking' version: 0.1.0 + wait: true + timeout: 300 + atomic: true + cleanupOnFail: true values: - replicas: 0 diff --git a/conf/helmfile.d/0600.prometheus-operator.yaml b/conf/helmfile.d/0600.prometheus-operator.yaml index 389c555d..2c511170 100644 --- a/conf/helmfile.d/0600.prometheus-operator.yaml +++ b/conf/helmfile.d/0600.prometheus-operator.yaml @@ -1,7 +1,7 @@ -helmDefaults: - wait: true - timeout: 600 - force: true +repositories: + # Stable repo of official helm charts + - name: stable + url: https://charts.helm.sh/stable releases: @@ -25,96 +25,340 @@ releases: vendor: coreos default: true chart: stable/prometheus-operator - version: 8.12.3 + version: 9.3.1 wait: true + timeout: 600 + atomic: true + cleanupOnFail: true values: # A list of all possible values can be found: # https://github.com/helm/charts/blob/master/stable/prometheus-operator/values.yaml - - additionalPrometheusRulesMap: + - additionalPrometheusRules: - name: custom-prometheus-rules groups: - - name: tf-serving-metrics - rules: - - record: tf_serving_gpu_usage - expr: |- - avg( - container_accelerator_duty_cycle{container_name="tf-serving"} + # alerts + - name: prometheus-alerts + rules: + - alert: PrometheusAllTargetsMissing + expr: count by (job) (up) == 0 + for: 5m + labels: + severity: critical + annotations: + summary: |- + {{`Prometheus all targets missing`}} + - alert: PrometheusTooManyRestarts + expr: changes(process_start_time_seconds{job=~"prometheus|pushgateway|alertmanager"}[15m]) > 2 + for: 5m + labels: + severity: warning + annotations: + summary: |- + {{`Prometheus job {{ $labels.job }} restarted {{ $value }} times`}} + - alert: PrometheusRuleEvaluationSlow + expr: prometheus_rule_group_last_duration_seconds > prometheus_rule_group_interval_seconds + for: 5m + labels: + severity: warning + annotations: + summary: |- + {{`Prometheus rule evaluation took more time than the scheduled interval. I indicates a slower storage backend access or too complex query. (rule_group {{ $labels.rule_group }})`}} + - alert: PrometheusNotificationsBacklog + expr: min_over_time(prometheus_notifications_queue_length[10m]) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: |- + {{`The Prometheus notification queue has not been empty for 10 minutes`}} + - alert: PrometheusNotConnectedToAlertmanager + expr: prometheus_notifications_alertmanagers_discovered < 1 + for: 5m + labels: + severity: critical + annotations: + summary: |- + {{`{{ $labels.pod }} cannot connect the alertmanager (pod {{ $labels.pod }})`}} + - alert: PrometheusTemplateTextExpansionFailures + expr: increase(prometheus_template_text_expansion_failures_total[3m]) > 0 + for: 5m + labels: + severity: critical + annotations: + summary: |- + {{`{{ $labels.pod }} encountered {{ $value }} template text expansion failures`}} + - alert: PrometheusAlertmanagerNotificationFailing + expr: rate(alertmanager_notifications_failed_total[1m]) > 0 + for: 5m + labels: + severity: critical + annotations: + summary: |- + {{`{{ $labels.service}} is failing to send {{ $labels.integration }} notifications`}} + - name: redis-alerts + rules: + - alert: RedisDown + expr: redis_up == 0 + for: 5m + labels: + severity: error + annotations: + summary: |- + {{`Redis is down (instance {{ $labels.instance }})`}} + - alert: RedisMissingMaster + expr: count(redis_instance_info{role="master"}) == 0 + for: 5m + labels: + severity: error + annotations: + summary: |- + {{`Redis missing master`}} + - alert: RedisTooManyMasters + expr: count(redis_instance_info{role="master"}) > 1 + for: 5m + labels: + severity: critical + annotations: + summary: |- + {{`Redis has {{ $value }} masters.`}} + - alert: RedisDisconnectedSlaves + expr: count without (instance, job) (redis_connected_slaves) - sum without (instance, job) (redis_connected_slaves) - 1 > 1 + for: 5m + labels: + severity: critical + annotations: + summary: |- + {{`Redis has {{ $value }} disconnected slaves.`}} + # TODO: Flapping alert keeps going off, usually with values from 4 - 10. + # - alert: RedisClusterFlapping + # expr: changes(redis_connected_slaves[5m]) > 3 + # for: 5m + # labels: + # severity: critical + # annotations: + # summary: |- + # {{`Redis replica connection flapping. Redis slaves have connected to {{ $value }} times in last 5 minutes (instance {{ $labels.instance }})`}} + - alert: RedisTooManyConnections + expr: redis_connected_clients > 100 + for: 5m + labels: + severity: warning + annotations: + summary: |- + {{`Redis instance has too many ({{ $value }}) connections`}} + - alert: RedisRejectedConnections + expr: increase(redis_rejected_connections_total[1m]) > 0 + for: 5m + labels: + severity: critical + annotations: + summary: |- + {{`Redis rejected {{ $value }} connections`}} + - name: kubernetes-alerts + rules: + - alert: KubernetesPodNotHealthy + expr: min_over_time(sum by (namespace, pod) (kube_pod_status_phase{phase=~"Pending|Unknown|Failed"})[1h:]) > 0 + for: 5m + labels: + severity: critical + annotations: + summary: |- + {{`Kubernetes Pod {{ $labels.pod }} in namespace {{ $labels.namespace }} has been unhealthy for more than an hour`}} + - alert: KubernetesPodCrashLooping + expr: rate(kube_pod_container_status_restarts_total[15m]) * 60 * 5 > 5 + for: 5m + labels: + severity: warning + annotations: + summary: |- + {{`Kubernetes pod {{ $labels.pod }} in namespace {{ $labels.namespace }} is crash looping`}} + - alert: KubernetesPersistentvolumeError + expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0 + for: 5m + labels: + severity: critical + annotations: + summary: |- + {{`Kubernetes PersistentVolume {{ $labels.persistentvolume }} in state {{ $labels.phase }}`}} + - alert: KubernetesNodeReady + expr: kube_node_status_condition{condition="Ready",status="true"} == 0 + for: 5m + labels: + severity: critical + annotations: + summary: |- + {{`Kubernetes Node {{ $labels.node }} has condition {{ $labels.condition }}`}} + - name: host-alerts + rules: + - alert: HostOutOfMemory + expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10 + for: 5m + labels: + severity: warning + annotations: + summary: |- + {{`Host out of memory for (pod {{ $labels.pod }}, instance {{ $labels.instance }})`}} + - alert: HostOomKillDetected + expr: increase(node_vmstat_oom_kill[5m]) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: |- + {{`Host OOM kill detected (pod {{ $labels.pod }}, instance {{ $labels.instance }})`}} + # recording rules + - name: tf-serving-metrics + rules: + - record: tf_serving_gpu_usage + expr: |- + avg( + container_accelerator_duty_cycle{container_name="tf-serving"} + ) or vector(0) + labels: + deployment: tf-serving + namespace: deepcell + - record: tf_serving_up + expr: |- + max( + clamp_max( + kube_deployment_status_replicas_available{deployment="tf-serving"}, + 1 ) or vector(0) - labels: - deployment: tf-serving - namespace: deepcell - - record: tf_serving_up - expr: |- - max( - clamp_max( - kube_deployment_status_replicas_available{deployment="tf-serving"}, - 1 - ) or vector(0) - ) - labels: - deployment: tf-serving - namespace: deepcell - - name: consumer-metrics - rules: - - record: consumers_per_gpu - expr: |- - kube_deployment_status_replicas_available{deployment=~".*-consumer"} - / on() group_left - max( - kube_deployment_status_replicas_available{deployment="tf-serving"} - or vector(1) - ) - labels: - namespace: deepcell - - record: consumer_key_ratio - expr: |- - redis_script_values - / on(deployment) - kube_deployment_spec_replicas - labels: - namespace: deepcell - - record: segmentation_consumer_key_ratio - # COMPLICATED scaling metric that prevents too many consumers - # per GPU. If too many consumers, scale down slightly. - # Structural outline follows: - # ( - # min(1 - GPU, keys/consumers) * !is_too_many_consumers - # + - # .75 * target * is_too_many_consumers) - # ) * is_tf_up - expr: |- + ) + labels: + deployment: tf-serving + namespace: deepcell + - name: consumer-metrics + rules: + - record: consumers_per_gpu + expr: |- + kube_deployment_status_replicas_available{deployment=~".*-consumer"} + / on() group_left + max( + kube_deployment_status_replicas_available{deployment="tf-serving"} + or vector(1) + ) + labels: + namespace: deepcell + - record: consumer_key_ratio + expr: |- + redis_script_values + / on(deployment) + kube_deployment_spec_replicas + labels: + namespace: deepcell + - record: segmentation_consumer_key_ratio + # COMPLICATED scaling metric that prevents too many consumers + # per GPU. If too many consumers, scale down slightly. + # Structural outline follows: + # ( + # min(1 - GPU, keys/consumers) * !is_too_many_consumers + # + + # .75 * target * is_too_many_consumers) + # ) * is_tf_up + expr: |- + ( ( - ( - min( - 1 - tf_serving_gpu_usage / 100 - < on(namespace) + min( + 1 - tf_serving_gpu_usage / 100 + < on(namespace) + consumer_key_ratio{deployment="segmentation-consumer"} / 100 + or + clamp_max( consumer_key_ratio{deployment="segmentation-consumer"} / 100 - or - clamp_max( - consumer_key_ratio{deployment="segmentation-consumer"} / 100 - , 1) - ) * - scalar(consumers_per_gpu{deployment="segmentation-consumer"} <= bool 150) - ) + ( - scalar(consumers_per_gpu{deployment="segmentation-consumer"} > bool 150) - ) * .75 * .15 - ) * scalar(tf_serving_up > bool 0) - labels: - deployment: segmentation-consumer - namespace: deepcell - - record: segmentation_zip_consumer_key_ratio - expr: |- - consumer_key_ratio{deployment="segmentation-zip-consumer"} - labels: - deployment: zip-consumer - namespace: deepcell - - record: tracking_consumer_key_ratio - expr: |- - consumer_key_ratio{deployment="tracking-consumer"} - * on() tf_serving_up - labels: - deployment: tracking-consumer - namespace: deepcell + , 1) + ) * + scalar(consumers_per_gpu{deployment="segmentation-consumer"} <= bool 150) + ) + ( + scalar(consumers_per_gpu{deployment="segmentation-consumer"} > bool 150) + ) * .75 * .15 + ) * scalar(tf_serving_up > bool 0) + labels: + deployment: segmentation-consumer + namespace: deepcell + - record: segmentation_zip_consumer_key_ratio + expr: |- + consumer_key_ratio{deployment="segmentation-zip-consumer"} + labels: + deployment: zip-consumer + namespace: deepcell + - record: tracking_consumer_key_ratio + expr: |- + consumer_key_ratio{deployment="tracking-consumer"} + * on() tf_serving_up + labels: + deployment: tracking-consumer + namespace: deepcell + + ## Configuration for alertmanager + ## ref: https://prometheus.io/docs/alerting/alertmanager/ + ## + alertmanager: + enabled: {{ env "ALERTMANAGER_INSTALLED" | default "true" }} + config: + global: + resolve_timeout: 5m + route: + group_by: + - 'alertname' + - 'namespace' + group_wait: 30s + group_interval: 5m + repeat_interval: 12h + receiver: 'general' + routes: + - match: + alertname: Watchdog + receiver: 'null' + - match: + # AggregatedAPIDown fires constantly for kube < 1.18 + # https://github.com/helm/charts/issues/22278 + alertname: AggregatedAPIDown + receiver: 'null' + - match: + # CPUThrottlingHigh is too sensitive, could block all INFO level instead. + # https://github.com/kubernetes-monitoring/kubernetes-mixin/issues/108 + alertname: CPUThrottlingHigh + receiver: 'null' + + receivers: + - name: 'null' + - name: 'general' + {{ if not (env "KUBE_PROMETHEUS_ALERT_MANAGER_SLACK_WEBHOOK_URL" | empty ) }} + slack_configs: + - api_url: '{{ env "KUBE_PROMETHEUS_ALERT_MANAGER_SLACK_WEBHOOK_URL" }}' + channel: '{{ env "KUBE_PROMETHEUS_ALERT_MANAGER_SLACK_CHANNEL" | default "alerts" }}' + send_resolved: true + username: '{{`{{ template "slack.default.username" . }}`}}' + color: '{{`{{ if eq .Status "firing" }}danger{{ else }}good{{ end }}`}}' + # title: '{{`{{ template "slack.custom.title" . }}`}}' + title_link: '{{`{{ template "slack.default.titlelink" . }}`}}' + pretext: '{{`{{ .CommonAnnotations.summary }}`}}' + fallback: '{{`{{ template "slack.default.fallback" . }}`}}' + icon_emoji: '{{`{{ template "slack.default.iconemoji" . }}`}}' + icon_url: https://avatars3.githubusercontent.com/u/3380462 + # text: '{{`{{ template "slack.custom.text" . }}`}}' + title: |- + {{`[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }} + {{- if gt (len .CommonLabels) (len .GroupLabels) -}} + {{" "}}( + {{- with .CommonLabels.Remove .GroupLabels.Names }} + {{- range $index, $label := .SortedPairs -}} + {{ if $index }}, {{ end }} + {{- $label.Name }}="{{ $label.Value -}}" + {{- end }} + {{- end -}} + ) + {{- end }}`}} + text: |- + {{`{{ with index .Alerts 0 -}} + :chart_with_upwards_trend: *<{{ .GeneratorURL }}|Graph>*{{- if .Annotations.runbook_url }} :notebook: *<{{ .Annotations.runbook_url }}|Runbook>*{{ end }}{{- if .Labels.severity }} `}}`{{`{{ .Labels.severity }}`}}`{{`{{ end }} + {{ end }} + {{ range .Alerts -}} + {{- if .Annotations.message }}{{ .Annotations.message }}{{ end }}{{ if .Annotations.summary }}{{ .Annotations.summary }} + {{ end }} + {{ end }}`}} + {{ end }} ## Manages Prometheus and Alertmanager components ## @@ -283,3 +527,23 @@ releases: regex: apiserver_response_size_buckets replacement: $1 action: drop + + ## Component scraping kube scheduler + ## + kubeScheduler: + enabled: false + + ## Component scraping the kube controller manager + ## + kubeControllerManager: + enabled: false + + ## Component scraping coreDns. Use either this or kubeDns + ## + coreDns: + enabled: false + + ## Component scraping kubeDns. Use either this or coreDns + ## + kubeDns: + enabled: true diff --git a/conf/helmfile.d/0610.prometheus-adapter.yaml b/conf/helmfile.d/0610.prometheus-adapter.yaml index 8001f144..a2f9454b 100644 --- a/conf/helmfile.d/0610.prometheus-adapter.yaml +++ b/conf/helmfile.d/0610.prometheus-adapter.yaml @@ -1,12 +1,12 @@ -helmDefaults: - wait: true - timeout: 600 - force: true +repositories: + # Stable repo of official helm charts + - name: stable + url: https://charts.helm.sh/stable releases: ################################################################################ -## prometheus-adapter ## +## Prometheus Adapter ## ## exports data for the cutom metrics API ## ################################################################################ @@ -25,8 +25,11 @@ releases: vendor: coreos default: true chart: stable/prometheus-adapter - version: 2.1.3 + version: 2.5.0 wait: true + timeout: 600 + atomic: true + cleanupOnFail: true values: - logLevel: 4 diff --git a/conf/helmfile.d/9999.openvpn.yaml b/conf/helmfile.d/9999.openvpn.yaml index 1655a993..e3f2c195 100644 --- a/conf/helmfile.d/9999.openvpn.yaml +++ b/conf/helmfile.d/9999.openvpn.yaml @@ -1,7 +1,7 @@ -helmDefaults: - wait: true - timeout: 600 - force: true +repositories: + # Stable repo of official helm charts + - name: stable + url: https://charts.helm.sh/stable releases: @@ -11,7 +11,7 @@ releases: # # References: -# - https://github.com/helm/charts/stable/openvpn +# - https://github.com/helm/charts/stable/openvpn/values.yaml # - name: openvpn namespace: kube-system @@ -22,4 +22,8 @@ releases: vendor: openvpn default: true chart: stable/openvpn - version: 4.2.1 + version: 4.2.3 + wait: true + timeout: 600 + atomic: true + cleanupOnFail: true diff --git a/conf/tasks/Makefile.gke b/conf/tasks/Makefile.gke index 554bd72b..abad234b 100644 --- a/conf/tasks/Makefile.gke +++ b/conf/tasks/Makefile.gke @@ -186,14 +186,6 @@ gke/destroy/node-pools: @echo " " @echo " " -## Destroy Orphaned GKE persistent disks -gke/destroy/pds: - @echo "Destroying orphaned persistent disks." - @gke-pd-destruction.sh - @echo "Orphaned persistent disks destroyed." - @echo " " - @echo " " - # https://cloud.google.com/storage/docs/access-control/iam-roles ## Create Service Account used by deepcell gke/create/service-account: @@ -201,6 +193,12 @@ gke/create/service-account: @gcloud iam service-accounts create $(CLOUDSDK_CONTAINER_CLUSTER) --display-name "Deepcell" || \ echo "No need to create service account; it probably already exists." @gcloud projects add-iam-policy-binding $(CLOUDSDK_CORE_PROJECT) --member serviceAccount:$(GCP_SERVICE_ACCOUNT) --role roles/storage.admin +ifneq "" "${CERTIFICATE_MANAGER_ENABLED}" + @gcloud projects add-iam-policy-binding $(CLOUDSDK_CORE_PROJECT) --member serviceAccount:$(GCP_SERVICE_ACCOUNT) --role roles/dns.admin + # @gcloud iam service-accounts add-iam-policy-binding $(GCP_SERVICE_ACCOUNT) \ + # --role roles/iam.workloadIdentityUser \ + # --member "serviceAccount:$(CLOUDSDK_CORE_PROJECT).svc.id.goog[cert-manager/cert-manager]" +endif @echo "GKE service account creation complete." @echo " " @echo " " @@ -209,16 +207,41 @@ gke/create/service-account: gke/destroy/service-account: @echo "Destroying GKE service-account..." @gcloud projects remove-iam-policy-binding $(CLOUDSDK_CORE_PROJECT) --member serviceAccount:$(GCP_SERVICE_ACCOUNT) --role roles/storage.admin +ifneq "" "${CERTIFICATE_MANAGER_ENABLED}" + @gcloud projects remove-iam-policy-binding $(CLOUDSDK_CORE_PROJECT) --member serviceAccount:$(GCP_SERVICE_ACCOUNT) --role roles/dns.admin +endif @-gcloud iam service-accounts delete $(GCP_SERVICE_ACCOUNT) --quiet @echo "GKE service-account destruction finished." @echo " " @echo " " +## Create Certificate Manager Secret +gke/create/certificate-manager-secret: +ifneq "" "${CERTIFICATE_MANAGER_ENABLED}" + @gcloud iam service-accounts keys create key.json \ + --iam-account $(GCP_SERVICE_ACCOUNT) + -@kubectl create namespace cert-manager + @kubectl -n cert-manager create secret generic clouddns-dns01-solver-svc-acct --from-file=key.json +else + @echo "Certificate Manager is Disabled" +endif + +## Remove Certifiate Manager Secret Key +gke/destroy/certifiate-manager-secret: KEY_ID = $(shell sh -c "cat key.json | jq '.private_key_id' -r" ) +gke/destroy/certifiate-manager-secret: +ifneq "" "${CERTIFICATE_MANAGER_ENABLED}" + @gcloud iam service-accounts keys delete $(KEY_ID) --quiet \ + --iam-account $(GCP_SERVICE_ACCOUNT) || \ + echo "Could not remove key from IAM account." +else + @echo "Certificate Manager is Disabled" +endif + ## Create bucket used by deepcell gke/create/bucket: - @echo "Creating Google Cloud Storage Bucket ${CLOUDSDK_CORE_PROJECT}..." + @echo "Creating Google Cloud Storage Bucket ${CLOUDSDK_BUCKET}..." @gsutil mb -p $(CLOUDSDK_CORE_PROJECT) gs://$(CLOUDSDK_BUCKET) \ - || echo "Bucket ${CLOUDSDK_CODE_PROJECT} already exists. No need to create that bucket." + || echo "Bucket ${CLOUDSDK_BUCKET} already exists. No need to create that bucket." @-gsutil acl ch -u $(GCP_SERVICE_ACCOUNT):O gs://$(CLOUDSDK_BUCKET) @echo "Google Cloud Storage Bucket creation finished." @echo " " @@ -254,6 +277,7 @@ gke/deploy/nvidia: ## Create cluster resources, after authentication gke/create/resources: \ gke/create/cluster \ + gke/create/certificate-manager-secret \ gke/create/node-pools \ gke/create/elk-node-pools \ gke/create/bucket \ @@ -268,15 +292,13 @@ gke/create/all: \ ## Destroy Cluster gke/destroy/all: \ - gke/destroy/node-pools \ gke/destroy/cluster \ - gke/destroy/service-account \ - gke/destroy/pds + gke/destroy/certifiate-manager-secret \ + gke/destroy/service-account @echo "GKE cluster destroyed" @exit 0 - # https://cloud.google.com/storage/docs/access-control/iam-roles # Currently, using Editor and Kubernetes Engine Admin roles for the testing-ci service account. # It might be possible to replace Editor with someting more specific, but more research would be needed. @@ -297,8 +319,7 @@ gke/test/create/all: \ ## Destroy Cluster gke/test/destroy/all: \ - gke/destroy/node-pools \ gke/destroy/cluster \ - gke/destroy/pds + gke/destroy/certifiate-manager-secret @echo "GKE cluster destroyed" @exit 0 diff --git a/conf/tasks/Makefile.helmfile b/conf/tasks/Makefile.helmfile index 6398ce85..7a76a666 100644 --- a/conf/tasks/Makefile.helmfile +++ b/conf/tasks/Makefile.helmfile @@ -8,8 +8,7 @@ helmfile/create/all: @if [ -n "${ELK_DEPLOYMENT_TOGGLE}" ]; then\ cp /conf/ELK_helmfiles/* /conf/helmfile.d;\ fi - @gke-helmfile-deployment.sh - @echo "Helmfile deployment finished." + @deploy-helmfiles.sh @echo " " @echo " " @@ -22,7 +21,7 @@ helmfile/create/elk: helmfile/destroy/all: @echo " " @echo "Destroying all Kubernetes resources using helmfiles..." - @helm ls --all --short | xargs helm delete + -@helm ls -a -A | awk 'NR > 1 { print $$2, $$1 }' | sort -r | xargs -n2 helm delete -n $0 $1 @echo "Helmfile destruction finished." @echo " " @echo " " diff --git a/conf/tasks/Makefile.kubectl b/conf/tasks/Makefile.kubectl index 86985f80..315b73a2 100644 --- a/conf/tasks/Makefile.kubectl +++ b/conf/tasks/Makefile.kubectl @@ -8,11 +8,6 @@ kubectl/create/rbac: @echo " " @echo " " -## Deploy ConfigMap with prometheus-redis-exporter script. -kubectl/create/prometheus-redis-exporter-script: - -@kubectl create namespace monitoring 2>/dev/null || true - gomplate -f addons/redis-exporter-script.yaml | kubectl apply -f - - ## Provision cluster autoscaler kubectl/create/autoscaler: gomplate -f addons/cluster-autoscaler.yaml | kubectl apply -f - @@ -31,13 +26,19 @@ kubectl/create/all: \ ## Display the cluster IP or URL kubectl/display/ip: IP_VAR = $(shell sh -c "kubectl describe service --namespace=kube-system ingress-nginx-ingress-controller | grep 'LoadBalancer Ingress:' | sed 's/\([[:graph:]]\+[[:space:]]\+\)\+\([0-9.]\+\|[[:graph:]]\+amazon[[:graph:]]\+\)/\2/'") kubectl/display/ip: - @echo " " @echo " " @echo Cluster address: ${IP_VAR} @echo " " @echo " " @echo export CLUSTER_ADDRESS=${IP_VAR} > ./cluster_address +## Destroy persistent volume claims +kubectl/destroy/pvc: + @echo " " + @echo "Destroying all Persistent Volume Claims." + @kubectl delete pvc --all -A + @echo " " + ## Tear down prometheus-operator and it's CRDs kubectl/destroy/prometheus/operator: @echo "Deleting prometheus-operator..." @@ -48,8 +49,3 @@ kubectl/destroy/prometheus/operator: -@kubectl delete crd alertmanagers.monitoring.coreos.com -@kubectl delete crd podmonitors.monitoring.coreos.com -@kubectl delete crd thanosrulers.monitoring.coreos.com - -## Create horizontal pod autoscalers for all relevant deployments -kubectl/implement/autoscaling: - gomplate -f addons/hpa.yaml | kubectl apply -f - - @kubens deepcell diff --git a/docs/images/Kiosk-Complete-Blur.png b/docs/images/Kiosk-Complete-Blur.png new file mode 100644 index 0000000000000000000000000000000000000000..9059f1ca78d5b13fc6f93efe7ee46af4bedc54e2 GIT binary patch literal 92125 zcmeFZXIN8f&^C%#K|vH11t}IlKtQ_mrXpRWcLnKH2)&7-f;6QC0-<*Zy+i0ldJi=q zNKZ&;ArJ_B%k9?f-se3(&UKyd-|KZ5!dmO8GtbOD_sk;bnTi|*IW0K}2?>S#(?`!q zNKT`0_OHJCo_t=gK4`o-8CJZvse22*9N` zDr?ZDW=%qWSz4M))o&4(Ok(slpla(XL%E6xLN53yk99%%+ zw{QO#==Z;$^E7j}`e!72=f9T)ERg%h9c~`32i*TtGmw?}|4XwUcYZeeYh6Ew6Z_Gb z$TKT)3Pd5%8ivTQR(pNo^q0rGkEM2@R7|go zkFj$B@!dTSs`=vXMn-Bw&L&plr=9yf5=B?NmlC|}y``m_;OZ4^o9{?S&oRCE=R>OJ z{M}Y>iX5hYUnU`?AU#!VK>n|1Nl4e9ykV+9g05bDOmgBBmDKN#r=Kpp%`MYwfZRpg zy^Cl)u(O>4m&@bIC3CCQZ?Y3Obf_v4tS zK4`Pcj&aJT8Ff>9i8M4ibsOcz&D`P3P>?Z>y<;8td!=>!@8!TL|(aO)&`tJ=4mJT4A|Z{ls7;6pn;!3C{-GWzvKO| zf=Aev&sJYEsA&-C6*@QTp!TFGm%jmisV)RlentB9!&Cq4q`xaWl{}Sxyv}jk-jMPg z_OR<7%7J-bbEdwid!W5^KbndcvUDzPBF8e?+s06D#EH3UkX#M6(~1Xfwg`E{ZkcFPYNIJNs758#Vao+ErWzZT{w?ykeW z+OI^VnVX&cQv=G!B3AYr8Psa%IlY{uz3Cp~rc}z`$V>*ugcLF6n>7 zo@`1UKu>SQj@?ZOy=2?KY=s=lOlMHa?T*aD^m?ebuXW<-P38IwAm~y98#*u6{Egx5Z z&V;Y6neMV4Od=Z2nzQ_=EPVr<(q)`v@>XZGI@{QkNKVg=t({1YW+O#ct(Alpr z_nk+5ORcTEN_NopC|@O4u^<;>ZFNlD$VhM^Ud^EvtXIA5j=r-@WvF;HuH5TDn|?)? z+L#7h%8@fw6Sbgltk~D7O8SRBP;ya;C7ky!u{Vh03w9qzhtvo2Dw#j_gt{_Zf@o!p z^w+&Kp1#EQ(kCshj5OHK0= zeMeG^J<+|{mEs`S;jSn4&a#vboENWvboAP*O`59@K1>%X9S&~sbr3O&_{Wk+HJbR} zkx}Aa=QmC1iqpF`Yh&d}{qbhT9#odSl5v?aOi)c@uNJH%yh78Idx0~%)YV|3;Fes1 zVT;=~mnhZsCI(yr+3V0t=oBRp*Qvm*v?a+dO5=rw+Lt|c#{Ce;!iT4s8v{2@>7z8R zf@?Eb-aeqQ=6X z%6pPLKxi^8%Tp^$@RS}Bk;Q^w@HstK^>B9jt9TH!GQ;w|+O`w!0)sNAk*i9OF4)B( z31Y<2d=d-458D=;En1kVAJxCu(byvSRjI%-kZvfL?qneH9XtN&-Vy^UYiw;qB6!$Z z9R*JGd*+s8-3?Lit|_$BG1V#!)K*Ei@;_H3uo4%-*Wd?Kz;Lr8czm zoxNvX=`lR_fKv=q?|8IY!Gn`>n|)CdvxXeYV)ui4qwu>{vD`=(#gcb7i!HozEY1mH z;j7wDH6}6I=7Hee)wBYAb-|GPLMLFiH-@+b;vUbz#bq@@4F7NlI(MjMvKF?c+cR`= zQqTeN8aWVJQnYp(WE@}R^=!1Sss0>HtoBwj$l@Fn0ncA@MNBRbU>yPrqjnk6jK!tv zj#2TR501w1bcqDMmEn+Ri~aVhvd>PMo-pBOm3LxX?JdD3iPW>`^eF@7)t1XB>ceFk zp@ho*tuj-Io}u$dDVsAY93^29<#U1GjytD5*SXrZaGhQzH*q;z$j6@r?tA&gqXO(H z(t822F%yqzNgH)_i%S{qMW>C-2be8`_f#%1X0cs7Rh&{j5&YXx4Dg(8K5t?}Db7Cp zl-`p`W!RG0V^s8mR`l`+TYH{jF84a<;gHSkQ}#6r1HuYr4~J8dy3hm%TD-1g+LY>@ z%K(+#d@ZZIHeW(6?w_vFDmok@+nc?MXtt+Q?|G}a2whVU9lNq?r*l}{9d=hdpq=dw zY5(qX)vQoy>X*1obs5kU<_ew2V2^}?L|tD+U+U`O;lDsd@|&#KUg7(_OI;$uX-STcWAwrv3XMB!MRw7p2DIN=45*r1s7hyn zTqSlbIar=E(cCGxrFoX}d4Sje|Rx^Wu`hNt!{cNHIq`o2=h4%gHFyOH0()*g;YzR$z)T&VBtcn+=u)`osz}9 zJ6dNXCg&I`$I?f#PUVy5vo(q2#HCH4wnt27`zh5RiCic&QlKn#>{Dipfvb}o+n|+K zN#e`A57~9x?Ny1_L3?u|OJcC^H&T`DH2aG@`5qDVUupxBJl{h|DR?aTVt)>dzWUQ{yb7n`|X zxjo*x)?O?;dq?vf@zuIR;E4(bb={Fl9>HfuqtEy>pzlE2>5)dxki>fR9Grjo;}#PI z8n^RC_0CVXscW_c9U@V}A|DkrV7$AGn^$GKJv?n?=n%tR4xYA(=gn+X7&TuZ3h3PS zIz@1=C*FsG&CZX;m|XhDHd}_1$_2<9(GEDgY)8yp)%?hyk@j&hU!+u{0a9YakdStH zZT@>t7B*x)V2G>!p%yBis}AI6CqwjS_=uGd-%^_TsExjkpqjq6EVM8&Rp~UR?QJDq zu@G?V>NLw0Tin32Yvkg&S|A;L)eI6Z2A8`7rFmt_z%yct1joZYM z{)A39EqHl^;cMY)dCg0^m+NEqbx&V7yY0O^6D)f zvvIK+aZo*Q2W{Z?WuK1NqK8eyIHksi*^9&RwfLD3d#{s;3{S&;O}u>P%=RExu2x?m}Ms)f5*E1gup~ zp`M_Nxq9Pn>Vx*az%Un7YSL{&C7MLJl~Y;T@WnIfsED=sT-vjlym%IunJ;C9%PDZO zdRrEYR2##aoI83O9;3T2DZZpcW~DT?w6uBVFhf2xB$aPOnDC`&>x0%HQ7kbq-o-~K zW0*|74iX*{E@RWs4EG|?UvC2s!m{wb_?dV$TeMoRb;8+i-OnRO9_BJiGmbKMtkZ&8 zE&a6An!im@*cdaGQ=(mEXT=wy-}-FRp0*9u=&3xRq^T@%P^?v<=~EFNzxo)r+2XQ2aM9mx`)o|!6`D$N^O%jH|+kB#?b zk{`aQSDKh`{tuY9dv9QxsY1LWPUOxXp&8(j%GCXzW>QxUXFj+4Gu)CQ@!#c=qMDlV z`b_wD3;L%(fOgVFQ2E~h+GH(>w5=CbdM*67oBJ^ki0Q=JamtYi{;U5?2KZ))1!BK- z_}LuEKUDl2(}4e}b0(tCEQU4yHJ4=gd8*b5JVW}?AJ~BDjnvu8Oo8iCP-XQ$EXe;( z{Qv8YM>JcbYG*kva7f1eYaanIZQORmV7KdDnchVNU-tOZwEzllBEwYoMJN2rYo)Rf zg+F~WAa)(r#;5r%h6?Zg-Nx(mIg|BueZ3F=(h(h2DJmO7@DS6#gx+X$;;oL?K}^i8 ze<>`2jH*>{EQ9~tzx1g{`wbIfrar^$UkdWZ1<*2t3me5BGypJh^i`(7@=6D*8~@V1 z7f=%E|DE#Rg8AQF{zvKlS26xLC;z`F##8bfM6Y}$O*-~*0>$?~ zwFQZ@QT*$WOC_HM?1w%TIQ2hvDwRWuiq23~LG}+f479YA27J%8)(p#k*%cE174m?} zB6U!|vei#(5pXR#PhRP1E(J78c^9F@Qt0JjV&GGEfUa(-05QV5;2ADMJb5D`z03Fi zYCx|Hc)&E=!}Z4`{$X6tG!t~|ij!0CWtH5^D`024l_4lxE0*S^HWOH0EURt^F)GXT zo7gSTJSh3-`+a>cphl}eKT-kbLiT#J)3^NjgVoB6$Ho=a&x0$%ntv|rA%sL4J@{%o z?1v@2SLA0mYq%X2lxKDL-B|mEw)2BD6YZ>6a+9m3SYw`Km`G#ra~UUP#GCckxcXWr z1vgMmp^_DGvI6!TRSD+1{g)=cI7KVj!SwdW@qSS$H8<4cMMF_u8`|ptm19&{WNdaB*N2(%i&oVM;#$8*uSSKYutbX!ji<&W`wpx^aHFsX-eOLv%Fk3l0Tj}jmike?YHkG_FfI%4`KG%fB_7zLQ&4=Tl&+|A_6aFFsF@`{(Xk(gSLP zc)j#P*iv-ouW;R%@41cfoh+Gd(pP?eMV4?{tvotsYK9I5C+i#)maVMG!d+vz$obIb z%Wc&xo=@K7;0v63BKNooUI(QGlyxy=acO@>zfw0rHn5CFXH{3;qfgQqUTnpAu;%kk zUCc)rmc1%6PF!aUajJMlF^xclF!w29E!-%{pc@5OYHs3ZeDVm4dv%2suW+*yHq931 z+M9$KKz?c>u!DME8t?f9?W$Uz-9QEH3l6RsxO;1EM+TAhw=#+d$eagcKb2q4_>CO8 ztX|mS5pYH(rKIq7kW!Y#C=l%kRnvz8IYXBc(YKoBb`!W&$Iu)RNzvu7dwRJ=r95wO zW_eFF7j)}|#yXd#u2VHxoBG%d&GdhWvF&$T-bvi`bqzOrlDts-ihH7rYa|B$gz-jG zO+Ng}rTeN?3PuF+agXtz-d>7>>~8sx$3u!A)r*S^k5tpKF8Rrd&XRvIYlF zY2EgQvgQF_@Yt>zV(p}E^npl*i*PdNv0B!AIlN~R8ipL%j}86gSJ5vf&+(WoA1Qtu z>r2Cbfq`9BVeBL%Az{=))yX_A^J~mHx`vo`$=u&q`FbDwc$(O!0_Dy{FA=Q%%!B>v z<$%Y1YpoC!Ta_%o7Li(Kjlf6FQ8HIhKH45H?R!&#Iz;Ze?X}lkLS2r?vt}PkS2G?P zw}7QxNu7mP1}6M88f5_h-c`(g&H^llN}qHz!|Ju?-JX=Bg?w^sbg(K6?R8Wag?>(P z`I_Hg_m$KkrEJ}@+@u@tl>5ZTO?$}IZI3#WuzQZ_Lsz|=u3i(_Tz(U5UfUm!w9wA- z7wr+#cka&K&3px39RqVp*w4;QLt@ZZoZREF4KJZcpKpUhmjG>VbgUfHsscYO2v^My zRSi_^%YKTfY%$c}xc468=4STvCvYDikoHoT$c56*5~7zv{+j(uvL0WH=txP46okA` z+{)`U^2bG8nQiXQXY2~u6a}oDUyL5zkId#&L=s-MK=Wur1n91AYJoE{UkPxX($tSL zsH?WXHHEAVX_z@vhC5U63Eo7_N%qY#^v}HdQo}aVNgWg6tieaEN8Qc#^0@0n8_Lsc z7YQBq%d(fDq-__Clf8ReoqP~`c?$np@A-2r^D%X_DNa1DB#UfuD_U8%}JNNV^`gb6tlkeg^0{~|Xq%$c0HZhY5 zKTJ%t*z%8^($h>H{%rTLpQd3Es;=O+5XZWBFPR;gSZUK#{1N|Qb2dtanFW8O>67Qp zahVa9m0R`F!Fu^JbyLjxY6*5=Sp6MC+F~$tU9V$YzLt8XA0xpIs$q`0OnFgG8;!Z^ z?2kPJx$p1RHA3!D!C=-^X*$%N&L3?tUPoCff_Tz4b5ngbVa?nAf8klF_;U{~PqCmA zuKYj+cK%!~f*4~YexSv{JKPpY=!>mNhb-i8twn&4S+sy{_5ppLS7qQBGuz9Ipb!^$ znq4a-ROjp2*SO9myy!AYHkz@z=;h>0zz>Ut=C%0Jv*M2jLT92CDqJ`uyy+Yljb^DTeY+}_ z?UMLpg7)?Z45r$#qp`FIeFU^6ItOJOmtDbVcNy0dD(==Nd<%Y1zk_+)c_X@fR6awW zE-5)4k+C-*PkU^|k%!fo1DRzMkCx#oxK?T6ot_eXOfegAiAP; zA85R;@BB7(Wj>25#am9BNH80e4cU2i`#97teLT4qv~CrX7c0lkKA|Lb?QU%gPL066)|2tMwbaW&AM> zwDWc(b??Iy=1dJ5C4=;J?kptdY2oWd>M+n=b_iEeDN zSE8HO+!_?NF8_s+{1I0Ix$oVpa#Q$DGnLXWj-ISMP7F+E0om_95I{o@QmO;zqYU4g|Mh+#s)#9nJ!BV=zP^7KZh_rOK4&FQ~`=0)0Trp=YhtKku5~5x#Tq^h=S~f5U4_Mp82Qax3V?C&dMy z0*1GDKXN_bsm|;a^JUJ~;pV9TmmWl;7tO9j^Ox&{`;R(AE{|$h69waXp(%FJ(*n0b zf1WUoa}S_ZS3e7%{^8yKy7cop2vEbc?@!tObrsM^CV*Qa|C;beCi2H-GKl~I`{3*J zEB`_4T2xGdcBfyd{5;OTyQlhr)K(roC;!i&8~|*sAmz>agLC{583cv_E*o@{?et%A z`~8u8^;Cr3+|B>a1xOrE@0%B{2qt_PU-5iS`=2q+lLKQgaXtPwId<+5Ktu@EBu4$4 z;b0ng3oMK?P;u}-`!EPl5upXF8UN-u&XpxgDdnpe71+*UR-@>~r~O(mv1fj;ou3Bs(|O_nXL zS6Z}pbogw?Yu{z2e?2F#y1GezY3rCc%YlhJli4roU~+uabina|Pv7?PfZ1+k1N@mW zqtL6bZ%X?_IE$=+3HdBTFP0PMgZqWXN;~kG9jIa9G;qU(j105mJs|Ncdw6*{NixN_ zK>`!%;^M;SC+5%#;odo$r3jvuen$wuDV*vsOX`+bIpgVoLeNk0S`Fsz2sD$F`3+=kD*=YrTlqd(EUfjFfCdd1rVt$l&4q419@ zUrcaTaXVWaFXW(Iyt%ole5QD&n}6A`<}3hk=2K8RC$G=3|{C?%{+t%Cq&e3 zPV#tv(yiVZc2L-RxF4*Qzh~g#x;_c)zN8~B&9!T*23A%%r;5YK{&wwDfoA@(=S=96 z&vV757b}Eby>C=kmu)#qz{LaI<{V%StiE;hrpTU6^ju@>^Jn=29vgK~)ZvDM_d8k7 zCqzgRK`9dZ`=ieo$+MKbWKO>OwAd$c5aLVp%t}wcLIbv)sVh%SZ5BpA0fdBCgT)~_ zW)HB(b}lZVDxO=Ntduu+pCx(X(Gz*9?Y;R=A`?~iQwl$$(dfK_nx4*l6BjZU%(T_k zf^$MB@jz~$O;e@wu8HxB>8e3;6I|tDFt=dU$J?6Q1Oh=-lBg7H4-w05x%>vv2OS{Ozz+J(HC0p`@lwb9TC=%Fg1e?v$h&Z(segfZ=g@U^q^^E%?E zw~JC)d_YsBSw;GlA98ZTax#ll;)brrJ)DY<+|bFC;N{XO(qR=@P5*w?)+S#w8kB3t z{=lNo`*9ILX}=rrh;GT&%gtFyTwCW& zt|Hj0MhiB-%N=yO`#F55m(a||H(Vfm__|LGXsHVE%j&f7*u;Fn+yU0l7E@HUnrXwL zEP_OfEGO{!Z3fS-6b|GS9tk}N)MT(bp1bJa3YMk)Wn?Y6NsDWXhIK6jylJPZomG{r zxD0C)_=p-A4n8Op?c%fM3*V|Zs$Ed7a3xh$>}cLcFRue-l&dJxCX3)lM1iaC=-3V3 z14&CS^zO{x7~wQ8{G42FW_thJ3UNFu94k23h|h19BJ}UCeG51~Mh>RWx%(IVjsm%8Co9!*zUgF%U9qLFV6`$m4oTPT#ojmxRcyKX0!XnWl z*H}|ov0SaYsM-zc)b$kp<+K|%?SImy`bp{d8=UIr7oM;+CXK39IGfO-OKFQ~`b3v@AljH6NRRt5% z3FI}Kn@hbmhZy-2u(&UxzIDgY-L9KWUfY&q7N`7f$lxJMJ@FUxoF=#}jEuku3HR;z zhtr|(pr9bVG=1)J*P2-_){DPvi4;@JX%kf9xFm7)24_x*!>n&_3MbkGZ-Q$G6gxECF7yv<2OC=Icy;0qI*Cwm}XZ0h0D1G=DwIEfJUl*rsfh1hir zdC2wG=rSw3f#>QK&S&}Sgn`3lNy&p2#+h|DKZtW6xKuPwUsz37H>bZ=Bxg1kmXh}+ zAz|SNr{6en6NMaJ!27-rD4_EaEqN1Xk>^$Cwr*WDJJZ9A5SvP#&iG-Asooc~ZH8_zQG&f7c z;5-LV`VR2g7flh-9Ahi$nHVIw=hy0G72B=VLg|Yaiibo%>xMGr|>%RSqAs(DA!5EpSl@MfTi04U;vn)#iJnOr*HN(x`aW6<)WJ>YSzO zXKbvZoOG0e*|^x#c=qwkxL|;O6ZtmK6v3`YZ zIjQidT$1N85i8V-jp3Y9qUc@A^e-PB4>P|cxiet6bSZOj@u~wHXY``t`m71d<-9qy ze2Y_wJihNAUHX`+r}Awe#ANneVGl{ZjBW-_Ahi8yyvI;cL4Wq4*#LFA64d3m+IX=S zRyR2guRXrgRi<_}@pbz{<{6LWliM2>4J{r)zkqRTzLf3h;t%rb-wLXlg#xp}&9Du} zYUuGe7m#MsMCmeoqd9L0zg{W%K;S}x8oHp`d}q}vgS@YN@WMeAzTLOR3vZvBo;Xd{ z*5ULBeWO(SDH?xAfS=UdgSB^kY~GP2ik{Crz2k8=g#QAHV7p}e;qb5^Q^K6+=!PuI zC@4_T_uSoqdz|#EKQx<_jV_;uBpT6GfjNJHU#TAQRud|Y`<#U)4CdLoRS*#EdG9`v zn%vmTJ2ieEFKjBe53f5CDAY7VIpZr%8NOw1^)D_$B{=sz6Br3tPV^FH7avmUzg{O} zk`kZ&PVR)Wem-x*0p>pJZhq_Drl}YMidkQ0%)@{vaou2{9%jIJbA$_<2Of;*F)aBVEn7^c z3Vg~^fjn4Qo?t6~2de3|%Q~ELBYMgP{ zbsmF;ngh3{w{31n934e89y_a#61#d0LqwOFStBd!T{Lyd$-DV1I{_$^{ITfLi95*y zD&g443cHWh!dmB3P)|Oj%{|(kx!~#Z0S9i{Z^rVpWxJ!>zeI zetdxAFQjv*^K5)muV0Bd?2t6lYw(TSHS$=@Q0Z$tW9=Cjr%YcJ{4ep@Q+ciBuu2|# z&tQ=63+oAp)}Snc52e>JF&u~LAT*wIt4VVHj)r4urV`5JEd??~F;x=|rYpR?)7liN zlypmUc*oB&@=lyvo=hGPcPRK>7$WSACZk)4T?+m%N# z@%A-@LAm36)pk8;f_8)Fh~Wu7h7O!Z3UTF?2X6=DlxXzA1-c358Rt=ey>RX zNF38Ff51Hb(MJ1gg4|77UW=C1Yg`7Yru+we*!!8u#inv>De;DQ434ufm8}Cn+2&v^ zC*CBst1C00VS&ooBfEw-)7=xwYIX+>mG=91Bz_p6{Hh#gjpXGilJ?3sa+LZcri3QO z53eURJ`uLx{P?J43q(IPejryXL#drUN9q{nQV}U7MY>d>Bm26{CFmr#>Bl}FLt!h{ z6kF-Se#awTextQMd1!zKbsSh^P+7?5dkv~!VB10S1hIUuLE-q@*;z!{rDZB+CEOvc zOD8i8L=4ka8-Dv|qz~AbR*2NkVt-o1m|ckzkjd-j?$1{HUcLxWrF(YE8K)?3-s9rp z8tk>2n26Im&#%|r-uf11E@*wZJW((%GLrs0E`{=3*&Y9wgYIW)YQ@eLFT)x3`@oj# zf$wvF@ZcgQ>hxwYO^A5s3EE%1V zSQD(tO6EUhd)2bK;X;kJmQdc*+LPMfwFh^osE31AFZ?>)zp{jXWVimLER+EDp#721 zv)_2-@8`LJ5Z);2>wjKvprQsY^9)jG{D;bD3~f4GT_ z+fr1h$3bGhOyu9DoaYvh*`!8FG5=o6&qe%Sp1M>7SZP5z+3qv~rhoiVQjD6B5#M-H zQdM4_%rK5M(R~*PM1?&-BXIoDT|T~+iObQC3=NS;H%ZXOi&af+LqUKNP-Baf`E%CE z@&H-G67-cSNJ>)^7jmOChTGT$H>!S~-1E`%095SOR<7NYtEH@+9n{8oV*!KoQZl!2 zv$KOmhH05Jy=BhRE>wTygD#d$)nr-JVJh;(O}foOH4P1&XFRCGC4F@?^sXRzam4OA zk0h885wP60J-xj*_$?dbEW%Jj%tGv%8C1Vi0N{K8^7=&eoy64i^o2MJqi5aRaF7B= zC^W;E zJ}zGWdgmrX*hGy>$JSzA^yb_1eOsTQ_hZhZr>hSD{s+ou;d^Cj;<8z;X9ICPNgzln zXuDq$6+j(^EH)kED6A{@JNEa&`s}Fs5tN@i@w^f*rS?mWq|TGIPB*InWPlCAyV!Y| ztJmYat+S7h&*R5tnwrrQn(FF+30^X+-95Dga!@-ew;A)gUs~jd&DFHg$TU8^^E?Y+ zfLxt#mv+Zhticm~g~MH4-JB=IVD0wPRhkNPg0_Q!d9@N@5nHkAwEH{{eIH@cMpkKG z>s34T0w9Qkoqf9YZKn9H*MR9e`wM`LX4u-DVzm6u({W*StWfovQR#aQ;gy3GOM`XK zgP{x#tV!r^!ph?xw(?&+d+ry&`2t;p2 zL)6dx3f=~WNO|}#*E-G(StR+W-8q@r3s5r_2|Ox{6+^?rH{W07kT_oCW?V&ZV!y~? z5Crk2MDZLrm58CI6ELqQc5Dcg$7AHhXGZ=#&=#;>`(DOG74ajS$YLv% z3%3W{@yY0%{xEZGJZ8TV>6*sZrjmj)bR#~gEvUtO>Wu7nrPP*TL;x}I$EYeCf=*S^ zC*<-BYIPHh87_b6*%=YtFODx%j<2XeCHFV!O@FoZI3`Z0O6=!*6>3R3HPjt$1p|2D z>%`WeTvGj3SNs}_MX-cwlI!HSgSV-*y*;Z%T4Xj1=7BEGO7QLP#}Zl?eOZfUp@+&W zbq2S5-`^8~0x^`Jkz>={2E!$0ZKpuI+Oy)_dNq6$J81&%q5+&3f2h)HYO0^wb9^)= zbhh4kSq0PZ0stlmyJ7-F)wtIHG+51$Cnxy(z)+kvF}u5Kd`#$nckxK&lC8J5cYh*s zcK6}3BlQgJ%BXQFuV;x%&gVK*aJ`>)hff%% zrrK0G#OV8QN*Hn1ye}ayfKqHR>Vu{=d~Epp1hH?Nh5Nnptk3Y3CToE-fC)VRD1EIE z0% z9Y|t<(Rs8R_8~(dRRgI-FX7Qx`fY!}aTC^u6blvxuxU~}=PLmI$O>tTWP(-7mO+Lu zn-%DO*DpUP8*aaQm;Z2ato0)h9C3eoQEuqBnIP#lXnlI|weZ`zZ#-5rgaBSul3;+Fv>9&LEe}S9jv2kr1T5}!uaI%pV*vqa%-qN0 zefId1d4Y|bHQ)&P{CTmAMO zW=!?6T%V<@EX$bRlaPnzdmFa^p7;g%$QTu%`6bO7u+B28;ffO{4_Q$K`jxjp4YghN zw{KvsntU1_V;s&pSh}3#+!xT=`l!L#k-0LMW1?_Oq)(9;q;3iwZ5H_GH(ACC?c|=! zsI&tx68+hLm|vhl^lD(xmvOMMuk3dR+R34)7t}qL$=|lR8^oLz$V>$AjD&q_u>B%$ z`1=8M5xLMU3jek0ODYw@&ev92;+jMB?e{Z-UX-Jn&$Ci6V!T$`Sd;EMI*E``JoCnS zjoFYq;){cx_S&ht#@f;v^j*L_8_Z`RNZH;OU#YWMD#+gI;v^Epf?8d$-9oGMRVdsP zTd5_Mt7qM0eLJeywyvvNuNigw;D(U>&T3Y;3pqgObZa*dnAJxBOG(WaPSt3EW_rAb6Rzqegkm;NT*Cd;LiP8HJvnYRG4EM{hU9$qo4q18#F=g;*Fya!bGWAqE0 z&0DR&iRLMO0y*QInQ~0*hR+5J83(WU2sgYtgB#wZ7j+d8h6ZSfyf#0%=A4Kr9p&0C z?q5*r0k-p8hQMc(lI!TgbGsW4lCh2cKFWgl24%!>=k&q^G{G)us}#obSi&IUgrWkLbtx#ha4!8PkWnrr%FbnZ_42S!+ulLRbxQkwL0QG z^?|n20IG=)+DE6beaz^)rwR~($5vZKELviF-=N<+l1w1jMd2QV?4Z=&W-8Ff|IXWX z!JL96M(e&J71*2MLel;H`UlS?-@iX_aw<`tJVZ@9Ss6i6#3%$-RP;jAtkXVp?-b6b z(%Dxb7hWIh8+f-uYRtj2=x6F-2lp}n+AQS#dGJcm6}@@Og|FZ7>r)M&Rq|3Kcb3P* zD{*LAT>`RmnuSbx6k%&bw3evwEmw#!5f;xj{rcJe(wLmFDv+v~s@b0)nn?!^)p>om z*NXo1jeKy&30F_)c4iG}F{dC%TkOYN>jnFsj+@R&Pk;2d2uKyYymq>3qq(&FEu>xq zq@~r}8RV_?xZ_KD#^0WOV2RX@v32cx)QwS4yC8kk-rff-~xzr-{q@kX< zC3T1igZNG}+kpI|!-tQM@EoiDgI)RoLK(ST+JlwFpZfMaS+^2VlnF-4< zO+kyWVqN(v?Lvl6wF$Pps@aD@)T&;K(Um(<;F1zvXJ>RyR+uYh?SjvzM$Zp@j5c1Q z-kgCM#uPs$ZLBCx|M1!2d~hM#hcSgVX&X8d_^`+JwYYOycuthB>GDQfFxbYmc$@c8 zs&L|*(Jk|8Tg@d)Rm7YE<)AwH)9DjP=YobZy*pCsfjD>eEQKVW3Johex0s8Lu3vtm zdXr~NESMmHJhMKCKnJ`JR%G52o8#w&=h}#Ww6>f*S9S5C`0f*-Op#ip>AdA8AOSKu zaXmVqTD`KSW@%JA%ciEJOYFdQ7GA4-bP8q>Dc5FxhGKQ)ON3rvw}wMU7}6I9YOkT; z-c_!oB$U2vF}E8_{w(@-w5y*1TQQ2>442qKe!e;3Y|;XkS?Y-^6wT9|qDsRUbV`dt zLj6fSCpQZHhIQrx}78{fB5!8wvWGk z7mS;VS{fb2ww%2P8QKMyk+@qA+gO2fx!nyeM7TCAqpshv7}pZgl-T-8T_dl_r^a22 zzAg9c^_Tg3%4!HBWaFysm_`j$)Xn4OigTC~YmHm~n7T#z;;8mKdTi^Z3x31Iau#tM8dChXY8JQRE%`;sxb)V!Bw zGF8Rzd#Ib=R-_m1w7>JUh5%~LyjcKdSoNY12J<=UbO%Uozd-O0iAJM=jH7s~HoHOR ztwzq7j0SeX*5g^A*s8ig1Jx79xs>SFu+*^K_fhYTvR~j1c4w85us+%yb5Gs4akVg2 z0pT2rvv_7P81)GNL_F|#{s|p?<*@pKR7?#@y7$Qpy!KgM-@SXIYprW($-UZv@$vC#=XB#--dj?n*c8H>l`aVG zZF=*3gE_=P*~S#eOEdWH**CxmnD&Wz6;V3kzrg9QE&d9et)Q^Wg^K zT>;{KM$Y;BZs_qZL@bhSjSv1?H3{-ap)F2Nqpl9tn{&XDi5>IH&+Q~lvjXr zKXfCy(BOI*X(VeBY<;O-S6O$Yu-?XTR&2V`p~3|m>(KekFnsG0e znPpbgddSP3u9}osa~-riQ(-4c>4WBqX-K4W8r4)M&My-P4_RGZtX(YQ?iov=@TV?? zYX?$}$O_;JW44_9)aIkGDvrz@N6~0ZVRkxl%=(4DG~OD6Im_6C;9MjqSPrE#ZM(l? zSiHgja)>lrbA&3|P#qGn+?K664$Q>U!^SGTFNODC&9=1c{eJ&=^8MdDPlZQ5s>o(` zt6SJQ%0WD_Un2f{rT9Gf(*@wATn6zi{>f`PUrWsLuyFh(J3qpzoQppA?QLYjjo4m? z;16hiTOBQ}FLA4MkNLxI;kv1j;4h;<#erqe5PuXb)}SR_K)^*s!u`0eZWJ0O>c zrAf%lVy*D^&dgo`Yef?vEIy3h%X7C!RT(FKU6@^RraM17d%*2^U&J>xVbS-$ax`cE zVwspX=7UxDg=5!!>EgF}wVs$j1uX-V8~VD_n6Iqwi3{&q3YQxe+eeLFXe41rxKzO6&?D9L^P zwJ(lrGlE0-6lLw*90Eh!-G!P1d`c?j}M+efzsB^6bR8u0oKoPP#%|N6- zKu&Z4Av2}O$6+twg&#G%zo-@eCE(pC!P(E3@6&u6yc6U0p!c6DcuQ^4_4|RnJxaqZ zF9eukVy{zQJS(DSTnYAMfF0R7So!Zdy)OfMR)GA}Bz`&HH&WGo&4SXkLL%Z0ro}o zsob#A?`8ZLqVWvy#t)t67s21|=C48iuXfGUIqxd5*;<<9SKnv#!rOcL7o|1S(+W@k zMn1R(fD^xW|9;okr`RE8e1sy`bs_sQ<6@wYARI`gP|%JX1HrGog2Ja+Be?CV=b6Of zPENO({TK4`t7G-9ok0E-(lecKfoFVoiJvtyEEIveA)N-PTN00{wnq{q59e;cof+M~ zElbaeGIVc7z(xx^xTA0VD7pVhN=lvkQF>wg?{b?O`=8}DYj@B>4IYo5KOfgb!oB(d zyD@zlfk4z43$g+wKGURQh@O_g)r4`1#LVRK!2m+R05FoBje@Z80K!DD$jFl4~Y}`E>yoc zdhYG`l*6&BnOO#1ev#kE!Kxa-~%U#1E$17oL_WNLPX*woO_6{Dm%t)6ZHM*Fx0xqpsJ z_eq>Z<)#BUc1B!s<9<#~4p6RfeC@uD z$g8i7zC^+)^MM%PmCJ$QwvqO*>D?Gq;Eyp7!~eq=mL1ah<{N;B1Vlwk^(zkp0={Zz z4_VrId&?*&l-S#Gvaz9N8!AC-YiojblbNA#&(m=W)y~Tdi}FB`iMVHg>?(L*JU{N_ z1ibc%P5)r}%DtmUdF#HK5mEqex7-IktT=#Q@Yx0GS-QH|=VIfp+gMuzp?hLn90UJX zJjX=0A_bp4vPPVhm7{2O)CQ8ACo}Te9~=P5H0R7y-&6k{4r^f2^V3RRma7ISP^0HfB*Om_o zRK{qSlq$iT()ozzp7TK#2-sBi3n|x} z9Jb{|(akOnZg~|IN^nWl-nYtn+&iFvNCg5~ic~#<0|jfcnGJhw^!VB1DIm3+T?X)K z^_Uut!%v!@GdZoStZqp-OKvm*nO6NpiH+4P#@^0OSBD@b`nC#D3jj7hi_{x6;#;4X zV2Ist+WnjPQ;XgFddJNMnYq7B6W7SgUD!?UHZA7(;<(X7++MfJaTP#DRi+(Wk;Y%v zZdioWE+bc2^eElb?2%h<&c zSL8XnwPcc`iYfb>V(xD|_Kyxqlf1Wd-EbJ7hzQ>ernvw|A;ZttSWb1_k@k72Nu|j^?Bhj@e;(7y*IQra!6DeYn5*qdl##bg^K!Y zb;`H_WTR86GqxZ%tGa|G-s^KOP$~#gEHe!CT=*ghp=IyWAiAwjrfpU>)_9233TqIZ zu<-B#-BP2L#Oi8j*;;f3@aLlLgiCm;jujlaQJziYaO`e4!X*LGgNvOu4M*m4jWm%4 zfJ^KKqdfN>>*?jXQqT$N)Plsl_vUOHn0CL1FSETX31#q!P_F{}3dOZp*FG^WAAxZ( zJRYNc!^z1>y;^4F=Zh{jj_a09Yhu|z=)gZvxRBfN)b^}iNfM3TiM32Y4eyvoa`Hy+ z`1W$tHDK$lL=;+ZK7=dT>t`viDCT6c>50G3WfntT1k7`?S5jOLQe3->7+%t#|v6^f`dq>lva>UvDqgMveXHKBGhP zdzC&U!JA{-APqlMViyQuwGlm)`Srdhn8yn#6K`k4h}~V*<5#Xx4i4WD2R{{=bzG4B z@Z)%Ee<->BXJ%yB-9dQs*?{GIw(S;3|0k1q-BT0ivOXwIoSQoz_V_<7iRTkMZ*`Q? zxv&Y964(`BD!YFO=uv*KV_l$v?S6N~dAj*s-CmySK=AH})!c4^b++8}vl{wtASIFG z(ZIykk`NL$o%8vUU{tJP!xglEwy35Yvyy-fIg0bdr67V@O+H}**iQ0nu2C6M6=FT z(KeGZgkpf1_fOT4pyto2BMhY!*N0{aBa(%Ew+%4~FfqcDtW_FA7IQ?3hbUC_Xm)QB z6l1s0)a~)g6f6_k*;`dLk>Ski`5>XZ7rYC#qOXcC)l)9Ny-pz$B``g7-6z|ha2rkl zAoySV&es7r9jbt_45R*0qt{>i7!p@LoZ;_bXv*;C%D;K?+~bSzS4WO zoA!EeYpT3H9-4o}U0e?}Stzk!ml^Y&$s7E$ zzRhga^u7D|&YQOK_>tgwl?5~VQ|J%sjp5uy^jBuEhQ!>5uqCS%DH3mx(v7UeQB)bt-i>&U~GEVt36aHwERn2F1ICLMYgYJt&hr+ zpDU1A8juu$oOPgBo}r;6&_nSSVcKKbShwk{ac7|6BN79bmV3EIgPl#>_Oip12CPZzyai{@VP(M?7;HL?$0+^wFqNj_lH zHyM_CRj{CS^<8G8+I^*qewszQ&6vX5{uCSnb>2#Fxpj~+w+Cr8n^E$j+dCoQg17G- zyAqshiQU;0{Wm3pbi;6$2~07hPwUG>hEMS=&g*OsjI1<{D#ok^Xis@vu{5iWPjKh3 z^T>YWZ1c|L@;M*Aa9^;l_&J6)XE!=FSh?zyz(^qlzwf?!lMt@6{}$_?pbHBw3;{1g z$pThH-KJ+1Y?idphZ_T`$3NRW?JTF7=5Vr2Y_^_o&O+xcMmjs2mXf*I~@^`JBf zp7~dn*{qUizB?(mk?I)}^8smkwWEXV_8puHFvI%4EQqjkqsjG3<1!cnErsuLcel4l zk=v6YA;%PwM$Nb?>3dI~K83OFs(HQ0cDGx{{9%puT3q6gf2r##T*f>mjpMEkH6+}U ziiWxL09_P9EZ9%Ce|&pIOdBw!NP?Kie&*Wda{$$C%t>5u*x%TG&V8bAGgj=8K2{}g zzNkx|p1l@jIFOLP5D=?>IxjX~e1S58!-o0Sf;Doy&sErU{ahUK1s+uM?ieW4kWX8P zGTvm@nk;G3m#!$ztf$WLHN3ZVY3%5SmYX7$X0L3|tIK=4#vauWKHV{!-~*KnffeFA zCuJm@9V*sXCHaoZrl?5D0^y^X`PD5cp9MXRM3zmKVZzd% zAkWx;-e#V6CUEF6;II1V@E>A-McB*NJ4qBOhRmfwDfG3@N8?7(QyRU-fmO6z?~fD< zl=hHqOb++JB&osHIMp>?pQ?PZ>rXtW5br|sMYBgkBSH24&R42k_Cj!3?Va}L;XLGH z*=;AKJl%H+&j1dd#TQ9&!ML@$)<7Z4IzzN9JUHvs8La5jZE#jJm~4B(_kw;(KcQ9^ z0!5hKX$)`BJd$B&V^_Bf+9|$kx*a3k9(2I?D_{5n#+-C`5G)_c%h}?9od~g%;X*r< zcnZvIbTX)~Mw0sY3-G8{xhL0Avl+{Nh{$PABEFfyvk&D0?+@sZ`D>CrncJ((Px)S~gu2#jualj6xyp zJX3rWq(=3SFZN3I%wUz(k8nZMOzS3&FeVzzKCn->FHAI~T$Fv!wYMXrl;a zvYi?zKd~1S8YkUd49(#Ajg_htD82GkhK^Cwc$eP`58l5_Z~3qkM5-rEJZ956SI^1U zEk+KaKBAfYNnPkHTwUvjd^Y>75oNDxT>f$OVinBpyqw74hq}=V4L`_KZ(`UT%ADT_ z4?8e+Ch+}Lf!p|cdtZS|kBi|NS&29~{+cJ>yxREL%GM?JOrz|K#h#h(c$oNoS<~l{ zRgFgeL()2SyPNkvM6V2|lrP9`)jhJ~6$(g+{PzlSxP z*vt?6P{88!^J9K-Q-S9l**b>*7^uoM8($ui6gQq~`WCNq^ViuPE(bFX?-jpNb!mGH zvD=$gZcV#(HZsP|;+M{N(b?^xnCZy{IV;&349i8}4G8do6WToZB8UcH6_41Bk>cGLwJR&1ITF!%{va+Cl zsRbt)6%`ZOBsNhQ^ph%Dhj)6pPs?8m3&_j}rMgd*jh)S31+akFFQd-)1Kf7D*v8~K zC0nu*>W{7OWJ5Q0Ni90|=CcknL+r24e#nb0cwALKWEHL=x0@lO9VN(0q-TfEPjo#| zm$I?3u?;f8ftbMfHmwZ91|=laB_&DmZ*ZuLolc5rs~@Rsn?b_(`CZrjJyOgo5?*ru zwfJAUV%@W*KJ5KsO&$KpngWn$H{RFHZhlVove(mFa=<5&qU>}_eo?M{W$I~Qe6$un z_6~ERU+@V&Pr1&-Ju&+xe=WJ`Fq%4z)xUB=ccz_YRBwFKl=YDo8J96WQMy&5YIRrb znp{EPuklsfoo@Err&9Qm{!ht>6dR<$W*z41FIC~#;u>5Xm%JW6T5vl`W;Y$rk5^RB z;_mMts&e4k`^TJRGXQX@2uml-&5velkt)T%7$;lgB=KDk@Yi>aca1Ro72AP)o+lgd zs35y%*k<+y1_?y)`OjFGODl;NGmX1IG75;q33&Vf?G*AyF==&p_cy)&KFq1`1^=p= ztLi=o!ev2#B$|&1^V7X=Hn%a63Hq~~^=mu}$%dzO^$SO727gKDewSnJKcJNbxLwh6 z5*h#8?fpC6dv}{%MU`Kg<-b3zL=D_+^s7gx{^ROFtxE>fx_Gam75=O8O980LRHh2q z5dS(Be*sD*^1$6@&?D=^Uyu3MRep>Hqov?L8~V=x-hX-*3zXc-HI1MC9nAcD$^VY> zzZ2u%<@fR59pgVX(0}*kf6lspC&s@MP^QmW%;fy zHg=LsAP~r$>%J@tD{B({iv&h1;8PG273HyK8XJPoNn>X`Z|le4-}oQ9$n%dMal9rG zbpQh%hfxEt=GFn2F)=>wupPy!^|Bf8i5>P+_Sp>1p!_pcdsZ4WX-Uvn`sAW--%^-$ z$AJ)_sL93T)vNFr0rkWvMs2uKa&mG&!F3&^2reUQ2Oy8&mDv`6(@=@%db-7j`gVAL z5CL!^zkEhB8E;=7P|cCO8%9vG9r#gEQ4zpyHxI-TpmbF=SC4eECgb6un0#}-5`I9- zROWK|6yx0H>NGhRr&y`lq8-3RCsq9Sb`YuVE;>3oS~7;gXH~94TUbOqrT;^9G2BnY zujzS1$j)8jGvDYclj7#E=j#i4hH2i8^90l?d_d=I_$`hlPYjJuG+jd|Lhh2 zoc|+f+0AR z1v?`zuVnLdT%zh`b#N>~MeR#aZ}LYWWCK-_f`WcPK4MU*up3A?K{QSEyp7%|Ze}q$ z0c>q`O7ABLtt+BWQUn1RFGve0m=oEyLIAzpG)YZ+70AKC-$$)iWQc$o6BJ)g?82TT zD3z}V161vIJfQqwe`~o# zMW~igf<;bBLr-%qgpU5S^EGU#7J^AG2@Ljgyw3(Oi0Pu&n6HF5CQGz}y%FERK87I` zd4L{WW%t(QPH-opeQj{JKYl8H4tKqtSfHV49 zhXD;;Rvy7tac_FV2}w$lchohn1R>z#U$2*gk6-wpyY5bwg1RI-ozg7(+1x~)b~{p` z4k6EMLVRtHM%^Q-qZY4EyK(gIG6Ohqg1xM_uxQ>hZDNRxvs%v9=!2mZ2~J6gQOI^W zpfjDi{v6!k^F~*vrb1h%jP_`;_0uy}atSP2O#%ANgb`34Amp}royqTds^j|p+xynZ z?d+mW58a`XOPBrm8?5LU1&#twU?s9=^gdfEuVl_qn_4;Ibwz}(JR!1kb=6##FP% zeu&>O;nn+MjburNyoC%%`>_VpC3I4J4Ve}>n>R7{jYA*(#D?x-vV%&w!5ybo%gU}6 zL=yK$fc?ZAzrGT;ml@sU_E>(Ek#%Q{HD*~K1{hfJn@b!uSM%~+#Pkc z;aC&4r224UXd?4X#=>2b`PPUBzCB=N(0#tW+=*_|&*dn7!j&B&P9`SNjaC&}kOZaf z@gBdqe9|cebMl$Q^Nnn=SznLySF5GAxU$ilp(S9jHIm|aE7h38z}55Lpb) zgKWfpR#hAWXD^tGEiZD(6elv>{ppuV+~&P#UL58!Dq2H}jmIOwOgg1Yj?ioBl9k#W zXbOvu)4IZFc4&_DRl$e+d_ZXZ0+y_gcq(QJNzfQQI@*K3US0$k;s|7?!Hn2xqElsA z03iJ$-|O3VLjT-71P&{t{YmS^c5rQPah`YKRc>#7LH8ITL<8VLKS~TA*0h z|CByK64JL8BbNdT%HUuDwX_^d|JD4Ykzm6t~~Kl zsNv>RuUta#_|<6I8nU`&Kz&u}6}Ir_PIbq#D3TYY)lOM!bGnu*4Cnhq(b6+T@Mtk~jPAX9_tMhb)G9mB00sRBb7_?aXo$(_19BjWxIYV$} z?@H>0lhVHMytZ}^#cOuiF6pvnB%`c5+aAl)=|qeQd~gTc8qE4tRDE3~63Gih(dBH6 z$>b|tB_dOzXOynGuH46!qYJLNA8TRiNEsMdxa}08a+xHeSam+V#bE;;E~}MrmDf^?7wHB+xq38ZYSYx-xGRBwxYoOY+t>^Z4rg3Vg*yvt7Lvr^$Ab=S6lWS$dNnW5K% zSNr>IG3t?UD9sJh-G>9y#Ai_)NM6HuBEmh5p|kRMazi}%z^>|SAhkIl@^b=P?If+I z-O`YtW(#^N2O|@)`>|Kchh89CLZpp9xGoR;K{2TPRX(S_JuJnkKE0bxkFx-j<%p4!J+@A{Fh z)fX+252oiB7<1sZ6w3`gm*^uWaDksq$DsC%Ei%sY5|7Dxh1nnH-^E5J3Z@nd=X}&x zk^9%40l8-v2<*H%%!bXU%Hla=>!DEU#x)4j?%>h4d`?;|4QapmPhs1HunHsTp7lBKd^HqFvpwBSvfmValqaNYFY)lyKb_^24-|$&s%4l1&U-Idu%Ns zJvN)V^OiaJ>FqWXEOhcS4jWlxkx=uG54{*`SBawnGXTc+w77}K7}#VoP;M`4n*H6OyXAIE1lF+(c$*{MW)2)u<+wE=ArG7Yb!x(N3dk}oN2BJLce$-$m!@;D!=9Y*4 zg!zjHu~B^Wu0w&9&Uu4=E9fl~mq4XwF2LVwGA0pCgIBx=7Fz$HtSU97I6BJl+!QRw zV#<)=}{UEcW9Gw zwAp%45ze0gqAEv-Qd<%~0~tph#ao7d z1_D9iIZ4PQ;?rK;jxxBYjq^=`T+H{Jn1Lc2{O`M57fU#C{0OezUqD{56kM-bJ?F?c zd?4G@?!~<%ZQM?ReTla~kw{+}YJRDPwCKi)O_#d2q?%l%!9lf5P#y~_5kuE%nOcnv z9Pn+DYq`bPk$@#38>%Bf`HK6z4h>@}DYz*Ff z30donKCmcjTu(4Ke#Bv-BLO$Q7Wz19)OCV61Qi=8ro5(9>+W(wV=Q0sK;I68R$0YC zO6u#MDRW|NU*KDsq;tj5E70T;{ByfpJ$p`fYTGDzQMXdDthotZ&ZoVcb4)SmXep)VHiEtAI1xm zA+6lBZ~3=hUgg}*86O$en82{t=uQ9^#Q8iEWZZi0cM zbPUdt#!x{wxU9F)a7fSvm4ww8DEwtVCKqK>n~TOkFi{G=Q!shGuEyC`ebi7NhjLJg;@H#Ggt^_OT3|Yin+DZ@k~g!$Ud^<(y}Y zXj*U!^s#R?HeQqB8`=S$eC$gtkzVK_veRWMHeDgyZm;}3;~sIbmOjkuTy1m7Wa!1^ zwvno~@GPlfK&?#VFlR_Pi9Gq9%ykpjmgm0K3p|$5Hw*xQu^*^HX@~9mNxBN0gmD&u z05zz1KZN*h8&$C2306P9zSjJ%&Y$~isFoQnAvbU?)UoV`pIeLN?3Gz8IlyX5gmD$d zRrls3#H45#g!pXYb4!__Sf$9(Ds zWJw2Q&PNK<3FCUz+%P46Q>%v-KY?H_-*R%=^Pp9rHLrWM4!8=T5b*)?Dsn9q+jG`f zXkC3yMI{;nEoEJ?dZP2RvDlji=Od0toc6>sX&*0x3BmfK7c0=Ydfc&Ww()t-!ydZR z*S0->ge+AV$~Sog|C3b(VS=zUhGTQ~qKGo}nlf06Titkbn6k03jZ$ile1UiG`---0 zptcrFGOyrgM$K{Nz6p()0$s?A^owx>e^%={>U!9E$9+g^!({w*m!NS?e^vybNBJQ^zM%j+O5fS$4UYcIzw|U zQM{|je-wKPgy|41D0AJ+yUJU&+XS}NFFD=U zC5HYL&`G)%z6UpT&_2ZONRaN{4Kom=*%wHs8Z}6h^Y$};v0$}4({Nl0Y*&7MLKSGm zSXVNO%$JIP2b4*?i7bfR?c=v=v3- z(&I3jgk@7}ZTCAt{XtlI;X|Ez^a>uU^>Ue&)&1)9`66ZyKRNCkyG_ooz~qUl+m>PS zhd?HkyM;S-Gg8|_eAYmmN9WZ`npI}FVb48IrH_6jG8Wvqd5GX_0bK(pnES zLJ=Tu2gRLVCr&*?in$%ysDw6F-*iVjl#aac`H}UXnXe4O&yo+$;`s%Ib_{|WwiRZ;)?-CC8t(|56{%iT7k%r>h{#Q7ib|pcoeio@ zV&HR@?vk^jd%^H7)5sUmu=gr~*St{W?;Hvv=w-b_@@K>4j#7acR869B%@6065#^xLQ;M5X2;{;>y!EWb$Rv5;2M~)OG?1}4EHr)EnJ|2P z-5rtP+zp~mLZJL0e1Oe(>vFJcX>+^#6jdE$OvgQxtg#F?>mapIC{knI_yEi_I$Nqm z0J78Jcl6>T-?LJXybStI>khKw=%L7oz*HF^80&h_cB-;k87pW#9JIw z98G4P5zo3IaJ7q6^%=;{p|cmGz<$8IKMzklX`BxZL2P;_U7nls&PIeX%_b0p7cF_h z_w^6h2SHa66W1UZ6iTV6D0@sAEsHEwf3TuSei);wa$zCjI0b4h4 zj$eCYR-R~*&7z)dw~AReNNvSMXla+9e**I5kG*+xHVBY z^R`Uy;Bam9a0Aum_0#=b6SZ7vQr1!DkwNW8K|vkPZ8!wSm(>VX)YHUeR<6|tN+3q(x@UK+j9B`{=K zpc=HOI#6g8*VeL_sY>>6UXQ07&%t6)>*jssTrL%VNF598_70m6Lo7O&y{Uq>4$_0S zUGlrNC*94u^z4-8KD}lgtw~OJOP{7Pif+G&+kazOTF5}>9LaJ_pwoljt!I5ve<4}M zRGuCk`LR!v!P3R|cN`5mbC!lo0`sOYyp!p45y?=CXj;Y1sdC9sJdd1BZ@6Gq77743 zu=F^w0JeA+Ax{PG&2eN%h1Wwdx2wHv=-mG8fTc+o?$LfUZ8NV6Ti+k%v**=wb@s;; zL>#-sk}+;8a6A^Xk>rvwKfBQ8n>{`KQ9MsYru%YI@RnyXw@Ek3^l9NyiF$NuA3TI^ z5a3j;B)~AExSsCX4RW*huT>ZWy}dyDxX4Fa`G(lN>ABgey(b(7e?oz3q#S;aS}COQ zZf?)aC1X+a$)}H-K1M`NAd-op2oEwgosT>lHsBHLQd*RMQrnoea(7t+$@S$Bevu;Zu!h>#{)FuBO#Qly_IU=i!pcLbV>FocdygkHJtxjzo2yg& z%n_Wk?R*-0Nm{FiH5A9}|FSP;pwThonYG>q9C0&lciO@OXIex;2jWC#0>bMpT&6{9 z*qzCPQ$PslrCs7&C~!OeUJ!rXbj73YFh#zuuw0R?_+z(?fmKP^Hh|1ImitAJ#)c9v z8^N88K1BN9cjmqFHGqpU?F>r<98#HY- zg(Wv7uFgF?_SGwX2;;wLk&{`O=)CG}_1g?H4)uJPa{wqHqM?@?qq3-Q za9IZ_^Gb8)G0r09I{ozW)G-Z@;}(E6fZj6%J2>sEYOB- ztar1@VWoz?@{+R|V5&vv#U%I0YxJ^8u}UQi-p%po5S9J8)#sHm78VUpU|k&IlO-9O zDY-v;!MX#tnG5M0ycAq7d=oDj!=#bo%a`0N8frIfz0x&+(Eg@`?%P?>M=N46jd&sO zh9zudYm=aMQ^xcKA#X)Ryy z+pcfe(TvNpnfz!G#jr|kW%?A4Ej16iO?dqQw$4;WCgyd}4sGIS&|_Yeqz$SIj)@hZ zlwCR@<`_EODt8y%4Z*cVkEbzQiG#qqY0Y7q0qC_yv%GFS3(V}w&Up!T5dNwK>-bu)8K3xz-Xjv}1eYUVLwyIKx z)x3$I`S=vW)av@&aF^U`t*?2Fkd=kad>Uwg5l0CZ$@<0tM05JqAGjO3R^-xhQe)6MGwO%G!gm?&G&*Kz8(5tjk6YIQMs9dfBC>O{t zrJ6ts6m8vc7vI1_QS<}52?Vb#pKDPvwHH066a)#6 zh!_stdW#&+ca)ALf7!BJ>5~4#=1)YqGZ=^LupKq$W=FGlILq34IJFT5%>h@eiH?Lp zH(xHOnj(qNkZF`u+YtZD0bXYVt=V|NV@#IfMkU6}^OX!`3i)sEdzf{JP%lZ5RrL>7 zWqan+JD<*ThIiCdfA0!a-e(BSUFWnUa%UbA6eJCLN*<=mx}Nb#kZmAQ^L9Hw(&Jp4 zzMg4hlor1+?hl}ILu%{4o{xojJd!X^fq>_b+A4KgVk<7Zs=khPrs3}ctrRrU*uoF_BijDMk5$kV2qg8dHnQ%K-qRj_ntWOQiz#})@SITY0ydj=mV=38-nk#jQ72FCgI(Zw~&b>I?L zP_3xfhyjl6cBO$sf?ohC?~L2s4W0pq#QEHQ5$h>U#-Sg65rtoTq&9wq6@fC{wX-oB%Qq z1vbaf?qz9?OHFGL%+%!3h6{a7k6NE7!MldO>ey)i6xw1loji4`SLOtPYRgI`;76R= zfJ1xfWN87c>!O!Mlm9oWRe-|s78*`&57sSH=^Yt(^@I@$##a_NeX(vjz@AEcK4NtS zB{av6z@fYV3QZ#l)v$UfA5|oCrRWtd%lk6Tlig_ENGbze`FMY3eNH7WT5oI z-M#(2n9$-PdB2A8{`}lWxkM!Y#<9kcp5-?G)vgIRnn_u0b-zBd#uFCT20JNrJbU2ebS&=8E*HlNGmq2G#Mf7Ia7ax$@ef^B2C#w* zFh%A5MdPzK@B?Gj+XIs1f+5iD!C+IiN#KKh~ltDNX1{`f@mZtqWmOj4mb!hWsm;YyO(mCttwNmTjoph=HTCM3sR<@2`2QrErjrcJ*^Dc0tD;Styawyv+0y3%Bhc zCnO=rYDgy&y$b2gYXr1*)rPc00sb8->IG1Imv$2^SeXnN-K^RE*0VPy3n zhZ{6{LXHnSdqcFP&!@s#fQ&w%m0j^`E72A3ubHrT$8)=eDEaR3?|GX}X7%O!Gj;UZ znBXUh2EK_Gdxe~uO`7yAFmN(F>OF}Re%di!jaL@q<6@^%I_c*UgSOlUmA8AAvc(!V zWZoYNRcha51$DhhVFj(oQl}`bWEet)5)YBE`%Sz9>^LDf&Np>aZ@whrUFZdflO5U| zPYkEs2wa)X=cIacMfB0BrE=Rc+_^!-RC8jM*7JY3;-b#U1<^PW%Qe>BZCYk{D}Idg zUrCF|_?)9>%Ck)*$I}0v0}1r``e={M?{d7tPnct#c+}2of9XD)4s0;tN0Imu{SAj< zt%LMn)g9+PWJ|i{|0H5JTqcAVB#Azv#417evu9444?*WORTwwxe^%j-;p!W1&J{F{ zO#AT)8M}%qu_f7!4{W>lN3rr4*6&(%0JoPOi_QUZNlV<>&H9R$2KK2fuRu;Nr*uJE z67S)xE#tds=Z&%DhTIk%Td_g3zO;>js}ZFVr@*OTee%_}*a{j`eQ~f-svqYj)6usq zuDJy?nNbkKUL68CtSH}_6AnK-Kf$D`qyrz>NH-P&B8uoTah;U}CZwk%)M~lI(Md(0 zSz|}8E$_TTVz+V8d!E>ys3ICzkJatTs?v;v9f-iFDre4j~0GSQG$verj(E@#kX zRfv9mSZWd%rWCzf@{qy?QXC<#TTZKKs|}68SL@BbXItdioyS|s8&Ppbic$`abI#*`8FQIt2xz~uF}UdY1%B)5ulK; z9>m8td!+eDo_6EnCq4`+ylmQCA5lO1I`v63!IA%b902<`Y_DWUaS7R$`Ol7*IZ=1K zL~2bl$B{E*d6~JilwAgDr`(q4$X2E{Qt$<|+csN2UBOmrp&Jh-F;1!tBw>yhZ?Y`g zyn>+VvU+5ED#w2tda<`xO_@l4pr{`G6mvfnjGhS5U!&J>MW7?$uw@UetiM~cnNX+x zoeYwaf7TN<%U7TuwgMb)H);si)2Yx4V1N3PS!wxUb1^1$SjiS7%sp_!V=#VqEMGF9 zU2>U{(l9A5(n~qA%Mf+IRX-_ye#IF?1yvor(FtggF!xF%x zZ0dv1yTMIM;-oO1rfu1CyhdPR za=pH%1f^>Xt6(RL=LHE}Rz!dD{Scct{@;dJsxR<+p~nr<`4zJx)J*~Nz#)DpQVRxG z44aY|-a^*CwUzU|>#@^hX2-=}Fnf%z=U(Lik3++pu*cnvifzgZ*LZjow>IJFg=>+G z$v3vRsNCivoU>NaqFusGw7hnkR1MAqgdIA9O{_nv_KJzWU+h`#=+>t70kB;8*cbCl zXNBA8T#}4gAIV}-{lHlDzE%h`vZ55b8PRyR{pN6(KWk!(`Wawo%E}hENj`{~2F~qR zN0pCJXuE#0J`$N`nPXpG^0WBad`GWVV`Y5?y?Kj^l)7*p@Z13D7{2WgM9I%(`Ir;h zn9-GOL@|{<#pnjP8*%|Nl}b0GShBC6(@C44&kHh3f^+wS&Xa*4_BKFbsHTX5*5&U_ zmSQrcaErw6UuI@Sw-<*!4*kQ8`}B(>-dlS+LAKO^%_lSBj-fvi;8ixAdm87=F@2t} znY(PQ_Ka>~5W6vr#nw3N*#cwnuX2$h^zm>%H2USCb(- zn~SDA>(R8^u>2#`L#m_6wEaH!!{+8hQJXU%l8E^l^n!TNh@TJQa?ralUPyRYr`=DQ zs{86zyZtgT>tmIugzdy!{_}Z7WZm5jKoCd9Kka6pU^|*8-_7L(9?hE%YbcDUv6gVX z*BtIq_m?CZ+;i-tWW-5KnsVuIrCLJ*^_PpSH!$|qEdFaC!KK~`6%RY8vR*OJZPhti zaM_VY<^M4^ZUnQtrYlJ!{RSIBi~lfO-sD^0 zNKHO)A*OuqxV;0_#<)35;0BDxb4+bp6^d>KvYE$LN)c!JG11&ii!94h3}j50B3dMJ z9W~)#$%TP9FYY2V!UL2>;7uYn3UR64IWUK@KAW1`KodJeA0QF+hy`_x`YX+%-C=5t zUl8u|4cNmEMQQ-qKM8|NzVb(PqsHgx=ixM3YJ0B9^)oBvf;^b$;;T$&kYU+7s&!vT zNy~XpkDm~o()hl%JrE2~y)~HqQSY#)^Ic`Q-fK&!gHE~34R8TpJmZAE;dpVRm?W)F zGLonoR;Sx~DBx}D*CtR{X9}AHnBUP`JOR2V)Q>d|u_(C3hWb}-!u^#%W+z_Y7^m)i$qw5(CZOj@!N4BIxAmPfCdvbhj z*1N2VuM6FkzBS-*s2H}PWHK5K(C<^Q`oXlXyo;zUV|h8`sliF>{|LF;<;I7JKn0Mt zGj_kelxi-%3g|8*#s9nSwnl?EY9N ztEBw}qv!B>IqdPIt#kzHx+f$!Y3j)^d=g~X$m~$->$@^LlHi>%2`NfH?+H8`+xhqe z3)PeT4rav`;}gG`Vx1p7M9wj(?nAx;iYngYS$1!!P&~t$DC5yXi;FG|tzX<=!^e$AW~46P|9#&4h@EVKLNF3ks;tQ^I{w`sE}{8v8* znvO^CU+Ct)9!MCIa%roc9SI68b0{pM7xNN+jwWYH;ob7()t?jsH9%0XPs?Pxud;o9 z4SNm)OxGC86prEzyDomLKPzc+-!BiIpMu$w{Msi>wm6>!aCe)jX76h`{vvs3L#!o1?M#1F$Skf0&Ne7`-nyI9Y1Hf@B#zr2!Y`1bcq{0TNbC zo}%`J3J=C)z$lkG{1-{)B@E_gMS3rMvGjW&99e)WjzF#a3e1Pq@RAcA1lSiea4A79wg1B;mA7SE{&q!&c|E^bY3 zIZyCUC)#|GHbT*we}JO4WVzuG@73#J@LFT?=GE zub7KePHtu_J!>q~*8yrW#I>v#;8{I9#?X| zLMhTr4LlSJBN%Xpt?B}OAwU_}jsfR43RdQNTDEoc!s_Q@nyf%PaCGH#oExjMSqtKh z)dp;E_$K=8?g%oDB+3V?tpUIy){)@{k!?!tbK3wKj1qPsar`4yYA`ohKMkbbjRDo6 z6ROj^bES9HVSl`Bs{1LN>kg1i@jBjCI3GFSM{alt??~2i)2`EB&yu9_@EujLf2hA7 zo3oM3k9$A;8w7icg_=CX z$I3vVPa#Lbyb)hrm(Q-t%kw3l9n%m#kKb)W0sPrmQRB0*TVY@2C7c{lOxL%B-jH%g17 zku(hQpd-|$lCkqH60`{zy?N}iSq31s1DK_*@6VB}KI4EG`eNj10@LvV-8{HzCrrSa zzl&#kN|OmET_FC?OhF!h?BG44jJWCu<{r2XKlrN$ayo#RYhA|g?IN^~2KRVC= zb_XKKQdJ}Y*o)U{Q46r|`?1xq7RvNhr^@ufXgWvhgyRMmP}$$z(iKw+s!cP+Dhy~^ z++GHMQ`W3=-1m6j)hAbBB)2~o&?dkQ4x|Qy0Y9Bi8PuCiTouREKPq_fqN(M3arPUL z^CKc6u~)t36VcU}Dx%U?-grUr()vIH7@+i0t=QZ$Hr5Zq8FG>TJcr81%E~!9BN5{TTqhP}JE2DO^xlIF}-3cVxRT4sSE z@y%gBt89TRwOY+*NAE*}c$PJJ2Gfb6=(+rtU2u=%jv4B5i#RhlcLiK$)Y2f?8Bn<) zHMJ9V(z7d^n@gf7sfc+WN z@ot@y9TIYJ29g^>{sjGx92VCzK%(Y?iHd(3PA=JWfI-X&?fF73>7LyEQ}?~Y3ZO8^>+4IphP5DdXP83HOXL_~>pU!wvGx4wo`4(9ck zJS@q1*#3Rsh!+Dv5~fkFW0NXbU|IwYfc5%)<6aMmdk1}&>05C)Y!2}fIY6T{c6x8- zer382WIQ&|>-cS7=(w!KL&#HR6uTo{9gs2~ z;4POzMCGW9PQHDh&G-)dOhexJqdwwhpe0*50lvxI;!1UPZ|aT*lLgB)K8Crd?H1hW z&L>3G>OEdrQ|=zUF#RE(L&#|{8^V-?+n4ULkBD$)y&BDJHz`Lp6To5Bpj5eTjnRC) z&-Dp61XO9|k2Bv8H=Rz;_>7L`1Ldl(oGdDH^|Py{XcUT+1<<8VCP#-~Bt!Ix*^37u zs_^xJAdMnMV44EG@?bD^JphG)pEsxWj0aM>mFt?xTDZbxQ!r;`mHM>txXq8y5a01I ztd1Lt=3VM2|S~MIasMk9H zpEf0pAlXEYd_Z*Hjl8@$b7ZfLVd!M!X;buW?7$-syv22Wqjhm;>1-K_4C1jOrBd(4 zr+Gl{y4~|CWQ9ec%VEgt2VOsJsTin#^*F>j9`O_TWF9y?&Q0>Mp)p9 zC~0I!FMZRfZ6BV?D8dugHXi1KJ`kjllGY9da5*L2MSz_nF4*CTDO1vncZz;RZ!a0p z5Xz$2a7+e@5a}j08hMU3hpwDlsSxUU%r7|?+=#*F+D{=Z;T7`%UDQybX-J2q896bKAqjmm3w1njpBpKVw4 zff-a<{D+fp)%-J63a@kH)k1QkNp-R^UEf1|dO9}-YgM?Sr2qHPDK*>)uIj>t#=?mC^|FU(v2r09C6!`qw0@rl9Q*zcZEpco1>5y| z3!>89-H4QQ3P?8+(%s$NDBUF;(nvQ*OE-ej-5}i!=R)u2e&Rjzp7Wk>zL`A^v$w$J z+E=ahU%&tItgXNijExk#NQ`v8m|5-oAus>e0xW|jkEg1&-)!84^U~_8YKeOvu|S{- z5__2P${M4c#+xkGUegKM^d##ed_+f)7{Ky zooI_Zh+ICW930WWasuyUnH`+khN@By*y$!!ol5;4StB}{bc0murlfPqAeOmichOmq zwJvd9?|ahZzWCKrZlwOThQsy~4*6aqB)p)nBD&@dlR;S=BLSzU-4IU+Tko$orfyqW zbPKadY)Daz%8H#2tW0$EavN1g`qJ&)$|@S9n?a@9-cFgafsqwFI+@JsOC{WAu&J7Z zUnY~MO>s$icP%=`AY$il_N=N9(p{=*NXl=i74lG~3*-bd1V8mMkI;e&2yYhEDYKh) zIEK+ZODP>WN=l2e&;nNJBwfc5nZ&lsm)&wxGubH^PYqAHg&1f3D2V*`gV2kP_O;x3 zT->H#{i&aiQhwap=-w@nyL#O4sJ3<{>xvy6?yVZ~y{>oMK?J~<>|jcwWpyOa*Z>}y z=H+?<)lt2ymWOJp%Pt)`7pJYZ3*6E6e)BqNccS;jJhcVJ@K>DOqG6D^CJS#p!kWEA zW)jWemci>^E`O=67Q}zc0xL&dM>-aCN?T;4PW_;gFu*4xxsr50V_vq_D7_HEp)#w{ zK)V<5bw7Yd>9qOc{9MAYo$n!aUF!U<(+R&zgDj(ZS~ESSBq^Dz}g=vjYq5-6J{jD|_!ndH2&@&@jHk;>mA zM3#&}ARKN|%V-!tN%Uo(k?3H5ZS~s^l*X7pA!+BQHgOSs4fB52Ws7JG1t7cRTwra-v8;(ZM!{VlJ;aCN@<+~v?AKPT?5YY$nJln8A)1Sa+gz~hX*!*zLC^=zV_2UE0*~PSJ(W0|!0kH}1An_%Oym@z{oX2{ZcIy%FK|twyk-z?yAdRq)wdW7j%FQP%pji+-amboGs}p%W*}e{n(@{1y z?k6@HsMcD1bu6*}egyib!1u&oliN=A5-N03_t!>&rVl;`P&1O&ZT|U@=KT!6HlV^VvY1&NJ zx?N~w@6EG%%y@H}2BofbK@8|=4i9a>8ROuL0DJvI7&H+i!a1sfXOhD98y?>_%r@{B zey-&#h@FVMVe3#@V`A>q$NxGKqWiuaX3iKdg~Fl@(91_wufplBTn^#kME}xc?R@!T za4JgFYM%6;?{QzrN7%lq#5)1)k(c$$J70KdQ*A)~rT&aRsy|E16txW?S8Dol!(a)A zdE-!A{-hMwIPb!R@MJ5JuEK8JIS48X+%D_2TS0>eOi>-mfkaFnnjZGG^f(;gukoG9~sn(;dNoLv?MWQrIQj-q}jIdmXQ z8W>2eHC74+Mzp8&jJ~BkU*#6GoV~wBNn5Tke7qAE3zzc#epS&I+3 z-!Q8I@cPHT1ekkvgTYJ#V{*0a0-$sQoAcNCm zEFS0vNKy_TAMOA$?;i?kr33p_E^oU6iK*T`uL@# zv&K9FP{z)RW$9KRAXOXkS+{XClEh2mhPbG}@#oPs4*0!mFp`Gj8{=It%{w|19n_v< z$3&vZ(iHxcYjhPRCgG_#|G{)6!~3Z(G5Wkz_h+N?X5&Fiw#D?AN(_%ZkW2D`xd6$b zIl`r6-^u_XbTYG*iwFzfIlVtoB)1t^B>oYDeN7$Y(+E4YBMk@k8V&Ee_fz2;r(*DY zv?{dX2g>3<{>VIK+gV)!2r2KPhAaR*$u0?D86F3U)0N*gm0@>Jjev*%hiy07ywr4~ zS#nCwxzp=(^aXwWXEr*2QXY(>FF7rq?#Q2!%To7oqoX5L63bqWeo!d~Hx|#sB`@Cw z^`s7ZJ%&H~|dT0!wCj~ouFobalLQ`Vz+oW6hz z=&>j@Mt=y<0XMk?tEFyZw6g}sBV zYOz)pqtDV3=4R(3zLZxd5s!I-7up;fKsmC;2fNE!wpdiJL#=tHG|000tH zhb?&~85$zOV>4pk`_@^6xtH0eTFwE9s(uhYVzsx~37mHQ+o(3FmM{C$nzHv%!kx1v z;rn`m^Wp4s|D?!+XnZ|D%`)MbS+hRD=KbwT;ALMqCBL^DxW@)l~$v1YqJEFee`ks9^ItP2K&k$GLphK5Y8J{p&8`Q0`)>*~ z_{n3MH8OlVMpLiH#Yx-P z$m)wCHQg4woh8XWNSM%7&%uB=Uk{{=@L31lF&hOEf27=>WK0hAFGD=7)Z!z74~z#> zBmohGWp9o`HzHi#n%m-}yk!mucSyp~E zKFewqE8^OCh*G@qMu_T!U| z8P>u?zd4N0-rrxHS3(gyK3X-+eg9Wr!VUw(os^VB#e~OTF$WqV9^oX*43Hwg1gca% zSKiHS92ipoq#0>(SCXL_0;=}a&tlb~@5CMhbR|Nd?5ZmS zV_m<{%AQFkRf8vK)MqP(&N4mb9 zsV_wP`z`(Bi=cGyleVU*^Z)t3{}Oum=^wR~fwuQ)|MjI)FbpCiGMsZ~Q<;Z#q|>U^J5Md_FyX zb9$w)Ac0!3HEM*-OsU@$@}7;oY-e zlST`*iT;aTV*vD;v%Z9sgxR`-%dPC~pY<+Blc8ANv56yEFOUeY*J6_-6TIxf2}z?| znrP1pyuug{E#<&>RV?$s5F?4~0vg>u^E~ln@L7_eVzf^Qj=)UfqQudJR4R_e{cRcO z8q($7b~0RqaiS!%0J%EKNA`-@)=lh~EsK1_9unp3svvi0Xfvw7Ur_`_%oY+xx06x~ zZ^?^iHS7nda0ui9DSkAeR~guePLG$PHM+X$L5EZqnF;e3b+^ zE+znYyJwAupi?uMd=~@>v(ooPMm1*dW#>3@1p#QkN%Sh7zWFRpiG?Cg!8~1J2;bD4 zvKPJjRVq?YqxU40ori63$3KK=_(1rA{$6CK2oob!OwF zH(atErmnk*>^s3Wqglc-8ZH8v>pV`oQy`^wvN>X0yOeba+GF*{)5XRs4Hu!i52^?V ze!00Yc9p(b9t-$dZFX;WGO4XNbUg+X{%od}^(^5~w*wLM^FG(h!(~=;Hgp;%H$YO6 z^uv@iGLY>>h#2ik^c>NU`QqDo9uV=JD85<@VHNap! zmnqozLzb^k43ER6vu?3;f8lO_@iuQGk?}$6>v3bYXcheT?r`tf5_M7Q72S^e3WGlO zg?LcW@qg5AfWmwG|0dpiOq=`Lv@o++C}jcbwcW9r?pm*J?-dBLaI=#)XI>EPDpeUd zYP(_0OUDnT)Kx;Ag6z}ri$Ud}i!nJeGAbNqru${qnVVCbCf$y1?gNV-4UUrgBK@(n z`?#KG6q?5ze~pnz*m)@^kK&k3)Z4~T z$dmHEJ9D1?Y(-P)3rx_`L0`C^S~E|wU4ZS5Myz%buGn z!S7A3KQ=QS*u9}utWLJFksgbA8Bd58Kwp^giq$9q$GiK%I&O07L`$|&+tYeh%eWr*4E<(?AJpyy8+*SLa%$!AV z9>WL_2R&s*qQGNX6Q@I=K4aa5e=AFa)rsaLtyxj5N`z&-8w(eDMMP8mv zn*XQvS%tF0Ci?Xle@oD5@HLwUCXEujC_bb8CcVqqcwIB~G_~JNsQ0w1LL}8N5Ppia z4_a62WI80Obs;0{a9imYlq1?ZKI9ryg<63AIoTq2u~=DoQsnHPW|(6w2ahge;Ugcl zo^8Bnn_sv;k~eErt2O}sfUREvuGKJjoUY>Hc(y*w^j7-tXn`ojRzSu8_K8u}gv;7wKbpCeOPJ?Lke8+T(ZNueO93=X)P>2A#gh@-+ z#W)G9TqERu9WE317#bQh4eOs?ES6=)f|jnRfZ+gI&m_eoy0j?gY#m>|3@AOm+e3YD z&WJ3+(HDeA6+^oK)+qoU$MAUs-8WDCX8}INJ)4*X z?8OeXAKae7u>pb!m|$*`)Jk->Sm%ChW(D}tV8`hiDBqeA!?PhG*uVRsDRck1>2gIf z)uU}{?QR2f)AV_!UaEo9`4|~UA=2dQgZ*P~4z**NdyI*|9U2OMlS_1-PX5AYwHObqLlU6P#Ll@BO7AhCO4sG2QDmE<6 zy6K2eX_5Ots%5V>#nZsGp#63Ie`rJ5GJmw8;${cnV<}ylD|r+earjAxfzh@n(z{y= zRi))teL;e>g8|7EW4AdFvBJDs(%OiwGa52$c5^ILVT4-(^4~2moZNLMdh21q z8WIG;_Fi$SHFn%(HTJ`=JT^;DB+H>)ye~U~N5$axg0u%{O>#nq)9;>zt~FEZ7jaYk zSnLi@d9udk=ykI6aejz98<;Hzj0z-d&2luw5HKU16%l&2=$vpSFF@Oo!@J zOM+7-clZs}$eD&hDb~Qj1n(ov!t8OI_4Rn796O)0r zOaDmZzRTd?7?HkWX#~(ZcR%E?VrgZXBii-3skme68Ujn+WZ9yh@E&Nn3yqK{c|PJasJ@SL zq~Z_PQ;lZTG%t!|+kdcKk$T|(Lj%9-PQF#4^l%MZVHT4K~wv5~E^S}nionX6Gh zRzc+7xk2sGxZt$ksEUU;rmbbiq>a&V);!^I9HuzD+B~@;xJ2uCwrCpp&+1ZA|2!M<_?HI7mw%)K9Fr7aDn-=QM1 zJDR;kUTr!~*67?tFe<4Qv=>R`unOvoq-)nPgAdFRe&9!*0}T1V*6W5&CFdSh#w@vA z`h5Kd2U8*IWR5eg!l32UU9p8C<*gYqTv5h@lSdE@f91WlE()?Q4;AQr)GL_8<=isR zvedI=;JM8ZXp=e>eb#|wj&cw}lYF_Gx-rCXb38!NbZj7==*nNArZb@FYKZ3E*Dnsh zr{fp37R&+Y~i^1x-cjQRti3rfZL17T=j^2)n=(#Ux(=; zaetnBhK&?U<`~osjY`!*0WRPP#iT!0dQY?foBI_1AS z*|E@oGm_#<(tto~YaZPj)yaR-PrJLI;qQOYPvb3hQCn-BBujLi`HdIxlGyt9^UnF@ z?>RP&%FDPKO9ndeyjK}&k4Ow6#GF&v(A#z8T3&t7>Xyy^JU-nU#6pTSPzb6`(!B=~ zy#GazBF0gtgg~5NOQ4r*)Guu$iNx?cXl`znPdXgoGvq}O zlx%2ug6%JW{=^dWxw9guy9=$p-0)mzc8VQ4;_Q^<|1+Th4VAb#=7;I?7Y!wy%CK05 zsRM>Mt#nNqt>duCHFUv~-F%1a#)47KgJSb*>ZRJld0oD8**(g)LSPU5MP67RbtH*- zBz0sLje{zoJ5&1V8JrR8TcL)kjBr}5Rx78NwLB4C93bg6;oq@3n6!@Yrd&MQk6OUP zO+MO>@+}^XPJA(}zj}41)h*~8mj$~Zs#vzqcRgFMgh9Nq>|(O^a*cXAIif){4xahb z5G1%6>7)sR#D|M)kE-Fu?V(JX7g2RVBwzZsQ-R3#Gz=Dq=s$Y_iLq1Vw9CbeWjyxt z7!!wj^C4R2c}HcO@&%1u`y(x`;gIMEJ!=e5D|E-~Kr%W%w4-&g(;r&y9XVHHKlzgI zalkG%k)7v1$%A0P8Jav6CDhpGS$EUH=1B@K*B>T|K~;(#6sep(LQy~ef})uI9g0c; zMin$9ftBS-JzCx5Ix~>F-I(EG&c9`@t3lKr;wytt0kV5~w>NbEkhy|bn4W#eGX2qg zD~Kgyx>Al@iz1VHIc%7qsTb)|;|s2p8TP~CGHPXte%asWl?9e^U3UgWGQd+T$>IwGw!SC;9p~&a~Xo;n(j;4(4)_rt>>#PgJOCh&-Np ziZV1JK|bz#B(N}^G>y{%8ggNX3d%uDf^hl=B8-#MO_@*QJTB(z0%bZh!RF~@NBU9n zp*lo}i(xv<*H>f9(f88~w&*civt*lQw#zBF7R_e}YE&!Xg2g)r7Visg|M58s-DJ>l z^PqSV)PufV00quBn2iA-K!f@ZOe)hrGBhK#=@nbe;Tv}KGzJ|YIZ8h(d)5ew2KiUN zCoqml-uuaB@TO?6n&r$k0c%5v4;lmUbakbUh+OwKiy52_Ta!p|(*PQ&>_N8;sT{hW zP$xRX;BenYs6TX=>{Vx42BEnH}UiOf~A*x_n+F6|!$bDpHO84CVx{O8{M#xhy z`wlK1cXROS!_fF3XVVv3bmGn}n}dw#{%z6}_dWVt*B>B93gVTxUHLrafbYu%n+@N) zxE35UFznwBBi}KPx%LU)U7m3z%*^*S#u<&(o1;g9n?vJ`eb68DEQ%W{W-!QdyzrR@ zC>QTbunSX&o=*h+6R<=dDR>WWxEKRDoOUI7F6@g9%d|8rxtshXR;j2O%?JD1fQYr6 z?XQLtZ-gnTMbxZeCeCZH?x;Rs$I5AiX1P*Ac5+45PtB^THEH z><9-^LQf*^Th+!mAF|5Bx%LT(Ex-ezvbD`3qR zFWGdSc}%OPe{$3sh`TeNY79Ph$aP10f1+=8I|IR1>+)>jLi#pFUn;!!`lkecukfwE z4ktImVftMycP9!1tOKUm`bb2DP|2j}J~V8iQ4OrbH=B$RReGsw4!arA!;n#1u03O78>MnQ*G=ww`y(X#4K%>Ik$)dqEN{INdAZ7Kki&j+*!T|SVCjknhBC_5 zz*b3X>0A*j$V0G3G%`?HA{B}9``44FSI35`mwyaPwI|(7@{om14IYR$unhKZd|1(WRtu3D`>2df?QF~>kW(2h9;4W0=tGLt~FC2$g z8NM{EJ*7C;YKTOR<26LYEqkPlc$N+`8b7lbRklhZT$cs8jRt0cxk zU>UVC=~8#qO!}9kK}D=-hA^4Vv4)6(i^cpMXQ!rfE=ZI25dLFN7#(i3@#0iW;yveYGoM&qf?Tgh?Bpb5WGDz5NNe%nVLCjQ%05jB5X{ zNh?sRloBt1a{ue_-z7L%-v6-#=kzZU+*jO32@Zw)zmwpK|5JiX{UK`EgivoTgjqug z0ADyRhOc#v1PZ#wDIcDlq}08_e>P&G_uvf@se}8$>jTON1lMt#2wl)@uE+UMxGeQ; zH%tTly9i(<_S|p3%zr=UV5FVE<}95PJyGGLZlRKnDKIrnnKh~#f}oz~j+DvYGtCV< zPv?gW0gfFRyM_zd1lK;&5X&+_XYCh=-+tW11AY#~;9#-xgm44C27d)n;gK%|DtHU1 zwnj3oS*WPAe>n7IF#OA16?^T}L%TmwyK*ZU>6c=$3+2uiJuMe_xUh^=3y#hYhIfQY zqM%0zq+z#ov-u=_Ez+}QyhL81!5E4G&^<@|gY9a%fgsZfuX=z$4tpKGW6y_mPpD0J zIg2fD;6M)NNzVd5DdWoI zz(jnwL+JbIe#48QbWQRy-`BMu1+HDJ5`Qt3NIy;C6lQ$kXtWV8@oDUX7uh7{3ZRmH z2n?jPeSS*@tWmznbu<}Q(D(u8YrY59`C@#+*Tg$#Xzey8s&!VL+iggEb7&7LDo3OQ z1erJajj6iN;F5_Q_TE-;hy?zwjp4;(TxUdm%`15kjZQNRX(RtfnN+oKX7N+Djlh918 zW~s~adD16v!>?5}SXMGyJ+9DpiY%{4AhAmq4k(^WZ~vuat`HYLrE6@O?$Uqcv?v-I z*%ma(YV!Q!;UGL(X_O2s^R1`KOKWt#8lXWK6*B6<;ECtS+=Yp1X{Z2B>-RvATpJ77 z1z^ycd_b;wA$>rI<5X))L{K|cZa5Q150UV7I5)KboX;WXgzB7RzcSsR{kPS{ifN>A9fvZ7J=61 zxd{tCFzOg)6U9bL$Sv#fN)~Z7p;Yz=YC?7!$%=(y|FD52#(D&?p7j{6xEr z@XfWb(#4yTL5_?${HGIj2cAx&-0Gh_VeC%EG>7NnBpm-{r8-l97UqJ+E8NeZRQ!_Y zW7o9&XZ}RXqWqwCc|j=oM1GJjQ$^;roY_5SeTWXfxH&&E+?B`I^}N5TDQoyu`CUED z9!+$bgc$(At zpRbTVdf18oR6%|Jk5B23*2@nipT_@9=t;TR_^;9Ip3MPgU}NR|2r3hXu>AkXZE=1} zB*tAAdbWndnK>C8)3By*A;3q}94T z1BFIwhto7NA^1ElTcAXZUh{fL)48TZbTW8Jq(A$5;>+sUSPjmOoig{SGxS|zE~b}J@vXmoKGapY0lRnk`X ztUMGc;u2t1rnW<&wg*Wgx6~6o!eu~+B9T>7n}&++r+S_@e|c1f>IzhEkGibJ-TB4= zpltOH2yQZugoTzytqoz=r@Mv0{5AVI?-7NLQ7<&APqRH1Bqv&`MUryC;`0IYTa zAomE-{wrZ+lT@l+mkEk4)(7&}M~K$^3V|VF1@G5)nIE3PqJItpO}r|Z^5t`3yNyob z&E!!PG6k-{?)U$~SS9>-j8&b+|Aw(Ddq+Yi#F^=MVVE;NS^zBMTU-tdRe%d!HF0F? zLfF!W%v^K=Sy({OYqD9*Ra!4DWC_d4O42AaxlE*d*!5aUO8T0#pjD_C+;(qj=Rf|Y zv0bo-h{xryhiDQ2$F!TPFCBz^ojc5ceS{w-OSkRwlkk;YBRp%2F!bjewU)En3C8KJ zdY%%XgsTZHj(s4GI>S!2(W#_O*?0fM*a4SwSW?ThBS5p8(%&x~CyQ7%DRUDU6qa%7 zj3hAdNsB1#FErPIvJIiJ%D?qKItnD?=bfumEUp)=2YD=Bf#bb7@NXD|@9H`#oX9>7 zXbTo4{8sd%k{`>M=PUF#3*^e^)T)27h^+TxxSj9HX9p}&H|Ts@2@4BO0{a}(J-qgB z2J0U5o`W=27{5R2#!d)eq+eyi*4GfES6KG!GmJ9NHHjln(1@8s{pC6Tp1iZ@zFj?8 z*;@FB5+g)|ie$}e!0|B_()6d5$|q9OemBm0OahHwWCH@&Rt>o#jKUEZfIeJXR4u zcI$s;<&7TqCeYtkS6SZ zvcG~svtm0{tQr=E4xzr9PZxaJLm?PjjO4g49ru};X^J$G$*~)GsBv}f>j^_M%M-p2 zjErY_vJdEd9}>-2JTBYn7QKkF>3^AxX36Etj1<1~;9v?e+KQ@Ur8kwxVWxLIoXu3_ zHkiC+RpbuUMJ0wntv80bOh5&;vz-e0!)-aBxug>hCj}*83XACs&}IFo-#d)WeO+An z3agt~^7-cNLD$Z&TuEhJ$l1%Ij`#L2wG_#4Rx*#WSg`94(H+msW@c(yzYWfQJ=x2! zEN(*GU}%~XHr-Gbe1bU?qGy;anIJEWt(Fxg#9x)xUJAEElZq_hR4`F)-DcvkI~6P* z%bU#O3jgjiOCqxW1i~zn)>a6%t%M@1v=&t7#c-PYQAaSPe?Y(om{WG1#$vUa^UmN{ zGLa@LBLO+d*M7p>M{d}j)y5-Ur_gcBRmKlOgT2vclrs3Lp*(juvP{Wu5pqE|noy5@M=iBOAsHZ;#rMGTB6Dm^qkbSH zG^G&~YL2#l|BkFNW;~MK;Q^shP?x-Sd2qHMl%!s_;*|t@Tz@H<5W&w!Nkt^859WlA z_}MRb=(`&+%PL0|1=8tYCYvIa^9?#w@4HF-Z%`t2U^=+n-x9Ul9(zNJ3dQ;CAv(V) zMYe^#Xwl947}NuZ%GRZFEbIX@)1jV#(B0|h2u!RuIX7es+=Q`V+SWk2BNVY6Ip$u}E+ zA;dtyn+lTmi=_Ajnl-W+rI+FXU!PF7UjI_xf?4#aD$s1CLbwYKe8m%8XZ>)x8p-E$ zf9u~DYrORA9JF2Kax|mL`1t~Ui`vd|_Icx$djRShN%c3?s=Qu7D5t%d(2H*-g3WF< zp_p`}veTAS#Wqm{=WAB;=_B!U$RAdbMv|kN)oLVOVu~Q?UidT)r@whYVzk!#c1(t5 z=aVl0`IHJ3>A9}o5(|(UJq7X{s}s_P3rZ;xdp4_qun7pU5aVgH@mC#j2nr+~QNX|D z#k*BtLAzcxQj6(&W*EqZc)~>Jnw=$awlgXH#IOGXwBCfRQL+qkrl{;v&?aTU#v*AU zL6O>lCEBP$O5cc54kIAg|ne(U;Tp7a4JIYr&phd zILejNdAGr`_n`x0U6; z)AQtIGKop}&`6MA@%TfgaV<~H>(0zyb{YFXiTOb78$!~@$6O=7V43o|Ygip6I;Yu2 zhbrPSj0HV9NoYe{h9aqp1q9ow6fJhqSj~8XSoG-PrD&hNac`MwaJ1LuDOSj*T&aM^ z=#Pi;3_|lC;Gw!}o?l)f zB!(j$Ct5JDs8r{;o+JX+ocM5n0q=0{_Gra(%`rGt-5k^Y;w&#I2lL2EH09|F)Yp}n ze);KU^r!FVK9#7kzJ(W&nX9FT3mk=s<+agdHI*MRgk&AlyH$)e4VaIROkNK%NWRD` zF&B^9jpvEkjwGq|Gh1k4)DVC+#E5k`#Kn3;Vf7G-Wlg_+c-8w^Tnl5)O{0<`da{S& zZpQKsw{V473KFc*3tEVW%1)<)g||TF3u7H*=;C2?6rKRhUq;`9Y0J39$;jx)ZaEVY zr}99SFB~l6Pd{0%OECTIeW9Ws2}9#>T{?r9{8Ih^`~Ivnp2|oS#fFic#xqB(Qw# z$Zw#0v5z^hgu)Qd#*xtKQlW30rdE^1_<=Lyy?$|zENiaO)Ly-x-Rlk6jC-Q|zPgJV z(;osYLV`?p?ms4K$f0Z{3Hw-jkmWw#J69PYM5%-=02>xeLj1MJh?hNeWFL85scU9! z`<@2BtFP+@qyFZGH4YyeoRh@9?omG8)D^zI&ec8T!)VO*uXkY5`~s~7mHR9;LGoAs zot`e50-8R~9^d__Z1@rF?c&35&noY6d%XYTtQkpq4H`q&xBz@*QNBa_yA3Q_m9X6Q zZRL*$<1h^UO!8BB4`bnFSwpo3fuqmY+o(fnhe)GrQB@T~6l%!@+@y-SE9Gx>wdn7c z=wL=p0bk_Dj8Q|%F@YJ#rO?<+-$;x^*I+(*Z&oC9Ima2uIDL4u|2SviOIW#^Xxx8D zsf4=JWlJk6LTRB3?0_QLOMZ8ZU}TaqdT)8Fjw=1 z=|dGJR%3dDoLvVJxKm|tc)_$%q)<9psCK6dmY9zbWg24aB=(;bJTpw1Hiuq}fKOg~ zFAgSpE&1_veU5gc*q1Bw0#u#E-&=?(r1Q&gFBu59(oT5xqmo)S;-$Rj*w`+C&J?WS zSRG&?Yn8WEoTBVlWM*MYoV#-CntR^t_1#<$33N!r!-Et-!Motq)k&C#!GP!aoA0oz z<-E>!Bk0R18f~<5W*mpqVv4CM|-Sjy5ejjx#A&o4&D z3MlGvUcENmOhvlFmNlxJb{U7o0lf0 zp{^*0*ZIJ>aG`%AEm+pO=L9bzC&*NW5AG8;haHh}Kc6DSeQbTX$;6zWwn9Dh)v2>s zy-wVuyd0I?)R7q<$`w; zQP_d+ZeJffh1g7Ugb$vN$Ot%1C&2Bdkb@R_mo}F2so2%p?KM;4_!fQ{b3m_#(*a`A zDCg2_^xN1b`zkuUQ;K694b-#bJEx^$uf6Nm0L=DdbeO0oFmnn}w*wWi>Zf`*BcMIu zbL}#|yJ|>R$v9|ssaN*upXZ^zmo2zk)=o&rRMxWVLgA|cDvx9G967kWHq)r5o{Ps* zLSCbavkM<)(ZVIMAGq(fQX!4|Aj*0M|BONDmXSI1e@bOQ%1gbyfk?sOXH1)^-|@Br zeKATC|JT4iQ{#ZxQR$1?8b3cWdbAuWROc656`+9mQbydiqCj1n^xVjei}a1Fu1}YP z>g7;3qOq;50^hC8-0n0{7%ppX?(@))mDNdoTh?y;rY{u*%J#R_FW+v)4cLlPWs7Bd zTf6ZvZLxRj!B@CDZz(2cl6{-&Ua?g9k_-<{Y$=@nGOxpcSZ7lsk z)y+(GjPUjVqHDp@(}br zeisO?<1aDKX~vyt+;qD_YO=UGo;^*F2?}gFXyJY@L*Wxjt77#{srH?=Q&N}Z6KnQE zsORfAZ-V?AnclcT1XMaC?q$ z7`wTkvY4(!y9q5+&YK}$#`$fbRffXaBU@uIR6c7fUWFem1=cbv7+`+;-vi7-EGYUs z?YT+JB(+4&Xo8rx1}eqQt<%++k;+hOTXpXbQIc4U8hOZtsT#fz96cTL=+T!YMwra{VOd#&oM|QeX$z5SPD7$*KzYmVbu)|v z=$(+psLvK>>>}6Q!mF$MOt(i*gu2y6Nv3(W@k~^GJWIbu=`_^LQuP1 zD+F86Ne~ymy$2y3rAstjc)4z+bfDLKGVF)OI`yWH5x3!qg8^1ewaF;7TlQf;bQcmX zEsMCL@>o#NVT)&@KD(w^bfOD9q#on!RE1!dBEKq`#E%3`4CD#rm)+kpfb|xO(=H#O zd$F$@rmpn#=lBm3sO$DhWlYNHu=Hy4|sD<~M;&Z_r#r%ey(DSPrT@4^dKn^iwY+&Z!y|i|W+o z((}_1(`N>h%5rvF97zXyaE0|sxQLq_rF1{HSSf+k__k`*e*FN1yP$D2VEKW^yy3YV zD8gbfnw5c4AeS9-%_Lc?`uwy9?5_sP(DeifKRYLMhuqQpO(Gnx*W~iUru#c&Ar4)LmOvqL@M;HjIvg2W@I-sPK&3j7*|&?=+2duF7;U9p6WOS^IK zb_W_3*3RnH;LpvGjGlJ71mmYWzz4d$m?Fk-LmH-u=#x*6y(%~f--@>vW6pNC$c z95Vaql??n$fVKhtN}t%R%T>$`G>bWQ;;CY`#h`95((^Ns^KvzjVBS83Z73o5kq&N> zpU6aqqoR0R!t9f@w4D;Bhe(fbq5GXAB+P8eJKSg$v8pIPHfErx_1}7|elz2=+~00{TnUyXy%rRw2DkhFZ#{ zjATv>=+eU_nHpvIHy4CBxyU}sIFg2!Fm`@y6wIfJbSt!Pc5)8t* zS_zVCb{=s1XK)_x#1O5d$au2|dr8d30h#Yf7m$pgt59qh|XYuwO4C#v9t?I;-!@2rlCx+Mw4lqX( z9mv}Sy_ehTHkAjC=^)=}jHkWQ%=?)zZ|GDd*2GcH_BqM0Iw)`-7e}?7)R!Zd0mc0X zofHG5&z|#zRgc~E)>fhX$1V~~6n<$83EK9oZjiGQoc1|-C*^&p$83RNXs%KDzH+!Z z7}8PX`JJ0(I0@k#0T*3gQ1sh;tt64yCaD9q&(x16r}p2Ekp!$r>V`ZQ^MU@{Kmnn|+^qDh}IXf;92?t)g~{)H8aB(DnNOrEyO z^39{d8Ln54!&~CPQ9X{}p9k`1)PaHDk+#?pBIFdr%||5O?xucf1)~Mqm6n^O=N5-c zymn*U@%(tGhf$B$=|2yqF@0 zhlJp>TNP-tM~9O^pdJ!JSa^?rQQ$uKlt4mZtI>~KOdI#t+qg}CQex({C~QIec(wSq zM;jfaqMmj1lK0<#>)&6rQt?wv95e{G2H#P$I=BW zh`6UA*6J~)$a&JCv(4@q#fZ(X-A<#HmTnBCHeLiE5;%h%>rQ)WlcfY{S2f1lxOe63 zRV=cLYzrS3PGeDaug`XXGdG|2{mRKmo$KYYSxc8m-8rbuhsWnw0hMzM&8HqfCt#_% zfpHz0V8Vg940+TD5lMY!NEo6I#Ol`m?-{u$|Wp`;tXWJtE2lH`_LB&<>ioGW3L36KhJ>vLC;7t zyc%+sEged%?QyNldVrJ$_?UKpZ#%UzeWBu)m^#_H z@l3#MFCmcz1f(|=J!_`&)3;ubp0}uDzcYa1wj(#4t<~HLG8)1KPwJ9{W<=GQkjVbE zUfMK8+K4_)0i-X3$Mu!>{dZu6`AjD~ukQur4>D0yL4jj*di;306gZ^!;DXh4yY~6> zuCK4}%hi=G^g(@UrQ)FP(?8{fB|3pgT)p8y_lD!bi|qAqJhO?}WsR?P3%u_HCiXh7 zi+~}p!2l+4{F1;S8kG#j%P2nD;?J+pClYlFktC$09aW@{zGcF%z6-gU>`T!uz&(*v zke4T6vKaVkw=O?21B*A46dxZS6@`tT+9($NI@GS%kljG2RWR%J>I7$HT(Ax`*eUD9 zk;)e%DEilNaXhQRkY^MhX#4u8=rtw!fTfy>$~})OzBTM1?^05n1HQ65IP%lVWTOX4 zc8&bid$hst`JPGUdh$H~fw!`>_o`SO+}MV{Yjm4L=ma!QSyamvwWWQ)JNfzLM?F0p zM#CFE3egB+M<4krp!PB&f-78p3v9J}&vxWOmRa3o%#47a<35KVCJd+XF#d6%xQN<= zlL4Zu;Kw0E5TFz8@I*oHRS;sZ9yrH+BwF$Dv8ac(wbiPRLd8*qjIQrSB;X3^UFp(f zs6JX&p}KBQZkjhIG;H`J_S4+cd8(ORIO)13^pvahpwi&^<4(XI_?X=R<-Tbw+XDbG z3(gA+#M+*%6WtyY$=06ZFKeqJ#hvS zB9qzne!OwN_bEd(_@a4}nV`L&LrKVeuB~3Tujo3I%E?GhUWWV8#>~vDC!!3Obba~e zAd-z%e%y5Pe6LY7Jo#O<$+B+8^R`I1_|*`|OnhLU2U@m`t=NDK{6~IIUjwqwBKp~D zw)4JPOZC3Q_eEZ5d=K1-LI&`s06Q^jJ`|Y!{*H!DE{`2bi2F!Qx7`oLWb^tg#ZNwb zbH?|nc0@=BOU`7prU9qdEqR{L{#3;azUlyhpMuiCLfp}e^!wU{W4tEc3pzX3z^|^h znuj;uI~Kz94EB2P?Ae{R&%`O*18vfcZ0xeTgAO-_X89gGv;^AazM_A1Ino1a^2>L` zij)&^nt7A(OUH?9mJ>kVKSKUU>%w9^mh;Viqy`ZdR$lUZYvfOj30-M0K_;-*RfkZd z?5k0_Rti;`T(0K6!j`eYZ}Y-x@+wd#A(xP(i7%EOkoX;j)tCUUOg@28ZsAW z6WNT24K~!7t)18Jk?H=dT&{EPyl@qWFMJo+WKk&0ui%tcF+5T8zi4|4psK#M4-^n- z5RgV1q`Mm>L>fUjbk`xJJES|LOHgT$lJ4%1k`N@NB_###I{FXa_q*SXxpQaEIA;J4 zd+)Va>sh~gf*x*1MWml!H#lt?cv`{v&X^eQD89aB^qt}KfLMa`+Crn(`^sipDLu=+Y&)OEq%6-3(~_8 zV2NEDVB|ak9egH1(=|GUqQ?~tPHjC-_?_;ASc~^O+4aA%={2I=pY4sMc%LqLq)j-0 zeJ`3YucQ4g>*>z)hZ<|Yz_dE2%>{8s$Vb{?19$w0iD6VE!;hs58u<;*jL(B|=>}yH zc@`M=8Y@~m%_vMmp_tc5QN17=B$a3T?ZB%(xVv(JsH&EW6|8F;C1E%PQhDR+(h}YW zpBo6p9W9?(S~8?c_;s=06hF6C&J++d?%p+jh*yIs3HflJUG3u&Ao+U|yUh7*Z%!PC z+i^W8{QeUH7WW0Yw~WHY?XO0HP$|&9j)zj=Ao63^(#3yxHPp`fCQ^@1dZ`Z_JUoagr*qYt90qWNJ7wm=w_Yv$ zsvyF*-XQ;_2;1#M9?f{{z(C)}RFJ)5_^gG8hbr&ZXWg=7>B(P|nfw10WtNBjzf)$i zhF;1y?hO;jcl-gacScJ8t_&=z$iM3ob<@Nmq zaezC%J%ko%$bCT3x~a3o4Ct`2Qi5R`2-# zCzXb*{}+{pC-DEAN^5TJ`(DZiXOwl_AJ?-w-h{ij)`B{kPLsNYQ}-4#VTCusU2`_) z5gA6m`+jg*9B2^-pFglZM95`3-^6ARw8Hq{>P{8Eovwk;Nf?oi4-g61XO$@Z)ShTt zGrfb|PR8lFhsSc&E|Q2tgG9ijt|Gcizdrr20mFLG&bm}mTIdrlU~?kaNDsJh+4Fxh zZAA6UvKR;Xe=u#PT_^r9rftS5A%&~4amC=J;_Xy8{x2ju;O&y9`w?# ztHv7imbGIolIl>n8s?6T)CCWGFc2@SCuupJ+<1y6o}|vRqZM>ot-@ z2wj=A`;#Bz!;!~7N3Uk#Tcb^Qo_hC`?lJ24<0DK)TJHy)HXF?k`Pjb6XLep$6gD~H z5SkX!cOZlf%ab(R`Q}uD9+-V#keAqh-924cDHN z3eRA)#Yn&%RH8YRPBmp-!Yih4gBoS<?u6b~-7+T~dKhDeB*+^Av4z1t7=Ul?&Zi zKm8f=YQLL0vF=MYMjlGFFNb7DPRF3>RR9L{K`nfG)k34NRGuB(W3C(3k7bn5mtRLn zy9R<)G&uKB*z;fPhr{o$_GCn=dBX-ubEB#iHc)dNUhfLma{4&$1LQ1;B#&HfFVD}Sw0(BTcp*#KAa}r zqpJZ)IO2Kue@DVG=*|ubC$eZEHnYX&dT&blgvSxy+QgA-u2mQv)LP=)$@=188+QB= zhr^RHs-|jZUT<@Ln#g|OwvPf?m1QP#Z3llW(8j-;r_Os89`5y5zoH)~oHl*~XpHa=Lrp1U^6HIHIEA zp<`)KPbu1sG?d-eJHEir5W5jqdS!oLJ((Zu%^HTZHDtH@4f{<)63PJ26X`q5b)h5a zyb>fwUg|Yavz1qmZ=)jiTFG`#2jqG9T%)Zj#4E6Q8g!bf5p9-t(ZwS5qeBNbdmD^B$|oAAv_mN?mP@{?m!9h* za&6~N?{B6x!VepquuQ5}))ue(cX5fetzF$U&+^kMei6%sg(%GoK{2O*KkV8g`aC}? ziCe;_^3Z#fxEG>|GhE$vb0*yBNF6SHhevr^*CqW^LYc4<&pm4)DZ}xcCH;?vs_AY6 zYy0cR8(93>A<7C0WY5E=6BoZNQZczO<0M$cL?AJKCND1kIjmoL;aWOLTQ0YAmR?5} z|8+5!uU)A07jJiMr^h%3JNNp_!+1M3wF_lh0^M+8yHy=qh(x<0CI`nM)`1+9ggpfG zD{eLnFxjS=nR_hFnYef2+Bp-!=@or}J z&_O$9;wg0&0!Y&t|8Vuzq8G~2WXD47A;nVJ0y0}72J%;K znKtwqXPsUMzII|)Ta$jb)om*`Sqfhl{aKHRMAP~0Y3(d(h>J5syTMX_{!PXD1D176 z#7wZe>&z#U#gPGYyFL$}?NhpmHes$et~X!YZsyYxU^~A$g4WaUU^BS;QAe#53--la zlq_`%oJ7)?1I>=9ncjO{_rzsX?}?~j-M+~j8qRdpRtVLvvwJmeXhdf>zI;#>>@fIJ zQ2KbB8U7<-JDanrP8GUWOos-B+oVIFsbvN-?#j825C6f0V1;%hv9bE?!qy__2YF}W9H6L z$hejK+>L4&8x*4H&AvD0QpR2M^+O?(HlI<(m0tVZZiA2)65KUxDDGDuO2b*LKEb>w z&Zad}ge?EtnswSukzxU zN&7|gF%1Hu&+Qk{SE=+j(FdL%#-F%xa;8jV^mnJ0yBX{5%H8R!7gH1T8n_oe(8L+ zOo_Z0o(v%6MRYV#S((e|$V&ZE-p?jR$mATt=E5KMg#;Qiwo)|-{6ZyD;#c+V*;u2Z zKGA(Bt4#Y1%h?$8R6op0ST$CyYn9p?IdzuuJnn>x9LPJ&h$b6dUCsBsSgD_@7AN9` z#>rz6;P>>E_pL&s1l`q4^vXxcEc9M_YW%7|rjAMZ-y;|Jht1W$82WkhrC^j#nYU82 z({*51y_mhJ&m>EUTZ$x7_2ajj_i9!lF)ntbIm%16s$2oBQde%F47!_3Q-JzA*ZYh5 zo28EHNt{h)C6#OjfRJ&)10(fr=rE{rA7pe;FM0Tp>99ECj-?(EXJ42qC@=MA=cSJf}J6@>UvBNX_>8=Bk~rx z(m^x-3A^E&M?xdt<&#+m^=A%SEI&GG)njbQcP}N5a|t@!Hk%A0W!4V^n2Bxs=K4Oc zhKDZ-Of#$5CLCe_S&Lp>ISzmxYD{-4Zfk-Zc@6|5qsK(gr$Cq~3U%fAfZr|mCx@Bc zP}c!#K5)u}Px5j5I}1R^B+GnSHeZlwzU7MxNn9=%nmarLyLfaHTk@v6-y~Lq&hueR z_Q6#ZHyvz>7a~?oI2(T;CG9XW)opUU^L1c3nzxod0F4$y>u`IsS>!qXG)vTPmM}B; ze_;ua9t3hE`$iLS>PrpU|IiTq=}r0~6=y#YzXtjpqWk&59<#9dFfHU{SqSCTO~shX zLM=c4M~|)Co$=3fzhK}FOg`rBOmnIRu`%5wh<@!m3~Wz zgsJ7lMMyMCw`l;hofQx{aHc;l?x_fEVy+6?QF5MPsm*vbw4&LFl|f>mj;w0s)1+F*kiCvAil{ z)TNBV>7w2j)<7O6rThr{hmdgz?Vd&~9GdC^c%)p$L&%kpxSrS~Hfdb(pgX~fWWCxs z*l@<-YSL(9ZLkVn){h(Jo`{2NEQkudH6DDaL|1~kGhLNG+=nMWI({PaStm_4#yvA| z)oBU$NO=3?Uaz-m^P)d7MPy;@gQt)0aOkiYv^GQ4cn@gO?h0#%N8a}oXnr^74j6kB zt=WwFC_15(wFKX`A+N!G%U7aad`B90-!H%5H@fjFdPdj1gUW(U2ICKFq<`~=?P$-w zn=*g$MJs@F`c&hv8(Ie0j8(04bAXj3wgLscGGn=_}s)>B&c%Rs# z_U*&qqSy$bbD%BOr45mn&cu&@DbpM6V*X*;-RBtT(bQuMfLqxtlD$F8@=~jY?f8Mt zBEJX>MFjYjm6f+Tg4q*r;d6ZaDnr?Tu0NmyT-L~?86ksa(DYSP7_nDJdDckKWtJc`B)tR zwx){KVkF4s*AD$?@y(0VZV8HHZoS21fL=v40u^@?D<#7M|3(>&nnQv!nun0}@6kpnjI z=k@xEb1uC?H+-`s_@eb+T(lE#z~<1x75&g7ZOK+bjk?JVd1tzTpV#`+PaijC_TR@D zIZAZq%3>mnSHIXN_h7hu%r)t0pw|KEg=%l4D#OAOfP1Lf{Rr2l8g!e&FK&MpTv+(* z4O@=_7s6|9- z)T&K?ifPOXK9yJ|N55r06}#1DF8vNNmeI;hp8PDaFoiBe^Sy4R>f6Q{`7G{I)pMy1 zT0doH?%?12pDaHh;@;Qnb?(=K*1vze z-x1OB`H-VtFVgRa^DjZy-Xag!!s9pt{0^=6L$ii|-OAqtEN;CU zDj*NtuCo*V`{N{%X(Z_QhFpkAe~EAZq#e)Mz{j22eNl7#?~eml2Od!an@Qk*lab3y zbP{xDOZ#4OO8?1hN+4*(NSv{JGUl6QMjXu&W*=nK<{2(+uePwyf1wm7?54CSi}Q*m zbh~IMMQz0KE>gOF1fY~Shmw+A7x&Zw9W$(?2NJsX5~Y7clkyLbA|WECX35DnL0y?N z+0;}(Bs92-FFyG$dM!|GP0$@{1PucXjYrt!$HmiXH9<#2(2>A0*Mp#hyQqjDCE99@ zmF**lGD)2##5MMuTOZ9@V9~6#$sgeudg&g?%)~MN3L)^Nf<1Ml4#3kC49v{O>^yfq zH$E$+Z;h)~ z_aY;uzijkfO!Urue0K!xRDmsWq2o(tq4G@W&Y7~3XU^_MtVvc@Z+Do7&G#(hNmAs?=& z*aDcv8BkWw#DxOYD@rT7o*aHCL3tW)#-Qmx({JhbMf=0kops(Ne5I@OTCCc1z_DWf zH@8dSTc3WnE{XY*Qr4Ay+A{E%Ta4X${HT8cO(^3sv#LcPz@1|X4F`{aE5@IKu`(2k zESty5)7ZF*iFIiSoJC-KHq=xmi1T6;6v}eGSKG|VYM~Ddtj;JDCooEmhX_6x%4Zv;zCRF5D)}=IF@Wtu} zC$<|d*5WlF%n2w}YBxxn1=1A&Zd_}Y3nkwZJyof3+;~%AeApeXw)@F`a3{=X+V_Xu z@(4q(Cy#_Tjlb;2Y;b;wfYXnO@IN_!?U&B>pwm9e$rHzSqa zvlbg9bwp`n$fD7gk#)pbv9zc7ab%vN`vr$}NDs*nAZkY(@!4#IzEir{O*XRWD{wMn zJ3Say8X$Zyii(S?pyPf~vEA8xvdNnKEaSZwi+ca*O7CM+^ahrEds=!5Hv)}r(9daZ zPhBN$d$o5Y!;gQ+tP*X?7{{=|F>eG-#Ca=hjxkc_o|EXXBd{-}X1%oP-^1%As8%Cb zXVuUG?U!w<5m;CKuB?*g=1BP!Gy}9*>^{IHaTjd!$PSZoS<5Un=a-bS5+qXyyLOgJ zM0Rwn>L)8E}(pNn&sxV&sc12LxI3YUoZqfl5J>&-Brt=mZ$2@EquJkXe41Rz|3*-#vQbj_Nry*|l-WkxjpsN}`A z>D_AGL!LAsj~benMy+z4vk}=vktT~}w+!j48A4!1@DKlT=Lqtap)y}PNLm$kCBJ1M z%hCw$0bG~t20<6+(yH4YQHi#V;XD~H z6agcoD+Du@k5y@&^hl%pm&tL7$C*W^*8J*Jpyr-T6j3>OZ?D)~tt~Sz(k)WHVn&Vq zmq$kkq~)8ye|PxVsVwlN-IaB)@E|ZNcFf-q2GOQJBz?YL0LWq-H|nEFF9xFy-(cYU zbTT9qonZJLodyx@$$AIrlJHl2oy`rczqcjOQ-Q+E(y{?1@ZY(#hw zzt2?xLTgJva z^}$?j-vG_YCc(8;H?BICDHur@0Q4#IwYHytmvEz&&R84|Vz6TTZK{ZS+}`N2+gn~? zGW|EfX*%{Tjx0nY_iELLO8-+h@C1H9B&k9PJ1Emp9a5h{Suo$8z;U-iHgj3&q&qAK zMddaw!8bklRNg24^AhXBT#=f12stxa&>BOoJUFKSoK0G<7OJVU$|vHuPML;89uC^e zQQ!NmYJ)wsLn`>L$-COC-wpP=yG=kC5b=_cNyOYz%k>I3YC65TnFt=M_9?gdzFg9? zlL55oA2;V@FlS+Yh7mCex&0wz!?6LA{8hKJXR5Nar^UQVplVJS2%ro^p*#Yee)99Z z*W;le%%wmfUUSn*@!eF4tSx>QnAT$S;IY{_R+)45tSMh?9c3b>3vIy z#*KFQ(ukR!swxh9^C8A6=Y~5#6c_6J(2_bmqaK53wbLoJ5$Su4b&8n(- zfHBedP-t>}r+y5i%JOGTPXS&N4ZhB{Yqoa(hZ{2es&4$uG`q z1tQbrel#{pH&vuo4YQr)ysXOxbORo03oh zAoN;dImLjlKaslCTrrg5?Ok-=+3L;MpVw_#M5~vc#F0N5py z0@2)3V@5U5^6n6WhTM=%zj-@W?&kn>dt%z1CKeY7V~Yxy&h9r+HP%R^h6jdBm+8p= zFmfeL$c^WcL0s+~U}@57zMV|}M8{=~y9V~`W~KJ?WXJ_MHv1uqZnIov&%_*za*`ST zO*!?*KPkO~6Ie4mQDg1JIYh`gPQ`hB*U6vtNBv;bC_LvUI+13SuW+x${k+6K{KNIk z#pPLy5w3LNOk+}rkssXsh{4j=<6tk_EBVI_2?YQhRF1;YPBUl>)j1I6&44?-Ev`i- z5a-WwPwc`!IK@Ua|En7)?ANe3vIh=Jrh%A9I+01OvkrkqTOU?tKcMDOzM zRWecnL5Y&?Abo49Tx8=j^a+V*o_C20VOb&`y*b?^%|8XpA9LcFy^uzBy#0YaIe)teyMNuNOayE+Gi%?3X3<%0fMF3n5wgq~DA3T!WnJ4|Sp#t>Nwf6E82m;`g6)s#nU^ zTGG@M`c$Rqh(D734HznCB?0@y*VDnwxrymQE381(Yid~9hn*2V;EuTNDh#>K?e)SA zkvFQ=UEQZ~)$BqNG`E^^FKxd0)MieNl#Rb!0y^3rtgPLFEvm>0!zgkdlX236VSn1| zoScJsCpT2#9D2X$pmCUiuhYe2XCg0y*&%a`T*SRw@L4xRp*?w&JoCg@FWTu{Smg!0 zhJg@Kl?AtfINKQ~*E_<{kZ?F?y7#4>XR!*6^y~(Dyv~5#XEdUpI?fSBS1sxH2@5<6 zL&^=@hAAB`8J`bUZsZKbo(MloV-c9G9D6l%@5lRws3)c!FY|LiDPf;HWIO=(pgL^> zw~3@tWLIRb-_*(~KZj#P>LE8G%Mx^B)V2|6cy+#l67&%(%ysp!eBr0pxbAM~a*0#& zNjDn}gQw+rHvtKo6Y9Ayo3fUaG25Fttw?i76kLc^GIV7J7nhgZ2ozLa;iR}jk}Ho5 z09J2f^`5-gD5aMtUP7e4B;0KFO)vou%TAR*lIS%V+l?71b_J0wO9-BgYcau_kdZgX!CYHx^4tnF(oxn}9q8a_G=7CrihjcU5lr5y z2r1v-lxQk=aVo>Bg}AhR>(3Xe>Y)_NphaZu%xX|XlcSkPWkL?0{0Txy2&#ML6 ziU$vpV?#iP0yi;cmVUz1;{SX zY=rO453qnGd7BDsB`mI49TeTO6-Fn30k#81m)o?8GGdHsbi!K!r>_>zIsp+t<9&5H z2k6oEXb{ST2AZtBHhBdJ!o1$?*A;>29(N6 z_%t4t;aw?f0QEss{wlG}QuD#ldLW}Zj2H0D^=5vV>v~hbdyiJT%CzB-HsU}!GMTja zE8vAB@^Lt_o~Bx{V&l6UEE%nK$1V@KeQCX8(yO3u-REk0jIEPt)D5CLx*Acb?j-a0{@{?w-zkI?R7l^W=y(}v!$>} zuMI>k*T#hKprYdCtSQEAXs*1;CZw_e3zyx1yv=!=330qw$c6k-j_JT57TE*gK61eX z0o(ZkhYQ-l#4mQ%DUAP?68#g16^uFF;C?B={r&}@4_E@T-3dU@dopbWn0wJZ!J>oa zxHGlA)ba%{)Nxzt{9AeWzzQ&WZcS}#r9rrJ4D!Ja&?;3Gk3C}sk%6r|ae|0HI&sle zSKgCdzv$}~BgRn`@jA;WS>qVF12uXp6Eo0zJlW<4auZ$$PUDgGKK2SiLKn6G4LLBy z<~Ans{e@FxBH-1tlX|Gf-$=iWAv5Vg`xn?#VE};~=?X)6Rly)nlE#S)4jHx<=7YUsrBCd7VbN(8|Soq^oQIB|K zPJ+M=1xWCBUJZjaYBbsP;I>O8SmeVY9-X;NeR)r~D zf|Npwm@IPloKo#p5+zw#^?TD5J@3RWBNP?cJF^8{(J*-z59T}H;>y@;B88=P1}p0E zq?tYX0kWQkN40snI+Nmi;*2>jCs(T&)%&MHq>v_?f`7>et9nUTN>L7-tX}Y6 zYG7+MeVt5a4PWkO`i&pY(mvDZhUo_=VHS}9K__z}|1ft+Q%|fLGoebRJLD)V-grZa z7nPM4E%5^)D9bm63!3kU>Lc+`PjVdUw49-CIw-$T&|h15oIfiQaljtQdfKxewlBxkL7uxW;^k_I#<)keyhSHqF-^X|Af5_fuAB6^BB1ro$f|;D z(~HD44Ts_1vaOZTu#ySPIt2Q7GliCO*#`{qHwf? zL#G@B7@aRGAz$pMsw@aeKMOV5+@WSAc&q(GR%;zJ<4#0&(bBacaLdz@2C{xwWf+I3 z;MZb`_De?nH_-YQ;@Yl^k~%;6PFuker{ARxil_Md|IDC& z-k8D+BWcp<-~ai_KX2Y;0E>Y$jVb&;=b?Yzqya~Zjd{Ge=s#ci=gsy*fG;niX#Jk& zfBq|Bj+29%q4iGdf6iKoMu1l<@;@f{qXPZw6XUdC*fW9E45+dH>$`Dw2>DHQ1x@QL zTJ6JWuR;5h_X=tKj{@}vKmc2EVnFoSJp?h(#9OyszME%R&V&f;#?gQ98RIFBm-hO}ILFa@N-akuZ z+T~=J%HGP#s@mMgD2Cd0|U&=wnsAJFv`V__F%T?;nqjm55WfyOl70~3HCS51MIN3TrC0Cp zsP}VHz*?U&3d=K)i|~G+f@EAdP3ah#_ZOG)d(0rPtedm*FK#HjxY48WG=*vft}|tB zUIejg;0gmwfnRIF);QJpTbsC>+#uaM40PsYKGFMe{>1PGdh)>X02&&>F)_b4{h;Xm z6TjbVX3C4B6&TJ_*bPFKfvYit%Pjvc{7oXQeL#G{KP-f7dlL*Cz5$BIXUztFH~T$= zCU-cd*{j7s6M_eV$KNqYYZI9X0UbLqT%7#1u0;YvTt?KR$s=3X@_2R(8i{@zgGrL1G3z6w0CZ1vl8KkuyaqO5@C*X76j^3O zJ&vcs?-!uq^-~aNjF&}FQw>1f6iOegv<)hRU&z@&N$=j=NgpW4sq=x= z8#Hcump$L$Gl1+ zYbb8yCn4F_^sy!(A_%JsDxI)cwmEVi)^EaHTHjS(Cn(A3+_0sulq{)J)no30+ycBm zc78Zi2y7fQhUr3-?ncr@RgN115hW&8TBZ&KAwRIt(NT*5i$U^iwZ&M6nmiO*92O+a z<7O8+jdS*#C>UD;Lu{3$)yO>22+^nTyc9^sJmGrC>cydW71Fgh?E`!m!7Rg5g;+?l zk%Vf%+skP+@#GbKtm@s*4NmM+s0{COb92G^prD`-64MBso(CIw4AD^|v_VlpxrSGO^k9AmD7aH`nmB=rz5uOiU)ni`D*@ckRm+W>QeTH?}-}z;Mu{Y(O2azp1mS zjk{s_Y@$eA5%@YNJy2r1tIZYTL!oe2C&1MaQ9@>qIsRmea#nVlhq_exDR*9(?-NpQ zVBF$iG3FW2No-g9a|;ndwA*zND&l6te2CVG9eQj%(kjQnSy{zPTsQF5iai9Lt6n)-9MtQ>;k4>pD)nM@-{p3F zk7God+fa&RpW}f|u-pc#Rs!OgI&aZF5#u|qi_7krcdbpp$yc9Ham!kuUntTjOJFI3 zPDg+|uE-c)6_yYYmz~qSIRle(O82S`^_7V{`|^w6X)Kak&~+x@D@bC*V8=Q{GWv~M zN?+&BqR`DGO97z+3b4mA-F;hW35Uh$LVSQ?if#sH4z;Y9E`IKNxuL}I&VKgr+Uvjv zuC5WpK_1Up=nR=&;cn|vF@XPHMQnFR26SNST|W>!~;r)m+Myw{7M(Uu^`pWFGeIAWBwu+KD?U`joRy0yIXfxXfkUnnW} z>5CTz)JWEkaw!L-Gb-NlS!#r{mlgE7(l$qoQKJ z#3edoPy;iPmSu&&^k0g6l^e|gma723y`Ozsyo;`Dq#QCuO*7M4OgHuZ#$mCs<{bj+ ztFPQ+B5b%1#JtrN!yQh(Mpx2As%TJAWQ);tFT1tG(`R5~nk*?C9)uihksh%mdBiU%dv?ONXQ&Ajt z#&!kg3kPbf%EpN~#dw9}NAQ+fyij%N6^^_Jv-c?Fx|is!oJNouqMt{7kEB9E2|L~y zY%v{|gCNo{WjVRty1v)%^%jQa`cwTP8*&)XP9pROP$Zed$OIgSTvdjdWk?_^7>t#4 z?8e;>)<6hdB&LfQK6NE)I7cLY10RUudfvVDXx31g%KQuHH}%Xy05ofXduIHsa_Dph zciVufmez4D!C7WLFL88dY7ZHUcKHxQYPC=K@Gc30zf1AF(jMD3i`@US8#Wd?GH?nf z^9}FOHKyY3gz$v9!1m8%(xnH2tK)`8s`T9vsl%6cQ(GTl@lLyZoI{l|(M-VVxL9Q- zTrAT(4_ErAhw4KgfL2S6sJ*{scByP1J((>ubM3m)$GKOa2o5Fsjm}AFj?C3GIjvaO z*5T9D)Hd?e3U_)Z41O4WA&wT!S=+s zosEuNgVnT;#JaOpH;tE9^l1BCVs>#bI?wc#twy>S-A$so zOT&)a;LmFi8vO_cBjl`fJ9gmuvbT~}Hc=f2t>}}@#jGnMf zlj`cLBlmCi<2j=s<+Ka)xtv@CFgsSp zXc9DeobV0zB!$)@xY8jo2$XkHL`@$y+@+T}B;~Ofr|hJGUG$~?%rZ`&M){_5gWzo=+qNm!+jpICd>)p-&Z5tF4tGQ z{_>L`{BC13iJ%yQD4yB~CYoIRd2@j5-G!NOIR4f=^bF;On4~iZb+Up%J-hQ=-}<@+DBBB1E^*VI>x{)CLcrbVzkn5tmy8~Cdi(kRno=Vgu@<%Mdd7Q1Kn;L(((2y@5tOAv?vY(?%5jAZ~oD;+&r6^J3 z%4vc#pT%%0CNC+uXXPCg2;|XhdXCN8qFKROz&C|(wYXo5OHOYNYr$!SV3LNSrp?;% zv+k4d1>R+b9}H|frF;2|$M*+vAp%sS@yNNSj+?i$u@vH4T@Ct{wXA3>Xu?zr|-9=h4g%ISy zmS2|7wrwyGOBj zT=+2T&%hp&W-2i!EKbpj^#Pz=TgN~h2n^NRm8(3&;DfkhgkTZl;hAR5Mp|$d5h3;d z*0`EIOO0uPv`VX{vV+iF4+ZzL_sv4M9eWrzh~-2R6YoQV5DA&o#~ZQ^KHECvl077L|5q|qP$Y;bWB8uTzslzJHGS}DWA_@(kU$D>zMvfpr1 zvEzMuBTdGZ=q&b2)0)Mi|Mbpxi7evXBraa`4o|~MS_pTGITu;@VO0)Hz8X*_9}=@K za__SlK@)6V3*k1{C=1cps>)gDu~nPA^Z}l1Hf6ylV82x;`f~7w6I|P9cc}^Jkt~y3 zq109V=WaEM_&LKM^MoVhkG0`%K+pTJzg($uMfJ0Px!}K9Ze+iZ9&RoYsH;FwlB6r2&+zsln|=x3;jb|HWF!l?Tn7Ncnxk zHZL+p^nXV_z$*fd)$m;1O%3wjzqEfI2664*n(Y3i;rshCWtX$5fxI$%#}4fuMcUsL zq81FDD_!)n5&gH=_&q2fPq46iNo`Tl=)XUX(i&vIWYk>8mi>{|^XqD0SC=&FRouk* z2k7$q<4kQq=1sNh&&uzAK3&**2W0^0c`3W_2h;s65%}xD1e7XLdL>)mez1DcOhvetY-)dzik)jm}}>`y^vn_AarAs{*AK!Jt@lX9}?2JUY{+3 zFoxUd605jF5Mh_ZOqj&1ISLNWWd>~nluk>{9*3d*)97GMoUNbD)b6I(c|(Mzch6ujk_Qn{gq}pFgM{PigIwM zL=zefTPP`sz1V&x2f~rMI*&I8*2a^2kQ?F%jo#sK0!XYoHOb>oqzf#EY$8@b5%zt) zW}of(CRrp62Of{>XgY0VN(^_vUcg+o!LP#za2}F)+v~@)>t~d7Xbdm@aX; zJXW2|_i3~kqW~%U5-;a#C_KR-x54R4?sXCIV#pr}`DR^zB;=Qu&GXV}*X(?{JZY0O z){?Kafp^)N@>~h71^S)u15s+)ly!c0CDLh!Kbvy+g-S zlA0>CWbE0MizFn*MTvz&^~Rn|=+=I&YOSp9AypjBlb8g$E>L~kZob|GfJ}U{>w*~` zM%m-nLDP8ova~W7VlhAbd>q&7C3(RH2qb4}Mo2-M>mYr?5LZD-tkq#90pw3aJ-8G~ zJkhxvfBfVXXg+{X$d5c4YFMYBs2Hs0xj+Bqc=O)==S!`$2@Og%JGSEcT8||qRn|bb zh2fg?3RDo7e&jzmZGsE~6e2W{H+92&*I=-I05vdik`Z95lD&o?DunI(Q{c0P0iMf zd*{ zFl5m4JG5jQ2XucY+)De;j0uj1>^il@01z+kweB}_MUhsrtgu{RE_Y_sZ)SM*W){e! zLxVnN)38=Il`SSqfgI&h^`Zw+I|yHk0H!&1OCr{|9JxE5`xXeErCNx)?;j8vav*!0 zngh)au)(}+Jd|6V*8GA8EyGLunDEX+Id$!NhuqRPBSDp4j!{>E1UX(Z_{&OEKMt?Axo0;fy zk~}mw)~Wk)34(HGjfm!ep##4Dqa#}==YL!u@EBzIA0$BvBO$zKDb1<*yUgihVhbel z&yPYIS}y{T^Ff+6@U4es`HEO4%lhdvz zF#{Z-DNZq5kyK=A083qT>6v8TP2IC zw<*+^nKCg=v?(>E&mRcH3GCY`qCiSDpf=V9(von)R3d!_JMLIh4UMB0x0e~D2eGxU4AyGo$WbLv$ z6QruH$k;`2-zS(3J5n{5bm%mVAP!SWK}5f{G_==L()v4{_AfVgC3*d$z_uYrAjk}v z>p315T*l7^9xTG@h65{`2>Ebp}E`m)8dZQ`s3P2W?yJjB0s=TY&s5W11 z(KajqGbTS$CvU_s)XV@e7#I&Ossu6;*BylNeBY7bH&qlLLF*x$p=!E&LK$%mQqg0X z9DK~Up283CB3Yr62IZ|%#?P7Z!%9eO9zTAQMzXD!J*$S?Y4@d79?f_A*$(LE-jOW5 z9omljFIn}gwDMMhEyMuWTDixhr@RijEu0%kSI!r zgQ&0?k1_?(TQ7`T!+5++-fGu>NQ!4-l6s@k3KKd*ULRu%$DDn;$rtDz&7&S~xweLf$Fgm8SCCN6_?)FpKuL1uEtzU5lLvQpYfVb zqyGx*RT+hylTU(Z3c*g*(<}C|c7G(=RK@GBat6i*Eacc*^lKGZWfa&q{h3|U&j`3(&S@Cx|PvGWtH-L+RW&U=Xr?zqi3o|e@ z9V+crJ!3xxLFkTBQm91jocKQ$>ey7%p+U5{Q$Jc={u8?g5*uzK8Z zC^7s2FH|-d&FlpeC8dNSsvbJ$xYM$(oMi<^yBKhYmbWd*D?aRE)#llm<U(G7WC0f1j!ZoQh+q@xpp0UElzPo$9Y85h#^cAO(r}==kXDV{aES= z2E{E2AHz%CEt0Jz5ND5k?^`t7&M}+mSYt@Tf6e&`NlHo)5pU9ln>>7nR!(=aI! z(Uvqt4@X=wn@`(N#FQ+c@NAUD=1V~Y zSqSd!e%h7JPW>8<>3dpYF83ex31z_l(cX1NHMO+sgN+V3azq6YI4U5BC`hk@ij5`$ zA{_*zh8}u|R0Y985fA}Ulp;k+r~#=;mEIG236Rhe`W--z$#=fB?ppWWKlk3lA2!3@ zv*#`I%=5mpXXf1Ha)V?hbLxB6r^z`hAZ|WT@O8p6KfLpAYurv7`hvxWtXZ#q?j8C| z>0pTk*xEBv*6LqFEwJUh#YemY+S0dl^*xEK)(DCkX*#@;@Q7f_y5#nKdtkZAMTiGW*Z6PF1L>e zWGi>RpMQ=vE1SN?Cp){4k(Dl=GXN6Emq>l#Jt7}*;=4(*qTuXpp|<2Wk#7%P1SDB} zpuf>uq|_?^F!A&WYKS#MABCl^7or_Tpj;e8=_h-op|Im^P;?F2+q5kVOaKS-I z!6`y{PKm5Ap)ag(gD`Uu20cpBY$-NS3Fal5koY^?(XV(eJn8ctUDgpZ?& zxeGH%B0GfRnXeBdc7J3tS)r;f1!sQ2)-~=QD%!0UapHhq_lc}aXUGbNX{fUQTPj7w zoahZ*Nw%?B?@V_y0y%oHaV!ptm6*w@VNw>`+E|=Wj*}2y{9asK9Pf^e1}Ev5jMVTm zai5g2>I?y&bV%U}ETV02g%gTON_b5T4Yccsb6#HFybfj^E&b_}jnQIT{MHrl3FlTU z6ty|%8&*wkX=TNw6yw@(QL!=3!(BsTMfT#x@X<`!`Kso$v^1H;Xx3pphWiEETk*Di z`6CHXJ&)N4qo6rWN5^_bZ#(?uzN&~QefRm0<84WeiA%@vAIUWG9O!=iE;wYsq0a_p zJ5a>0z6ut`YG)5eXL>I;kJKe6!*B&7OL?wC)gH)V-tp z68Lv7mFW@*Qy<_cbCpHR_9ko^Tk{(3ZJ<9;j3hW%xk|5&)`?Y}*U1-5r7Nq{!0=Sn z2#faHHFjs4U_2eC7n{o<0qDH6sr)|Fa*62&~S{ny`khXOks#w+bU~?ON_t6Z@ z^pvf}cGciBbXWVqa<@>i;ViyP-NRZ~a9n1PMP`77%dL{W%Z#gAmC3e0PGt7u!!6Efv}1Faf_T3XD|dI6 z+e~&_25PNN)pg|Q(;?r_i4bGUpeoMHrf5fFb91c%jM!b7^_lIw+}tWZ*Q$W7*zk+E(c)unO@B;%q%@m%U~^;W@wWBXAM*28<~*Kh zxu)NBMZPzMx`wuMG?rW?fkedT!w_S>e4FV~Be3{QyJh4NW8TJ)ZQtalo*b30mP{ur z+Pk~g!;>LSmH5g1*+VyH4jye_R5`{#I20J1Yt@DNL&n?#?dDh;a-6A`t|U$@t(jj3 zO(U$xF6}%h^d#p#!{_7IsJ}=qPLvX2C@o~ek=NcfCx)calj{AEPjgiOq;Fm0l6vw9 zaUylLO<&W;FGGRsB3Y&;CMK>WZMcC_P7#`qoQJ`>Ah6y18n(j<Yf+vk8md8v)lsT^uLVQN_T{{5L$I)gEZ zxq(S{3nHD3YucwzgvAWCoL~X{WAZaAv!xWox54gsg;11;i2H(icZk9EJUhYPa~)Ug zQ9Nua(5enw<{sw&!`hvm`mSd_s6 zYuWt#7rb3lOl5f^Bb+!)*u2(*@SxWh3YQ7ai}7vlFDB7i$Uej!>TF@*4lUjJL_2Pq z$ff3f_i5I1aIVPk#5OL8TxjcQXG=rYXaIv7l_n8hvHZLxbx_HjI z0BATp#q!%==Br&6Np`I8=hjknin2hp->nPwyR9uHRWgOKvT|2J-)E@K3DfLTzzaF+ za^3#rzvbT#-sG<8^u(&iAFC@-5IbjG4>EIpkoDGEnLEJ0B_D?nik5P_bNBcgGq7-^ zq2W*>mwUimi=qnka?9qomLpwR;K1Xu<(_6VL^j@Hd$y`a%NTr^^n~bPEF`I7QaEV! z2S7-3mgpsK7_GoGYU*Rp{_C^Hp`y9_ZlM zJS0+fuKbjdkgg1Um>7e917EgUj@Xxg#!Vy>`@$FHmh;|Z1fyII3}yDrZda9Xwq6Dx_xD~&t8&jpRJnb!Rt`D6+4&n-F6)eWn*L8&yvqjV(u{*>f0W2 zCEk~Af}h0Ss20je{LIa>zXePtD@NWiN3rBIK zghY+BH?&6$D=VE9OkLaUu4kFYIp6nu;@9pDEGg}^mix@45;^F1fb~s>6JQtyumKHP z=GUyEv|YRktE!wLpfxC{xO=bqL-wP?-iMeg)PpHCuL+l}ImB;?Qrfu9DZkY_6kT&+ zZ6^NNVN#r;;Zzm3>CG)${i>rdgMG)!-e4&XMZZ(!uqe-F=2eDN>ZbAiu9m59Tww>l*NART=F9GJ@9608 zl}Zw`m3EowJ}#788%smd>i}K!@u8GF<6})lYApy)2%xLS6ia{7&s}mYw+{>?`@RN{ zven-Z6@VD~+qct_N=9`WP;jpQ`AStcAd*n@qq&#mi7bzP^&jx+*7M+hu`ZVv$r)}H zV7OHu>973i;^3)Oz;IV-dgI9|ILrhwW=ax)kclx0Xva9;LpYgkuHTjmQVQvnBUfW4 zVBMJso1`X@Pm$V zMh}#@1=F;KDgTP*0R$y19aNT@&ox$Rjp#Q1#u`P|aUGa>8X~&E8qray&unCmam4L| zVK!&e9|Op4)D}LKJ9Tr0>I*iK>~R_a5JRKyhLM>9fT{dHxBfc~sHzjsFJIqKplm2H|o$UtUh3ucIMTZ#KPX;KCu$Ta6P{G(Dt%b%)U-OknVB3hcuCzgCfKBeev zR@x7J%E>JBsu+E$78}LBI)hvOpg8teNMC05vt-hOM{%gZqCq_E*hrJtDADm`5HIBx zx^$``ihaxJWBetn*_2ybZic=MG5l8N`-qEtf>z;s_>Oo42EBtu?bJK{?=-)vy4J;v z`9W%t+wW1?d@G4-$^F}}*!wick%oN<~&8j>UNT#DV@5t`Bb>!eAZO{C&4@xS91R%o`kGn|lz) znJYfYQ~SEZ9&2iIB(R{u^Sj>N-OLjR+FZ9~D^07140puiphl#=$;q^o9DCtql-`>y zEg;kFfq}|o<}p6i+isv(%_+?jsB(dYx|xg#f6(iNP0TeMB;Z`VaNt*MWob4Ri)$)+ z=(yRYW#c)%pn6hvYIZfAIQ? z?%ZOwm@$%HTEJnll!^;2LdQ2yW@s`O_8@l%x(i+Oj-bJ8uR})2# z`N(TDS75M8Swq5`xlN4Q_Vopiy+p6Fe{VRnYdjt(VykR^Y<6(+hbHY6o&2puHkVNC zy#pHaGtwg~)@)OKk9ZB>3q{1aeDewz7E(O7^VX)G`aAhko){>rICg33TchhGiT3(km>MCP!jq!HX7ZnUXF& zsznp=wL!^u7x+(e4Bq^~V&$}H+dE|3n~9rd zEU3}Z9=JIDh~138GX1J-|5k55Zskr%MNgYv;R>Qin|7sRMmBw<8SCoorqy-`UXT;n zvYWXIj{`%zDRPwI6?iK(ty8Y%=2B=tq``{UgwnCiR#{saZ2NwhikOLT`3D1Q@;2it ziuDxm!O*zk^sTH}?WB+tIBfh8hB8^#qMnRl*aW^re#O4p)zcqZ zgFa>R)p1XD^V){FG=39|vKvTxNzo-ewohz2_AQP4jtnJIk`A@{Liuhr8(RVaUN)=H zS`#*pPL*1v1_tDOaW(I4>B5oE&80`-b)uu&inxca>(!fG4}yjHNA>`s6=XnfnXonN z_+64-LTTYzHvC~XvCQj6dTQ$M>o8;-Ou_1^Glk5AwqBpQMqk{jx^@R}b79@KYtol)PxDER4>ISakKk+sa2@!r0~-axCI7M&b{4I>Q=gFgr6 zdn+R2YW>#6+1g+ws0(OV)9^N(iZ@NHu*pgQF(%qDNkq+5X@Ejp99?SX|K^*JN0}zK zd+D6rKph;|9Y>fm*n>ZAjw*>=Sm3Zg@Tb*Etu+Redhkm2@}AtT4JcKjl)B*>lrx1* zpTw2v#+0g?d*D0jd)EkDP50Ue`mnLJc!MKIoic1!QStf~RjA~~Y~_bcTBx)03Or4z zU+}zBUfAP$`^4g;d~^6Aye+#m!8J{mP|JAPv9PKjhn7mgHP90`nv!*sO>s>8-#@ zMXAkEBIyOZ8xJ~5pl*0c^^;{I@fc(egmtD*ee>%q{m}mGM#+G0f>nr1r*@?aYqAOWaJOLpiUs{Uk8)5-i zs~$GXCcejQ_A0ppyf@kQ+wC1_{7b4a5Nk)`dnFxy6&;{@rwn54ffvyiNNzk(r|eF)UT~;TXc;da$dx30it04?GyHC#aKh>HPF&#InMP^`(3VPO zyE87`PA+7{h<89O@2a0F{`8k!87cuE4OB8$ByB9@`hC#a`~9wDt+`+geCGd2r3qXg zz$$-tQ{~yfR-(X3F*BaV+#M_@u|r0c1SM&gkmC>r4r&=$cZJkz<>Z2#8A$pl@HxvJ zmpvEpm~67t2Qqzvv@iaKXa}7!MVpt2Y*Legvs0YB87aR5XU)0GMLwS^pYBN3*aLS? z0Z%ncxlKOr>UVE~R3zZ0*MQ+0??m&E2zmfCq>1cr$n~cnmXCQas7B^B5aYQ0WFGU3 z1*DCHZ<)R&D+Pf6kK+C_#UWzeHZMVC7BpKX49V{0ct!&-K|T96*9|fYm+q5KUnDE} k$L9ZZmj4>cAhs!0B?XV?y_CHE8~C|#RryMWyy2t&0M8HASpWb4 literal 0 HcmV?d00001 diff --git a/docs/rtd-requirements.txt b/docs/rtd-requirements.txt index e32451b1..102c792a 100644 --- a/docs/rtd-requirements.txt +++ b/docs/rtd-requirements.txt @@ -1,3 +1,3 @@ m2r mock==3.0.5 -Sphinx==2.3.1 \ No newline at end of file +Sphinx==2.3.1 diff --git a/docs/source/CUSTOM-JOB.rst b/docs/source/CUSTOM-JOB.rst index 8d510a03..32c61214 100644 --- a/docs/source/CUSTOM-JOB.rst +++ b/docs/source/CUSTOM-JOB.rst @@ -25,7 +25,7 @@ The following variables will be used throughout the setup of the custom consumer .. py:data:: queue_name - Specifies the queue name that will be used to identify jobs for the :mod:`redis_consumer`, e.g. ``'track'`` + Specifies the queue name that will be used to identify jobs for the :doc:`redis_consumer:README`, e.g. ``'track'`` .. py:data:: consumer_name @@ -38,7 +38,7 @@ The following variables will be used throughout the setup of the custom consumer Designing a custom consumer --------------------------- -For guidance on the changes that need to be made to |kiosk-redis-consumer|, please see :doc:`redis_consumer:CUSTOM_CONSUMER` +For guidance on the changes that need to be made to |kiosk-redis-consumer|, please see :doc:`redis_consumer:README` .. |kiosk-redis-consumer| raw:: html @@ -47,7 +47,7 @@ For guidance on the changes that need to be made to |kiosk-redis-consumer|, plea Deploying a custom consumer --------------------------- -The DeepCell Kiosk uses |helm| and |helmfile| to coordinate Docker containers. This allows the :mod:`redis_consumer` to be easily extended by setting up a new docker image with your custom consumer. +The DeepCell Kiosk uses |helm| and |helmfile| to coordinate Docker containers. This allows the :doc:`redis_consumer:README` to be easily extended by setting up a new docker image with your custom consumer. 1. If you do not already have an account on `Docker Hub `_. Sign in to docker in your local environment using ``docker login``. @@ -110,6 +110,20 @@ The DeepCell Kiosk uses |helm| and |helmfile| to coordinate Docker containers. T nodeSelector: consumer: "yes" + hpa: + enabled: true + minReplicas: 1 + maxReplicas: 50 + metrics: + - type: Object + object: + metricName: tracking_consumer_key_ratio + target: + apiVersion: v1 + kind: Namespace + name: tracking_consumer_key_ratio + targetValue: 1 + env: DEBUG: "true" INTERVAL: 1 @@ -178,7 +192,7 @@ The DeepCell Kiosk uses |helm| and |helmfile| to coordinate Docker containers. T Autoscaling custom consumers ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Kubernetes scales each consumer using a `Horizonal Pod Autoscaler "https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/>`_ (HPA). +Kubernetes scales each consumer using a `Horizonal Pod Autoscaler `_ (HPA). Each HPA is configured in |/conf/addons/hpa.yaml|. The HPA reads a consumer-specific custom metric, defined in |/conf/helmfile.d/0600.prometheus-operator.yaml|. Each custom metric maximizes the work being done by balancing the amount of work left in the consumer's Redis queue (made available by the ``prometheus-redis-exporter``) and the current GPU utilization. @@ -188,13 +202,13 @@ For example, the ``segmentation_consumer_key_ratio`` in |/conf/helmfile.d/0600.p To effectively scale your new consumer, some small edits will be needed in the following files: -* |/conf/addons/redis-exporter-script.yaml| +* |/conf/helmfile.d/0110.prometheus-redis-exporter.yaml| * |/conf/helmfile.d/0600.prometheus-operator.yaml| -* |/conf/addons/hpa.yaml| +* ``/conf/helmfile.d/02##.custom-consumer.yaml`` -1. |/conf/addons/redis-exporter-script.yaml| +1. |/conf/helmfile.d/0110.prometheus-redis-exporter.yaml| - Within ``data.script`` modify the section ``All Queues to Monitor`` to include the new queue (:data:`queue_name`). + Within the ``data.script`` section of the ``prometheus-redis-exporter-script`` ConfigMap, modify the section ``All Queues to Monitor`` to include the new queue (:data:`queue_name`). .. code-block:: lua @@ -232,9 +246,9 @@ To effectively scale your new consumer, some small edits will be needed in the f namespace: deepcell service: tracking-scaling-service -3. |/conf/addons/hpa.yaml| +3. ``/conf/helmfile.d/02##.custom-consumer.yaml`` - Add a new section based on the example below to the bottom of ``hpa.yaml`` following a ``---``. + Finally, in the new consumer's helmfile, add the new metric to the ``hpa`` block. * Change ``metadata.name`` and ``spec.scaleTargetRef.name`` to :data:`consumer_name` * Change ``spec.metrics.object.metricName`` and ``spec.metrics.object.target.name`` to :data:`consumer_type` @@ -242,27 +256,19 @@ To effectively scale your new consumer, some small edits will be needed in the f .. code-block:: yaml :linenos: - apiVersion: autoscaling/v2beta1 - kind: HorizontalPodAutoscaler - metadata: - name: tracking-consumer - namespace: deepcell - spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: tracking-consumer - minReplicas: 1 - maxReplicas: {{ mul $max_gpus 50 }} - metrics: - - type: Object - object: - metricName: tracking_consumer_key_ratio - target: - apiVersion: v1 - kind: Namespace - name: tracking_consumer_key_ratio - targetValue: 1 + hpa: + enabled: true + minReplicas: 1 + maxReplicas: 50 + metrics: + - type: Object + object: + metricName: tracking_consumer_key_ratio + target: + apiVersion: v1 + kind: Namespace + name: tracking_consumer_key_ratio + targetValue: 1 .. |/conf/addons/hpa.yaml| raw:: html @@ -272,9 +278,9 @@ To effectively scale your new consumer, some small edits will be needed in the f /conf/helmfile.d/0600.prometheus-operator.yaml -.. |/conf/addons/redis-exporter-script.yaml| raw:: html +.. |/conf/helmfile.d/0110.prometheus-redis-exporter.yaml| raw:: html - /conf/addons/redis-exporter-script.yaml + /conf/helmfile.d/0110.prometheus-redis-exporter.yaml .. |/conf/helmfile.d/0230.redis-consumer.yaml| raw:: html diff --git a/docs/source/GETTING_STARTED.rst b/docs/source/GETTING_STARTED.rst index 49abd371..edf006b8 100644 --- a/docs/source/GETTING_STARTED.rst +++ b/docs/source/GETTING_STARTED.rst @@ -55,6 +55,7 @@ One of the enabling technologies the DeepCell Kiosk utilizes is `Docker