From 32d84e131dad0ba01c50a797480a703252ddd3fa Mon Sep 17 00:00:00 2001 From: Reuven Gonzales Date: Fri, 17 May 2024 16:23:05 -0700 Subject: [PATCH] ravenac95/basic dagster deployment (#1435) * Run proxy first * Updates for docker builds * Moving files around * move cloudquery images * Create a general container build/publisher * Re-enable main --- .github/scripts/publish-cloudquery-plugins.sh | 8 +- .github/scripts/publish-docker-containers.sh | 78 +++++-------------- docker/cloudquery-py-base.Dockerfile | 9 --- docker/cloudquery-py.Dockerfile | 7 -- docker/cloudquery-ts-base.Dockerfile | 9 --- docker/cloudquery-ts.Dockerfile | 9 --- docker/deploy-connector.sh | 20 ----- docker/images/cloudflare-tunnel/Dockerfile | 6 -- .../cloudflare-tunnel/cf-tunnel-wrapper.sh | 9 --- .../Dockerfile} | 0 docker/install.sh | 17 ---- .../warehouse/flux-system/gotk-sync.yaml | 2 +- ops/clusters/warehouse/production-apps.yaml | 1 + ops/clusters/warehouse/staging-apps.yaml | 1 + ops/helm-charts/oso-dagster/Chart.lock | 6 +- ops/helm-charts/oso-dagster/Chart.yaml | 6 +- .../oso-dagster/templates/_helpers.tpl | 14 +++- .../oso-dagster/templates/config-map.yaml | 8 +- ops/helm-charts/oso-dagster/values.yaml | 18 +---- .../base/cloudsql-proxy/kustomization.yaml | 5 ++ ops/k8s-apps/base/cloudsql-proxy/proxy.yaml | 42 ++++++++++ ops/k8s-apps/base/dagster/dagster.yaml | 44 +++++++---- ops/k8s-apps/base/dagster/kustomization.yaml | 3 +- ...{proxy.yaml => proxy-for-deployments.yaml} | 10 ++- ops/k8s-apps/base/dagster/proxy-for-jobs.yaml | 19 +++++ .../dagster/webserver-init-container.yaml | 2 +- ops/k8s-apps/base/podinfo/podinfo.yaml | 6 +- .../cloudsql-proxy/custom-helm-values.yaml | 12 +++ .../cloudsql-proxy/kustomization.yaml | 11 +++ .../production/custom-helm-values.yaml | 13 ---- .../dagster/custom-helm-values.yaml | 24 ++++++ .../production/dagster/kustomization.yaml | 11 +++ ops/k8s-apps/production/kustomization.yaml | 8 +- ops/k8s-apps/staging/kustomization.yaml | 3 +- .../staging/podinfo/kustomization.yaml | 5 ++ 35 files changed, 232 insertions(+), 214 deletions(-) delete mode 100644 docker/cloudquery-py-base.Dockerfile delete mode 100644 docker/cloudquery-py.Dockerfile delete mode 100644 docker/cloudquery-ts-base.Dockerfile delete mode 100644 docker/cloudquery-ts.Dockerfile delete mode 100644 docker/deploy-connector.sh delete mode 100644 docker/images/cloudflare-tunnel/Dockerfile delete mode 100644 docker/images/cloudflare-tunnel/cf-tunnel-wrapper.sh rename docker/images/{dagster-dask.Dockerfile => dagster-dask/Dockerfile} (100%) delete mode 100644 docker/install.sh create mode 100644 ops/k8s-apps/base/cloudsql-proxy/kustomization.yaml create mode 100644 ops/k8s-apps/base/cloudsql-proxy/proxy.yaml rename ops/k8s-apps/base/dagster/{proxy.yaml => proxy-for-deployments.yaml} (69%) create mode 100644 ops/k8s-apps/base/dagster/proxy-for-jobs.yaml create mode 100644 ops/k8s-apps/production/cloudsql-proxy/custom-helm-values.yaml create mode 100644 ops/k8s-apps/production/cloudsql-proxy/kustomization.yaml delete mode 100644 ops/k8s-apps/production/custom-helm-values.yaml create mode 100644 ops/k8s-apps/production/dagster/custom-helm-values.yaml create mode 100644 ops/k8s-apps/production/dagster/kustomization.yaml create mode 100644 ops/k8s-apps/staging/podinfo/kustomization.yaml diff --git a/.github/scripts/publish-cloudquery-plugins.sh b/.github/scripts/publish-cloudquery-plugins.sh index 476de0139..740b127ef 100644 --- a/.github/scripts/publish-cloudquery-plugins.sh +++ b/.github/scripts/publish-cloudquery-plugins.sh @@ -14,8 +14,8 @@ tag="$(git rev-parse HEAD)" build_base_image() { language="$1" tag="$2" - base_image="ghcr.io/opensource-observer/cloudquery-${language}-base:${tag}" - dockerfile_path="./docker/cloudquery-${language}-base.Dockerfile" + base_image="ghcr.io/opensource-observer/${language}-base:${tag}" + dockerfile_path="./docker/cloudquery/${language}-base.Dockerfile" docker build -t "${base_image}" -f "${dockerfile_path}" . echo $base_image } @@ -36,7 +36,7 @@ for path in $ts_plugins; do docker build -t ${plugin_image} \ --build-arg PLUGIN_NAME=${plugin_name} \ --build-arg BASE_IMAGE=${ts_base_image} \ - -f docker/cloudquery-ts.Dockerfile \ + -f docker/cloudquery/ts.Dockerfile \ . echo "Publishing the plugin to ${plugin_image}" docker push ${plugin_image} @@ -60,7 +60,7 @@ for path in $python_plugins; do --build-arg PLUGIN_NAME=${plugin_name} \ --build-arg PLUGIN_CMD=${plugin_cmd} \ --build-arg BASE_IMAGE=${ts_base_image} \ - -f docker/cloudquery-py.Dockerfile \ + -f docker/cloudquery/py.Dockerfile \ . echo "Publishing the plugin to ${plugin_image}" diff --git a/.github/scripts/publish-docker-containers.sh b/.github/scripts/publish-docker-containers.sh index 740b127ef..d1be002fd 100644 --- a/.github/scripts/publish-docker-containers.sh +++ b/.github/scripts/publish-docker-containers.sh @@ -5,64 +5,28 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "${SCRIPT_DIR}/../../" REPO_DIR=$(pwd) -# For now if a plugin has both a pyproject.toml and a package.json. The python -# will be used by nature of the order of docker image publishing -python_plugins="$(find ./warehouse/cloudquery-* -type f -name 'pyproject.toml' -exec sh -c 'dirname $0' {} \;)" -ts_plugins="$(find ./warehouse/cloudquery-* -type f -name 'package.json' -exec sh -c 'dirname $0' {} \;)" +# Publish all images +images_to_build="$(find ./docker/images/* -type f -name 'Dockerfile' -exec sh -c 'dirname $0' {} \;)" tag="$(git rev-parse HEAD)" -build_base_image() { - language="$1" - tag="$2" - base_image="ghcr.io/opensource-observer/${language}-base:${tag}" - dockerfile_path="./docker/cloudquery/${language}-base.Dockerfile" - docker build -t "${base_image}" -f "${dockerfile_path}" . - echo $base_image -} - -# Build the base images -py_base_image=$(build_base_image py $tag) -ts_base_image=$(build_base_image ts $tag) -prefix="cloudquery-" - -for path in $ts_plugins; do - plugin_name=$(basename $path) - # Remove the cloudquery prefix - plugin_name=${plugin_name#"$prefix"} - - plugin_image="ghcr.io/opensource-observer/cloudquery-${plugin_name}:${tag}" - - echo "Building ${plugin_name} plugin" - docker build -t ${plugin_image} \ - --build-arg PLUGIN_NAME=${plugin_name} \ - --build-arg BASE_IMAGE=${ts_base_image} \ - -f docker/cloudquery/ts.Dockerfile \ +for path in $images_to_build; do + image_name=$(basename $path) + + image_repo="ghcr.io/opensource-observer/${image_name}" + sha_image="${image_repo}:${tag}" + latest_image="${image_repo}:latest" + + echo "Building ${image_name} plugin" + docker build \ + -t ${sha_image} \ + -t ${latest_image} \ + --label "org.opencontainers.image.source=https://github.com/opensource-observer/oso" \ + --label "observer.opensource.oso.sha=${tag}" \ + --build-arg IMAGE_NAME=${image_name} + -f docker/images/${IMAGE_NAME} \ . - echo "Publishing the plugin to ${plugin_image}" - docker push ${plugin_image} + echo "Publishing the image to ${sha_image}" + docker push "${sha_image}" + echo "Publishing latest to ${latest_image}" + docker push "${latest_image}" done - -for path in $python_plugins; do - plugin_name=$(basename $path) - # Remove the cloudquery prefix - plugin_name=${plugin_name#"$prefix"} - - plugin_cmd=$(echo $plugin_name | sed "s/-/_/g") - plugin_image="ghcr.io/opensource-observer/cloudquery-${plugin_name}:${tag}" - - # Skip the example - if [[ $plugin_name = "example_plugin" ]]; then - continue - fi - echo "Building ${plugin_name} plugin" - - docker build -t ${plugin_image} \ - --build-arg PLUGIN_NAME=${plugin_name} \ - --build-arg PLUGIN_CMD=${plugin_cmd} \ - --build-arg BASE_IMAGE=${ts_base_image} \ - -f docker/cloudquery/py.Dockerfile \ - . - - echo "Publishing the plugin to ${plugin_image}" - docker push ${plugin_image} -done \ No newline at end of file diff --git a/docker/cloudquery-py-base.Dockerfile b/docker/cloudquery-py-base.Dockerfile deleted file mode 100644 index ed89157aa..000000000 --- a/docker/cloudquery-py-base.Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM python:3.12-bookworm - -RUN pip install poetry - -COPY . /usr/src/app - -WORKDIR /usr/src/app - -RUN poetry install diff --git a/docker/cloudquery-py.Dockerfile b/docker/cloudquery-py.Dockerfile deleted file mode 100644 index d08df2762..000000000 --- a/docker/cloudquery-py.Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -ARG BASE_IMAGE=ghcr.io/opensource-observer/cloudquery-py-base:latest - -FROM ${BASE_IMAGE} - -ARG PLUGIN_NAME - -ENTRYPOINT [ "${PLUGIN_NAME}" ] \ No newline at end of file diff --git a/docker/cloudquery-ts-base.Dockerfile b/docker/cloudquery-ts-base.Dockerfile deleted file mode 100644 index e5e477c08..000000000 --- a/docker/cloudquery-ts-base.Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM node:20 as build - -RUN npm install -g pnpm@^9.0.0 - -COPY . /usr/src/app - -WORKDIR /usr/src/app - -RUN pnpm install && pnpm build:cloudquery \ No newline at end of file diff --git a/docker/cloudquery-ts.Dockerfile b/docker/cloudquery-ts.Dockerfile deleted file mode 100644 index dcb4c4b46..000000000 --- a/docker/cloudquery-ts.Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -ARG BASE_IMAGE=ghcr.io/opensource-observer/cloudquery-ts-base:latest - -FROM ${BASE_IMAGE} - -ARG PLUGIN_NAME - -WORKDIR /usr/src/app/warehouse/cloudquery-${PLUGIN_NAME} - -ENTRYPOINT [ "pnpm", "node", "--loader", "ts-node/esm", "src/main.ts" ] \ No newline at end of file diff --git a/docker/deploy-connector.sh b/docker/deploy-connector.sh deleted file mode 100644 index 8efa3f837..000000000 --- a/docker/deploy-connector.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -# Builds a ts connector into a docker container and pushes it to github packages. -# -# Usage: -# deploy-connector.sh {connector_source_name} {docker_image_version} -set -euxo pipefail - -connector_source_name=$1 -docker_image_name=${connector_source_name}-airbyte-connector -docker_image_version=$2 - -DOCKER_REPO=${DOCKER_REPO:-ghcr.io/opensource-observer/${docker_image_name}} -docker_tag="${DOCKER_REPO}:${docker_image_version}" -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -# Go to the root -cd "${SCRIPT_DIR}/../" - -docker build --build-arg SOURCE_DIR="connectors/${connector_source_name}" -t "${docker_tag}" -f ./docker/ts-connector.Dockerfile . -docker push "${docker_tag}" \ No newline at end of file diff --git a/docker/images/cloudflare-tunnel/Dockerfile b/docker/images/cloudflare-tunnel/Dockerfile deleted file mode 100644 index 7ed0f4896..000000000 --- a/docker/images/cloudflare-tunnel/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -FROM ghcr.io/strrl/cloudflare-tunnel-ingress-controller:latest as binary - -FROM alpine:3.19 -COPY --from=binary /usr/bin/cloudflare-tunnel-ingress-controller /usr/bin/cloudflare-tunnel-ingress-controller -COPY ./cf-tunnel-wrapper.sh /usr/bin/cf-tunnel-wrapper.sh - diff --git a/docker/images/cloudflare-tunnel/cf-tunnel-wrapper.sh b/docker/images/cloudflare-tunnel/cf-tunnel-wrapper.sh deleted file mode 100644 index cdcdd7eee..000000000 --- a/docker/images/cloudflare-tunnel/cf-tunnel-wrapper.sh +++ /dev/null @@ -1,9 +0,0 @@ -# This is to enable the use of kube-secrets-init -#!/bin/sh -cloudflare-tunnel-ingress-controller \ ---ingress-class=${INGRESS_CLASS} \ ---controller-class=${CONTROLLER_CLASS} \ ---cloudflare-api-token=${CLOUDFLARE_API_TOKEN} \ ---cloudflare-account-id=${CLOUDFLARE_ACCOUNT_ID} \ ---cloudflare-tunnel-name=${CLOUDFLARE_TUNNEL_NAME} \ ---namespace=${NAMESPACE} diff --git a/docker/images/dagster-dask.Dockerfile b/docker/images/dagster-dask/Dockerfile similarity index 100% rename from docker/images/dagster-dask.Dockerfile rename to docker/images/dagster-dask/Dockerfile diff --git a/docker/install.sh b/docker/install.sh deleted file mode 100644 index ecc89af7a..000000000 --- a/docker/install.sh +++ /dev/null @@ -1,17 +0,0 @@ -set -euxo pipefail - -# Ensure all of the node modules directorys aren't empty. A change in pnpm v8.12 -# caused it to attempt deletion of empty node_modules directories. This fixes -# that issue. -touch /usr/src/app/node_modules/.noop -touch /usr/src/app/indexer/node_modules/.noop -touch /usr/src/app/frontend/node_modules/.noop -touch /usr/src/app/docs/node_modules/.noop - -npm install -g pnpm -node -v - -pnpm i --frozen-lockfile --ignore-scripts - -# For now let's just build the indexer -pnpm build:indexer \ No newline at end of file diff --git a/ops/clusters/warehouse/flux-system/gotk-sync.yaml b/ops/clusters/warehouse/flux-system/gotk-sync.yaml index 31d5b0858..90a97f81a 100644 --- a/ops/clusters/warehouse/flux-system/gotk-sync.yaml +++ b/ops/clusters/warehouse/flux-system/gotk-sync.yaml @@ -6,7 +6,7 @@ metadata: name: flux-system namespace: flux-system spec: - interval: 1m0s + interval: 1m ref: branch: main secretRef: diff --git a/ops/clusters/warehouse/production-apps.yaml b/ops/clusters/warehouse/production-apps.yaml index 32d1b6243..f99c3afb0 100644 --- a/ops/clusters/warehouse/production-apps.yaml +++ b/ops/clusters/warehouse/production-apps.yaml @@ -14,5 +14,6 @@ spec: kind: GitRepository name: flux-system path: ./ops/k8s-apps/production + force: true prune: true wait: true \ No newline at end of file diff --git a/ops/clusters/warehouse/staging-apps.yaml b/ops/clusters/warehouse/staging-apps.yaml index 527cc5069..e13fe7b8e 100644 --- a/ops/clusters/warehouse/staging-apps.yaml +++ b/ops/clusters/warehouse/staging-apps.yaml @@ -14,5 +14,6 @@ spec: kind: GitRepository name: flux-system path: ./ops/k8s-apps/staging + force: true prune: true wait: true \ No newline at end of file diff --git a/ops/helm-charts/oso-dagster/Chart.lock b/ops/helm-charts/oso-dagster/Chart.lock index 60cf27d9b..6aba000b6 100644 --- a/ops/helm-charts/oso-dagster/Chart.lock +++ b/ops/helm-charts/oso-dagster/Chart.lock @@ -1,6 +1,6 @@ dependencies: - name: dagster repository: https://dagster-io.github.io/helm - version: 1.2.3 -digest: sha256:0848a02ca7fb9b0566de902f707cfdd65ea5073be847f3de49253d56bfd59a8c -generated: "2024-05-16T22:56:26.894848565Z" + version: 1.7.6 +digest: sha256:3f50e0ae14880ffe7c03295ce0fc605fb4e4c9c32fdf11b24606285807c5de99 +generated: "2024-05-17T17:18:41.876405486Z" diff --git a/ops/helm-charts/oso-dagster/Chart.yaml b/ops/helm-charts/oso-dagster/Chart.yaml index c43899a1c..2a760a0ff 100644 --- a/ops/helm-charts/oso-dagster/Chart.yaml +++ b/ops/helm-charts/oso-dagster/Chart.yaml @@ -3,9 +3,9 @@ name: oso-dagster description: Extension of the dagster template type: application -version: 0.1.2 -appVersion: "1.16.0" +version: 0.1.10 +appVersion: "1.0.0" dependencies: - name: dagster - version: "1.2.3" + version: "1.7.6" repository: "https://dagster-io.github.io/helm" \ No newline at end of file diff --git a/ops/helm-charts/oso-dagster/templates/_helpers.tpl b/ops/helm-charts/oso-dagster/templates/_helpers.tpl index 8b3597627..87d766aa7 100644 --- a/ops/helm-charts/oso-dagster/templates/_helpers.tpl +++ b/ops/helm-charts/oso-dagster/templates/_helpers.tpl @@ -2,8 +2,10 @@ Expand the name of the chart. */}} +# Disable the pgisready check due to our use of cloudsql proxy injected into the +pod. {{- define "dagster.postgresql.pgisready" -}} -until pg_isready -h ${DAGSTER_PG_HOST} -p ${DAGSTER_PG_PORT} -U ${DAGSTER_PG_USER}; do echo waiting for database; sleep 2; done; +sleep 5; {{- end }} {{- define "dagsterYaml.postgresql.config" }} @@ -22,4 +24,12 @@ postgres_db: {{- if .Values.postgresql.postgresqlScheme }} scheme: {{ .Values.postgresql.postgresqlScheme }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} + +# Fix issues with the full name +{{- define "dagster.webserver.fullname" -}} +{{- $name := default "webserver" .Values.dagsterWebserver.nameOverride -}} +{{- $fullname := include "dagster.fullname" . -}} +{{- printf "%s-%s" $fullname $name | trunc 63 | trimSuffix "-" -}} +{{- if .webserverReadOnly -}} -read-only {{- end -}} +{{- end -}} \ No newline at end of file diff --git a/ops/helm-charts/oso-dagster/templates/config-map.yaml b/ops/helm-charts/oso-dagster/templates/config-map.yaml index 590771e7b..a198bea3b 100644 --- a/ops/helm-charts/oso-dagster/templates/config-map.yaml +++ b/ops/helm-charts/oso-dagster/templates/config-map.yaml @@ -1,7 +1,9 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ .Release.Name }}-oso-extra-env + name: dagster-oso-extra-env data: - DAGSTER_PG_USER: {{ .Values.configMap.secretPrefix }}-{{ .Values.secretmanagerKeys.dbUser }} - DAGSTER_PG_DB_NAME: {{ .Values.configMap.secretPrefix }}-{{ .Values.secretmanagerKeys.dbName }} \ No newline at end of file + DAGSTER_PG_USER: "{{ .Values.configMap.secretPrefix }}-{{ .Values.secretmanagerKeys.dbUser }}" + DAGSTER_PG_DB_NAME: "{{ .Values.configMap.secretPrefix }}-{{ .Values.secretmanagerKeys.dbName }}" + DAGSTER_PG_PORT: "{{ .Values.pg.port }}" + DAGSTER_PG_HOST: "{{ .Values.pg.host }}" \ No newline at end of file diff --git a/ops/helm-charts/oso-dagster/values.yaml b/ops/helm-charts/oso-dagster/values.yaml index 4fb0e1c26..4e57cfc47 100644 --- a/ops/helm-charts/oso-dagster/values.yaml +++ b/ops/helm-charts/oso-dagster/values.yaml @@ -10,19 +10,6 @@ dagster: postgresqlPassword: "gcp:secretmanager:dagster-db-password/versions/1" enableReadOnly: true - dagsterWebserver: - env: - - name: DAGSTER_PG_HOST - value: set-by-proxy-operator - - name: DAGSTER_PG_PORT - value: set-by-proxy-operator - - dagsterDaemon: - env: - - name: DAGSTER_PG_HOST - value: set-by-proxy-operator - - name: DAGSTER_PG_PORT - value: set-by-proxy-operator configMap: name: "dagster-extra-env-config-map" secretPrefix: "gcp:secretmanager:dagster" @@ -30,4 +17,7 @@ secretmanagerKeys: dbUser: "db-user/versions/1" dbHost: "db-host/versions/1" dbPort: "db-port/versions/1" - dbName: "db-name/versions/1" \ No newline at end of file + dbName: "db-name/versions/1" +pg: + port: "5432" + host: "127.0.0.1" \ No newline at end of file diff --git a/ops/k8s-apps/base/cloudsql-proxy/kustomization.yaml b/ops/k8s-apps/base/cloudsql-proxy/kustomization.yaml new file mode 100644 index 000000000..9f6a927db --- /dev/null +++ b/ops/k8s-apps/base/cloudsql-proxy/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: base-cloudsql-proxy +resources: + - proxy.yaml \ No newline at end of file diff --git a/ops/k8s-apps/base/cloudsql-proxy/proxy.yaml b/ops/k8s-apps/base/cloudsql-proxy/proxy.yaml new file mode 100644 index 000000000..3e5370204 --- /dev/null +++ b/ops/k8s-apps/base/cloudsql-proxy/proxy.yaml @@ -0,0 +1,42 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: base-cloudsql-proxy + labels: + toolkit.fluxcd.io/tenant: apps + ops.opensource.observer/environment: base + kube-secrets-init.doit-intl.com/enable-mutation: "true" +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: rimusz + namespace: base-cloudsql-proxy +spec: + interval: 5m + url: https://charts.rimusz.net +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: base-cloudsql-proxy + namespace: base-cloudsql-proxy +spec: + chart: + spec: + chart: gcloud-sqlproxy + version: "0.25.6" + sourceRef: + kind: HelmRepository + name: rimusz + interval: 50m + install: + remediation: + retries: 3 + values: + cloudsql: + instances: + - instance: oso-playground-psql + region: us-central1 + project: opensource-observer + port: 5432 \ No newline at end of file diff --git a/ops/k8s-apps/base/dagster/dagster.yaml b/ops/k8s-apps/base/dagster/dagster.yaml index 8e8724d3a..7c45742a6 100644 --- a/ops/k8s-apps/base/dagster/dagster.yaml +++ b/ops/k8s-apps/base/dagster/dagster.yaml @@ -36,12 +36,40 @@ spec: host: base-admin-dagster.opensource.observer readOnlyDagsterWebserver: host: base-readonly-dagster.opensource.observer + runLauncher: + config: + k8sRunLauncher: + envConfigMaps: + - name: dagster-oso-extra-env + # runK8sConfig: + # podSpecConfig: + # tolerations: + # - key: pool_type + # effect: NoSchedule + # operator: Equal + # value: spot + # nodeSelector: + # pool_type: spot dagsterWebserver: envConfigMaps: - name: dagster-oso-extra-env + resources: + limits: + cpu: 150m + memory: 128Mi + requests: + cpu: 100m + memory: 128Mi dagsterDaemon: envConfigMaps: - name: dagster-oso-extra-env + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 400m + memory: 256Mi postRenderers: - kustomize: patches: @@ -49,7 +77,7 @@ spec: apiVersion: apps/v1 kind: Deployment metadata: - name: dagster-daemon + name: all-deployments spec: template: spec: @@ -58,16 +86,6 @@ spec: envFrom: - configMapRef: name: dagster-oso-extra-env - - patch: | - apiVersion: apps/v1 + target: kind: Deployment - metadata: - name: dagster-dagit - spec: - template: - spec: - initContainers: - - name: check-db-ready - envFrom: - - configMapRef: - name: dagster-oso-extra-env + labelSelector: app.kubernetes.io/name=dagster diff --git a/ops/k8s-apps/base/dagster/kustomization.yaml b/ops/k8s-apps/base/dagster/kustomization.yaml index e537137bd..5a5cb0523 100644 --- a/ops/k8s-apps/base/dagster/kustomization.yaml +++ b/ops/k8s-apps/base/dagster/kustomization.yaml @@ -2,8 +2,9 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: base-dagster resources: + #- proxy-for-deployments.yaml + #- proxy-for-jobs.yaml - dagster.yaml - - proxy.yaml # patchesStrategicMerge: # - ./daemon-init-container.yaml # - ./webserver-init-container.yaml \ No newline at end of file diff --git a/ops/k8s-apps/base/dagster/proxy.yaml b/ops/k8s-apps/base/dagster/proxy-for-deployments.yaml similarity index 69% rename from ops/k8s-apps/base/dagster/proxy.yaml rename to ops/k8s-apps/base/dagster/proxy-for-deployments.yaml index e3ae1faad..a3e604c45 100644 --- a/ops/k8s-apps/base/dagster/proxy.yaml +++ b/ops/k8s-apps/base/dagster/proxy-for-deployments.yaml @@ -1,16 +1,18 @@ apiVersion: cloudsql.cloud.google.com/v1 kind: AuthProxyWorkload metadata: - name: dagster-auth-proxy + name: dagster-deployment-cloudsql-proxy spec: authProxyContainer: resources: requests: - cpu: "1" - memory: "500Mi" + cpu: "50m" + memory: "64Mi" workloadSelector: kind: "Deployment" # Applies to a "Deployment" - name: dagster-daemon + selector: + matchLabels: + app.kubernetes.io/name: dagster instances: - connectionString: "opensource-observer:us-central1:oso-psql" portEnvName: "DAGSTER_PG_PORT" diff --git a/ops/k8s-apps/base/dagster/proxy-for-jobs.yaml b/ops/k8s-apps/base/dagster/proxy-for-jobs.yaml new file mode 100644 index 000000000..dfe56d6e2 --- /dev/null +++ b/ops/k8s-apps/base/dagster/proxy-for-jobs.yaml @@ -0,0 +1,19 @@ +apiVersion: cloudsql.cloud.google.com/v1 +kind: AuthProxyWorkload +metadata: + name: dagster-job-cloudsql-proxy +spec: + authProxyContainer: + resources: + requests: + cpu: "50m" + memory: "64Mi" + workloadSelector: + kind: "Job" + selector: + matchLabels: + app.kubernetes.io/name: dagster + instances: + - connectionString: "opensource-observer:us-central1:oso-psql" + portEnvName: "DAGSTER_PG_PORT" + hostEnvName: "DAGSTER_PG_HOST" \ No newline at end of file diff --git a/ops/k8s-apps/base/dagster/webserver-init-container.yaml b/ops/k8s-apps/base/dagster/webserver-init-container.yaml index 4de32e14d..275884912 100644 --- a/ops/k8s-apps/base/dagster/webserver-init-container.yaml +++ b/ops/k8s-apps/base/dagster/webserver-init-container.yaml @@ -1,7 +1,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: dagster-dagit + name: dagster-webserver spec: template: spec: diff --git a/ops/k8s-apps/base/podinfo/podinfo.yaml b/ops/k8s-apps/base/podinfo/podinfo.yaml index ddbbf4516..b5891188b 100644 --- a/ops/k8s-apps/base/podinfo/podinfo.yaml +++ b/ops/k8s-apps/base/podinfo/podinfo.yaml @@ -47,4 +47,8 @@ spec: enabled: true extraEnvs: - name: CLOUDFLARE_SECRET - value: "gcp:secretmanager:podinfo-test-secret/versions/1" \ No newline at end of file + value: "gcp:secretmanager:podinfo-test-secret/versions/1" + resources: + limits: + cpu: 50m + memory: 50Mi \ No newline at end of file diff --git a/ops/k8s-apps/production/cloudsql-proxy/custom-helm-values.yaml b/ops/k8s-apps/production/cloudsql-proxy/custom-helm-values.yaml new file mode 100644 index 000000000..bde40d252 --- /dev/null +++ b/ops/k8s-apps/production/cloudsql-proxy/custom-helm-values.yaml @@ -0,0 +1,12 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: production-cloudsql-proxy +spec: + values: + cloudsql: + instances: + - instance: oso-psql + region: us-central1 + project: opensource-observer + port: 5432 \ No newline at end of file diff --git a/ops/k8s-apps/production/cloudsql-proxy/kustomization.yaml b/ops/k8s-apps/production/cloudsql-proxy/kustomization.yaml new file mode 100644 index 000000000..81ffb281f --- /dev/null +++ b/ops/k8s-apps/production/cloudsql-proxy/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../../base/cloudsql-proxy +namespace: production-cloudsql-proxy +patches: + - path: ./custom-helm-values.yaml + target: + kind: HelmRelease + options: + allowNameChange: true \ No newline at end of file diff --git a/ops/k8s-apps/production/custom-helm-values.yaml b/ops/k8s-apps/production/custom-helm-values.yaml deleted file mode 100644 index baf0b74d5..000000000 --- a/ops/k8s-apps/production/custom-helm-values.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: dagster -spec: - values: - global: - serviceAccountName: production-dagster - configMap: - secretPrefix: "gcp:secretmanager:production-dagster" - dagster: - global: - serviceAccountName: production-dagster \ No newline at end of file diff --git a/ops/k8s-apps/production/dagster/custom-helm-values.yaml b/ops/k8s-apps/production/dagster/custom-helm-values.yaml new file mode 100644 index 000000000..5393a1978 --- /dev/null +++ b/ops/k8s-apps/production/dagster/custom-helm-values.yaml @@ -0,0 +1,24 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: production-dagster +spec: + values: + pg: + host: production-cloudsql-proxy-gcloud-sqlproxy.production-cloudsql-proxy.svc.cluster.local + port: "5432" + global: + serviceAccountName: production-dagster + configMap: + secretPrefix: "gcp:secretmanager:production-dagster" + dagster: + global: + serviceAccountName: production-dagster + ingress: + enabled: true + ingressClassName: ingress-internal-cloudflare + dagsterWebserver: + host: admin-dagster.opensource.observer + readOnlyDagsterWebserver: + host: readonly-dagster.opensource.observer + \ No newline at end of file diff --git a/ops/k8s-apps/production/dagster/kustomization.yaml b/ops/k8s-apps/production/dagster/kustomization.yaml new file mode 100644 index 000000000..b48dca704 --- /dev/null +++ b/ops/k8s-apps/production/dagster/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../../base/dagster +namespace: production-dagster +patches: + - path: ./custom-helm-values.yaml + target: + kind: HelmRelease + options: + allowNameChange: true diff --git a/ops/k8s-apps/production/kustomization.yaml b/ops/k8s-apps/production/kustomization.yaml index a750670f0..44ba1573c 100644 --- a/ops/k8s-apps/production/kustomization.yaml +++ b/ops/k8s-apps/production/kustomization.yaml @@ -1,9 +1,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - ../base/dagster -namespace: production-dagster -patches: - - path: ./custom-helm-values.yaml - target: - type: HelmRelease \ No newline at end of file + - ./dagster + - ./cloudsql-proxy diff --git a/ops/k8s-apps/staging/kustomization.yaml b/ops/k8s-apps/staging/kustomization.yaml index 47b1eabdd..52db45ed6 100644 --- a/ops/k8s-apps/staging/kustomization.yaml +++ b/ops/k8s-apps/staging/kustomization.yaml @@ -1,5 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -namespace: staging-podinfo resources: - - ../base/podinfo \ No newline at end of file + - ./podinfo \ No newline at end of file diff --git a/ops/k8s-apps/staging/podinfo/kustomization.yaml b/ops/k8s-apps/staging/podinfo/kustomization.yaml new file mode 100644 index 000000000..a6c7d9dc2 --- /dev/null +++ b/ops/k8s-apps/staging/podinfo/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: staging-podinfo +resources: + - ../../base/podinfo \ No newline at end of file