ravenac95/basic dagster deployment (#1435)

* Run proxy first

* Updates for docker builds

* Moving files around

* move cloudquery images

* Create a general container build/publisher

* Re-enable main

.github/scripts/publish-cloudquery-plugins.sh

@@ -14,8 +14,8 @@ tag="$(git rev-parse HEAD)"
 build_base_image() {
     language="$1"
     tag="$2"
-    base_image="ghcr.io/opensource-observer/cloudquery-${language}-base:${tag}"
-    dockerfile_path="./docker/cloudquery-${language}-base.Dockerfile"
+    base_image="ghcr.io/opensource-observer/${language}-base:${tag}"
+    dockerfile_path="./docker/cloudquery/${language}-base.Dockerfile"
     docker build -t "${base_image}" -f "${dockerfile_path}" .
     echo $base_image
 }
@@ -36,7 +36,7 @@ for path in $ts_plugins; do
     docker build -t ${plugin_image} \
         --build-arg PLUGIN_NAME=${plugin_name} \
         --build-arg BASE_IMAGE=${ts_base_image} \
-        -f docker/cloudquery-ts.Dockerfile \
+        -f docker/cloudquery/ts.Dockerfile \
         .
     echo "Publishing the plugin to ${plugin_image}"
     docker push ${plugin_image}
@@ -60,7 +60,7 @@ for path in $python_plugins; do
         --build-arg PLUGIN_NAME=${plugin_name} \
         --build-arg PLUGIN_CMD=${plugin_cmd} \
         --build-arg BASE_IMAGE=${ts_base_image} \
-        -f docker/cloudquery-py.Dockerfile \
+        -f docker/cloudquery/py.Dockerfile \
         .
 
     echo "Publishing the plugin to ${plugin_image}"

.github/scripts/publish-docker-containers.sh

@@ -5,64 +5,28 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 cd "${SCRIPT_DIR}/../../"
 REPO_DIR=$(pwd)
 
-# For now if a plugin has both a pyproject.toml and a package.json. The python
-# will be used by nature of the order of docker image publishing
-python_plugins="$(find ./warehouse/cloudquery-* -type f -name 'pyproject.toml' -exec sh -c 'dirname $0' {} \;)"
-ts_plugins="$(find ./warehouse/cloudquery-* -type f -name 'package.json' -exec sh -c 'dirname $0' {} \;)"
+# Publish all images 
+images_to_build="$(find ./docker/images/* -type f -name 'Dockerfile' -exec sh -c 'dirname $0' {} \;)"
 tag="$(git rev-parse HEAD)"
 
-build_base_image() {
-    language="$1"
-    tag="$2"
-    base_image="ghcr.io/opensource-observer/${language}-base:${tag}"
-    dockerfile_path="./docker/cloudquery/${language}-base.Dockerfile"
-    docker build -t "${base_image}" -f "${dockerfile_path}" .
-    echo $base_image
-}
-
-# Build the base images
-py_base_image=$(build_base_image py $tag)
-ts_base_image=$(build_base_image ts $tag)
-prefix="cloudquery-"
-
-for path in $ts_plugins; do
-    plugin_name=$(basename $path)
-    # Remove the cloudquery prefix
-    plugin_name=${plugin_name#"$prefix"}
-
-    plugin_image="ghcr.io/opensource-observer/cloudquery-${plugin_name}:${tag}"
-
-    echo "Building ${plugin_name} plugin"
-    docker build -t ${plugin_image} \
-        --build-arg PLUGIN_NAME=${plugin_name} \
-        --build-arg BASE_IMAGE=${ts_base_image} \
-        -f docker/cloudquery/ts.Dockerfile \
+for path in $images_to_build; do
+    image_name=$(basename $path)
+
+    image_repo="ghcr.io/opensource-observer/${image_name}"
+    sha_image="${image_repo}:${tag}"
+    latest_image="${image_repo}:latest"
+
+    echo "Building ${image_name} plugin"
+    docker build \
+        -t ${sha_image} \
+        -t ${latest_image} \
+        --label "org.opencontainers.image.source=https://github.com/opensource-observer/oso" \
+        --label "observer.opensource.oso.sha=${tag}" \
+        --build-arg IMAGE_NAME=${image_name}
+        -f docker/images/${IMAGE_NAME} \
         .
-    echo "Publishing the plugin to ${plugin_image}"
-    docker push ${plugin_image}
+    echo "Publishing the image to ${sha_image}"
+    docker push "${sha_image}"
+    echo "Publishing latest to ${latest_image}"
+    docker push "${latest_image}"
 done
-
-for path in $python_plugins; do
-    plugin_name=$(basename $path)
-    # Remove the cloudquery prefix
-    plugin_name=${plugin_name#"$prefix"}
-
-    plugin_cmd=$(echo $plugin_name | sed "s/-/_/g")
-    plugin_image="ghcr.io/opensource-observer/cloudquery-${plugin_name}:${tag}"
-
-    # Skip the example
-    if [[ $plugin_name = "example_plugin" ]]; then
-        continue
-    fi
-    echo "Building ${plugin_name} plugin"
-
-    docker build -t ${plugin_image} \
-        --build-arg PLUGIN_NAME=${plugin_name} \
-        --build-arg PLUGIN_CMD=${plugin_cmd} \
-        --build-arg BASE_IMAGE=${ts_base_image} \
-        -f docker/cloudquery/py.Dockerfile \
-        .
-
-    echo "Publishing the plugin to ${plugin_image}"
-    docker push ${plugin_image}
-done

ops/clusters/warehouse/flux-system/gotk-sync.yaml

@@ -6,7 +6,7 @@ metadata:
   name: flux-system
   namespace: flux-system
 spec:
-  interval: 1m0s
+  interval: 1m
   ref:
     branch: main
   secretRef:

ops/clusters/warehouse/production-apps.yaml

@@ -14,5 +14,6 @@ spec:
     kind: GitRepository
     name: flux-system
   path: ./ops/k8s-apps/production
+  force: true
   prune: true
   wait: true

ops/clusters/warehouse/staging-apps.yaml

@@ -14,5 +14,6 @@ spec:
     kind: GitRepository
     name: flux-system
   path: ./ops/k8s-apps/staging
+  force: true
   prune: true
   wait: true

ops/helm-charts/oso-dagster/Chart.lock

@@ -1,6 +1,6 @@
 dependencies:
 - name: dagster
   repository: https://dagster-io.github.io/helm
-  version: 1.2.3
-digest: sha256:0848a02ca7fb9b0566de902f707cfdd65ea5073be847f3de49253d56bfd59a8c
-generated: "2024-05-16T22:56:26.894848565Z"
+  version: 1.7.6
+digest: sha256:3f50e0ae14880ffe7c03295ce0fc605fb4e4c9c32fdf11b24606285807c5de99
+generated: "2024-05-17T17:18:41.876405486Z"

ops/helm-charts/oso-dagster/Chart.yaml

@@ -3,9 +3,9 @@ name: oso-dagster
 description: Extension of the dagster template
 
 type: application
-version: 0.1.2
-appVersion: "1.16.0"
+version: 0.1.10
+appVersion: "1.0.0"
 dependencies:
 - name: dagster
-  version: "1.2.3"
+  version: "1.7.6"
   repository: "https://dagster-io.github.io/helm"

ops/helm-charts/oso-dagster/templates/_helpers.tpl

@@ -2,8 +2,10 @@
 Expand the name of the chart.
 */}}
 
+# Disable the pgisready check due to our use of cloudsql proxy injected into the
+pod.
 {{- define "dagster.postgresql.pgisready" -}}
-until pg_isready -h ${DAGSTER_PG_HOST} -p ${DAGSTER_PG_PORT} -U ${DAGSTER_PG_USER}; do echo waiting for database; sleep 2; done;
+sleep 5;
 {{- end }}
 
 {{- define "dagsterYaml.postgresql.config" }}
@@ -22,4 +24,12 @@ postgres_db:
   {{- if .Values.postgresql.postgresqlScheme }}
   scheme: {{ .Values.postgresql.postgresqlScheme }}
   {{- end }}
-{{- end }}
+{{- end }}
+
+# Fix issues with the full name
+{{- define "dagster.webserver.fullname" -}}
+{{- $name := default "webserver" .Values.dagsterWebserver.nameOverride -}}
+{{- $fullname := include "dagster.fullname" . -}}
+{{- printf "%s-%s" $fullname  $name | trunc 63 | trimSuffix "-" -}}
+{{- if .webserverReadOnly -}} -read-only {{- end -}}
+{{- end -}}

ops/helm-charts/oso-dagster/templates/config-map.yaml

@@ -1,7 +1,9 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: {{ .Release.Name }}-oso-extra-env
+  name: dagster-oso-extra-env
 data:
-  DAGSTER_PG_USER: {{ .Values.configMap.secretPrefix }}-{{ .Values.secretmanagerKeys.dbUser }}
-  DAGSTER_PG_DB_NAME: {{ .Values.configMap.secretPrefix }}-{{ .Values.secretmanagerKeys.dbName }}
+  DAGSTER_PG_USER: "{{ .Values.configMap.secretPrefix }}-{{ .Values.secretmanagerKeys.dbUser }}"
+  DAGSTER_PG_DB_NAME: "{{ .Values.configMap.secretPrefix }}-{{ .Values.secretmanagerKeys.dbName }}"
+  DAGSTER_PG_PORT: "{{ .Values.pg.port }}"
+  DAGSTER_PG_HOST: "{{ .Values.pg.host }}"

ops/helm-charts/oso-dagster/values.yaml

@@ -10,24 +10,14 @@ dagster:
     postgresqlPassword: "gcp:secretmanager:dagster-db-password/versions/1"
   enableReadOnly: true
 
-  dagsterWebserver:
-    env:
-      - name: DAGSTER_PG_HOST
-        value: set-by-proxy-operator
-      - name: DAGSTER_PG_PORT
-        value: set-by-proxy-operator
-
-  dagsterDaemon:
-    env:
-      - name: DAGSTER_PG_HOST
-        value: set-by-proxy-operator
-      - name: DAGSTER_PG_PORT
-        value: set-by-proxy-operator
 configMap:
   name: "dagster-extra-env-config-map"
   secretPrefix: "gcp:secretmanager:dagster"
 secretmanagerKeys: 
   dbUser: "db-user/versions/1"
   dbHost: "db-host/versions/1"
   dbPort: "db-port/versions/1"
-  dbName: "db-name/versions/1"
+  dbName: "db-name/versions/1"
+pg:
+  port: "5432"
+  host: "127.0.0.1"

ops/k8s-apps/base/cloudsql-proxy/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: base-cloudsql-proxy
+resources:
+  - proxy.yaml

ops/k8s-apps/base/cloudsql-proxy/proxy.yaml

@@ -0,0 +1,42 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: base-cloudsql-proxy
+  labels:
+    toolkit.fluxcd.io/tenant: apps
+    ops.opensource.observer/environment: base
+    kube-secrets-init.doit-intl.com/enable-mutation: "true"
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: rimusz
+  namespace: base-cloudsql-proxy
+spec:
+  interval: 5m
+  url: https://charts.rimusz.net
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: base-cloudsql-proxy
+  namespace: base-cloudsql-proxy
+spec:
+  chart:
+    spec:
+      chart: gcloud-sqlproxy
+      version: "0.25.6"
+      sourceRef:
+        kind: HelmRepository
+        name: rimusz
+  interval: 50m
+  install:
+    remediation:
+      retries: 3
+  values:
+    cloudsql:
+      instances:
+        - instance: oso-playground-psql
+          region: us-central1
+          project: opensource-observer
+          port: 5432