Merge branch 'develop'

sillsdev · Nov 13, 2023 · 5e46a3f · 5e46a3f
2 parents 22c7c09 + baade5f
commit 5e46a3f
Show file tree

Hide file tree

Showing 38 changed files with 431 additions and 206 deletions.
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
@@ -69,4 +69,6 @@ jobs:
           cache: "npm"
       -
         name: Run prettier check
-        run: npx prettier --check .
+        run: |
+          npx prettier -v
+          npx prettier --check .
diff --git a/docker/deployment/Makefile b/docker/deployment/Makefile
@@ -35,40 +35,20 @@ shell-next-app:
 shell-next-proxy:
 	kubectl exec -it deploy/next-proxy -- sh
 
+forward-db:
+	kubectl port-forward pod/db-6b494cd89f-whfr7 27018:27017 -n languageforge
+
 init-secrets:
 	kubectl apply -f secrets.yaml
 
 create-new-deployment-mail:
 	kubectl create deployment mail --image=juanluisbaptiste/postfix:1.0.0 --dry-run=client -o yaml > mail-deployment-new.yaml
 
-deploy-staging: deploy-db deploy-mail-staging deploy-app-staging deploy-lfmerge-staging deploy-next-proxy-staging deploy-next-app-staging
-deploy-mail-staging:
-	sed -e s/{{SERVER_HOSTNAME}}/staging.languageforge.org/ mail-deployment.yaml | kubectl apply -f -
-deploy-app-staging:
-	sed -e s/{{WEBSITE}}/staging.languageforge.org/ app-deployment.yaml \
-  | sed -e s/{{VERSION}}/$(VERSION_APP)/ | kubectl apply -f -
-deploy-lfmerge-staging:
-	sed -e s/{{VERSION_LFMERGE}}/$(VERSION_LFMERGE)/ lfmerge-deployment.yaml | kubectl apply -f -
-deploy-next-proxy-staging:
-	sed -e s/{{WEBSITE}}/staging.languageforge.org/ next-proxy-deployment.yaml \
-  | sed -e s/{{VERSION}}/$(VERSION_PROXY)/ | kubectl apply -f -
-deploy-next-app-staging:
-	sed -e s/{{VERSION}}/$(VERSION_NEXT_APP)/ next-app-deployment.yaml | kubectl apply -f -
-deploy-prod: deploy-db deploy-mail-prod deploy-app-prod deploy-lfmerge-prod deploy-next-proxy-prod deploy-next-app-prod
-deploy-mail-prod:
-	sed -e s/{{SERVER_HOSTNAME}}/languageforge.org/ mail-deployment.yaml | kubectl apply -f -
-deploy-app-prod:
-	sed -e s/{{WEBSITE}}/languageforge.org/ app-deployment.yaml \
-  | sed -e s/{{VERSION}}/$(VERSION_APP)/ | kubectl apply -f -
-deploy-lfmerge-prod:
-	sed -e s/{{VERSION_LFMERGE}}/$(VERSION_LFMERGE)/ lfmerge-deployment.yaml | kubectl apply -f -
-deploy-next-proxy-prod:
-	sed -e s/{{WEBSITE}}/languageforge.org/ next-proxy-deployment.yaml \
-  | sed -e s/{{VERSION}}/$(VERSION_PROXY)/ | kubectl apply -f -
-deploy-next-app-prod:
-	sed -e s/{{VERSION}}/$(VERSION_NEXT_APP)/ next-app-deployment.yaml | kubectl apply -f -
-deploy-db:
-	kubectl apply -f db-deployment.yaml
+deploy-staging:
+	kubectl --context dallas-rke apply -k staging/
+deploy-prod:
+	kubectl --context aws-rke apply -k prod/
+
 
 delete: delete-app delete-lfmerge delete-mail delete-db delete-next-proxy delete-next-app
 delete-db: # does NOT delete the volume, i.e., the data in the database

diff --git a/docker/deployment/base/app-config.yaml b/docker/deployment/base/app-config.yaml
@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: app-config
+data:
+  website: "localhost"
+  hg-domain: "hg-staging.languageforge.org"
+  hg-protocol: "https"
+  hg-username: "lf-merge"
diff --git a/docker/deployment/app-deployment.yaml → docker/deployment/base/app-deployment.yaml b/docker/deployment/app-deployment.yaml → docker/deployment/base/app-deployment.yaml
@@ -18,35 +18,6 @@ spec:
 
 ---
 
-# https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: lf-project-assets
-spec:
-  accessModes:
-    - ReadWriteMany
-  resources:
-    requests:
-      storage: 10Gi
-  storageClassName: weekly-snapshots-retain-4 # provided by LTOps
-
----
-
-# https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: lfmerge-sendreceive-data
-spec:
-  accessModes:
-    - ReadWriteMany
-  resources:
-    requests:
-      storage: 50Gi
-  storageClassName: weekly-snapshots-retain-4 # provided by LTOps
-
----
 
 # https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#writing-a-deployment-spec
 apiVersion: apps/v1
@@ -93,7 +64,7 @@ spec:
             name: sendreceive-data
       containers:
       - name: app
-        image: sillsdev/web-languageforge:{{VERSION}}
+        image: sillsdev/web-languageforge:latest
         imagePullPolicy: Always
         # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers
         resources:
@@ -113,7 +84,10 @@ spec:
           - name: ENVIRONMENT
             value: production
           - name: WEBSITE
-            value: {{WEBSITE}}
+            valueFrom:
+              configMapKeyRef:
+                name: app-config
+                key: website
           - name: MAIL_HOST
             value: mail
           - name: LFMERGE_LOGGING_DEST
@@ -143,6 +117,7 @@ spec:
               secretKeyRef:
                 key: LEX_BOX_HOST
                 name: ld-api
+                optional: true
           - name: FACEBOOK_CLIENT_ID
             valueFrom:
               secretKeyRef:

diff --git a/docker/deployment/db-deployment.yaml → docker/deployment/base/db-deployment.yaml b/docker/deployment/db-deployment.yaml → docker/deployment/base/db-deployment.yaml
@@ -51,6 +51,18 @@ spec:
       - name: db
         image: mongo:6
         # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers
+        livenessProbe:
+          exec:
+            command:
+              - mongosh
+              - '--quiet'
+              - '--eval'
+              - db.runCommand('ping')
+          failureThreshold: 3
+          initialDelaySeconds: 30
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 5
         resources:
           requests:
             memory: 260Mi

diff --git a/docker/deployment/base/ingress-config.yaml b/docker/deployment/base/ingress-config.yaml
@@ -0,0 +1,9 @@
+# template, copy into env folder and change the values, then add as a patch to the kustomization.yaml file
+
+- op: replace
+  path: /spec/rules/0/host
+  value: localhost
+- op: replace
+  path: /spec/tls/0/hosts
+  value:
+    - localhost
diff --git a/docker/deployment/base/kustomization.yaml b/docker/deployment/base/kustomization.yaml
@@ -0,0 +1,13 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: languageforge
+
+resources:
+  - app-deployment.yaml
+  - db-deployment.yaml
+  - lfmerge-deployment.yaml
+  - lfmerge-pvcs.yaml
+  - mail-deployment.yaml
+  - next-app-deployment.yaml
+  - next-proxy-deployment.yaml
+  - app-config.yaml
diff --git a/docker/deployment/lfmerge-deployment.yaml → ...r/deployment/base/lfmerge-deployment.yaml b/docker/deployment/lfmerge-deployment.yaml → ...r/deployment/base/lfmerge-deployment.yaml
@@ -18,36 +18,6 @@ spec:
 
 ---
 
-# https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: lf-project-assets
-spec:
-  accessModes:
-    - ReadWriteMany
-  resources:
-    requests:
-      storage: 10Gi
-  storageClassName: weekly-snapshots-retain-4 # provided by LTOps
-
----
-
-# https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: lfmerge-sendreceive-data
-spec:
-  accessModes:
-    - ReadWriteMany
-  resources:
-    requests:
-      storage: 50Gi
-  storageClassName: weekly-snapshots-retain-4 # provided by LTOps
-
----
-
 # https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#writing-a-deployment-spec
 apiVersion: apps/v1
 kind: Deployment
@@ -100,7 +70,7 @@ spec:
             name: sendreceive-data
       containers:
       - name: lfmerge
-        image: ghcr.io/sillsdev/lfmerge:{{VERSION_LFMERGE}}
+        image: ghcr.io/sillsdev/lfmerge:latest
         imagePullPolicy: Always
         # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers
         resources:
@@ -133,8 +103,24 @@ spec:
             value: sf_
           - name: LFMERGE_VERBOSE_PROGRESS
             value: "true"
+          - name: LFMERGE_LANGUAGE_DEPOT_HG_PUBLIC_HOSTNAME
+            valueFrom:
+              configMapKeyRef:
+                name: app-config
+                key: hg-domain
+          - name: LFMERGE_LANGUAGE_DEPOT_HG_PROTOCOL
+            valueFrom:
+              configMapKeyRef:
+                name: app-config
+                key: hg-protocol
           - name: LANGUAGE_DEPOT_TRUST_TOKEN
             valueFrom:
               secretKeyRef:
                 key: LD_TRUST_TOKEN
                 name: ld-trust-token
+          - name: LANGUAGE_DEPOT_HG_USERNAME
+            valueFrom:
+              configMapKeyRef:
+                name: app-config
+                key: hg-username
+
diff --git a/docker/deployment/base/lfmerge-pvcs.yaml b/docker/deployment/base/lfmerge-pvcs.yaml
@@ -0,0 +1,28 @@
+
+# https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: lf-project-assets
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 10Gi
+  storageClassName: weekly-snapshots-retain-4 # provided by LTOps
+
+---
+
+# https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: lfmerge-sendreceive-data
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 50Gi
+  storageClassName: weekly-snapshots-retain-4 # provided by LTOps
diff --git a/docker/deployment/mail-deployment.yaml → docker/deployment/base/mail-deployment.yaml b/docker/deployment/mail-deployment.yaml → docker/deployment/base/mail-deployment.yaml
@@ -41,7 +41,10 @@ spec:
             memory: 100Mi
         env:
           - name: SERVER_HOSTNAME
-            value: {{SERVER_HOSTNAME}}
+            valueFrom:
+              configMapKeyRef:
+                name: app-config
+                key: website
           - name: SMTP_SERVER
             valueFrom:
               secretKeyRef:

diff --git a/docker/deployment/next-app-deployment.yaml → .../deployment/base/next-app-deployment.yaml b/docker/deployment/next-app-deployment.yaml → .../deployment/base/next-app-deployment.yaml
@@ -37,7 +37,9 @@ spec:
     spec:
       containers:
       - name: next-app
-        image: sillsdev/web-languageforge:{{VERSION}}
+        # this image doesn't actually exist as for some reason tags were used for different image types instead of just version number.
+        # this is a workaround because kustomize is designed to work with image names
+        image: sillsdev/web-languageforge-next-app:latest
         imagePullPolicy: Always
         # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers
         resources:

diff --git a/docker/deployment/next-proxy-deployment.yaml → ...eployment/base/next-proxy-deployment.yaml b/docker/deployment/next-proxy-deployment.yaml → ...eployment/base/next-proxy-deployment.yaml
@@ -9,7 +9,7 @@ metadata:
     nginx.ingress.kubernetes.io/proxy-body-size: 60M
 spec:
   rules:
-  - host: {{WEBSITE}}
+  - host: localhost
     http:
       paths:
       - path: /
@@ -21,7 +21,7 @@ spec:
               number: 80
   tls:
   - hosts:
-    - {{WEBSITE}}
+    - localhost
     secretName: languageforge-tls
 
 ---
@@ -64,7 +64,9 @@ spec:
     spec:
       containers:
       - name: next-proxy
-        image: sillsdev/web-languageforge:{{VERSION}}
+        # this image doesn't actually exist as for some reason tags were used for different image types instead of just version number.
+        # this is a workaround because kustomize is designed to work with image names
+        image: sillsdev/web-languageforge-next-proxy:latest
         # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers
         imagePullPolicy: Always
         resources:

diff --git a/docker/deployment/secrets.yaml → docker/deployment/base/secrets.yaml b/docker/deployment/secrets.yaml → docker/deployment/base/secrets.yaml
diff --git a/docker/deployment/prod/app-config-patch.yaml b/docker/deployment/prod/app-config-patch.yaml
@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: app-config
+data:
+  website: "languageforge.org"
+  hg-domain: "hg-public.languagforge.org"
+
diff --git a/docker/deployment/prod/ingress-config-patch.yaml b/docker/deployment/prod/ingress-config-patch.yaml
@@ -0,0 +1,9 @@
+# template, copy into env folder and change the values, then add as a patch to the kustomization.yaml file
+
+- op: replace
+  path: /spec/rules/0/host
+  value: languageforge.org
+- op: replace
+  path: /spec/tls/0/hosts
+  value:
+    - languageforge.org
diff --git a/docker/deployment/prod/kustomization.yaml b/docker/deployment/prod/kustomization.yaml
@@ -0,0 +1,30 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: languageforge
+
+resources:
+  - ../base
+
+# both next app and next proxy require their own images, but the image name web-languageforge was reused for them
+# with a different tag to select the correct image instead of making a new image name
+# because of that we have to do a bit of a workaround to allow setting the image tag for the correct container
+images:
+  - name: sillsdev/web-languageforge
+    newTag: 2023-09-21
+  - name: sillsdev/web-languageforge-next-proxy
+    newName: sillsdev/web-languageforge
+    newTag: next-proxy-2023-09-21
+  - name: sillsdev/web-languageforge-next-app
+    newName: sillsdev/web-languageforge
+    newTag: next-app-2023-09-21
+  - name: ghcr.io/sillsdev/lfmerge
+    newTag: 2.0.135
+
+patches:
+  - path: app-config-patch.yaml
+  - path: lfmerge-pvcs-patch.yaml
+  - path: ingress-config-patch.yaml
+    target:
+      kind: Ingress
+      name: languageforge-app
+      namespace: languageforge
diff --git a/docker/deployment/prod/lfmerge-pvcs-patch.yaml b/docker/deployment/prod/lfmerge-pvcs-patch.yaml
@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: lfmerge-sendreceive-data
+spec:
+  resources:
+    requests:
+      storage: 70Gi
diff --git a/docker/deployment/staging/app-config-patch.yaml b/docker/deployment/staging/app-config-patch.yaml
@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: app-config
+data:
+  website: "staging.languageforge.org"
+  hg-domain: "lexbox.languagedepot:5158" #reference lexbox service in the same cluster
+  hg-protocol: "http"