feat: add consumers to consume rabbitmq queue for users and locations…

… queues (#150)

* feat(cluster): install secrets for webhooked

* feat(cluster): install webhooked 🎉

* feat(cluster): configure the webhook routing and enable intra settings

* feat(cluster): add prometheus scrape on webhooked & set a rollout strategy

* fix(cluster): disable istio sidecar for cronjob

* feat: implement a basic consumer

* feat: implement webhook rabbitmq consumer for all campus

* feat: add webhhoks-processor on production

* fix: check the qos error of rmq channel

* fix: log of webhooks have wrong type

.env.example

@@ -2,6 +2,7 @@ GO_ENV=development
 APP_VERSION=indev
 CORS_ORIGIN=http://localhost:3000
 DATABASE_URL=postgresql://postgres:postgres@localhost:5432/stud42_dev
+AMQP_URL=amqp://rabbitmq:s42@localhost:5672/
 S42_SERVICE_TOKEN=private-cross-service-token
 
 GITHUB_TOKEN=

.github/workflows/deployer.yaml

@@ -171,4 +171,8 @@ jobs:
 
         cd ${{github.workspace }}/deploy/app/crawler-locations/overlays/live
         kustomize edit set image app=ghcr.io/42atomys/stud42:${{ github.event.release.tag_name }}
+        kustomize build . | kubectl apply -f -
+
+        cd ${{github.workspace }}/deploy/app/webhooks-processor/overlays/live
+        kustomize edit set image app=ghcr.io/42atomys/stud42:${{ github.event.release.tag_name }}
         kustomize build . | kubectl apply -f -

cmd/locations.go

@@ -34,7 +34,6 @@ import (
 	modelgen "atomys.codes/stud42/internal/models/generated"
 	"atomys.codes/stud42/internal/models/generated/campus"
 	"atomys.codes/stud42/internal/models/generated/location"
-	"atomys.codes/stud42/internal/models/generated/user"
 	"atomys.codes/stud42/pkg/duoapi"
 )
 
@@ -71,42 +70,17 @@ For any closed locations, the location will be marked as inactive in the databas
 		client := modelsutils.Client()
 		bulk := []*modelgen.LocationCreate{}
 		for _, l := range locations {
-			u, err := client.User.Query().Where(user.DuoID(l.User.ID)).Only(cmd.Context())
+			u, err := modelsutils.UserFirstOrCreateFromComplexLocation(cmd.Context(), l)
 			if err != nil {
-				if modelgen.IsNotFound(err) {
-					err := client.User.Create().
-						SetEmail(l.User.Email).
-						SetDuoID(l.User.ID).
-						SetDuoLogin(l.User.Login).
-						SetFirstName(l.User.FirstName).
-						SetLastName(l.User.LastName).
-						SetUsualFirstName(l.User.UsualFirstName).
-						SetPhone(l.User.Phone).
-						SetPoolMonth(l.User.PoolMonth).
-						SetPoolYear(l.User.PoolYear).
-						SetIsStaff(l.User.Staff).
-						SetIsAUser(false).
-						OnConflictColumns(user.FieldDuoID).
-						DoNothing().
-						Exec(cmd.Context())
-					if err != nil {
-						log.Fatal().Err(err).Msg("Failed to create user")
-					}
-					u, err = client.User.Query().Where(user.DuoID(l.User.ID)).Only(cmd.Context())
-					if err != nil {
-						log.Fatal().Err(err).Msg("Failed to get user 1")
-					}
-				} else {
-					log.Fatal().Err(err).Msg("Failed to get user 2")
-				}
+				log.Fatal().Err(err).Msg("Failed to create user")
 			}
 
 			bulk = append(bulk, client.Location.Create().
 				SetCampus(campus).
 				SetUser(u).
 				SetDuoID(l.ID).
-				SetBeginAt(l.BeginAt).
-				SetNillableEndAt(l.EndAt).
+				SetBeginAt(l.BeginAt.Time()).
+				SetNillableEndAt(l.EndAt.NillableTime()).
 				SetIdentifier(l.Host).
 				SetUserDuoID(l.User.ID).
 				SetUserDuoLogin(l.User.Login))

cmd/webhooks.go

@@ -0,0 +1,67 @@
+/*
+Copyright © 2022 42Atomys
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+package cmd
+
+import (
+	"os"
+
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/cobra"
+
+	"atomys.codes/stud42/internal/webhooks"
+)
+
+// webhooksCmd represents the webhooks command
+var webhooksCmd = &cobra.Command{
+	Use:   "webhooks",
+	Short: "A brief description of your command",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		var amqpURL = os.Getenv("AMQP_URL")
+		if amqpURL == "" {
+			log.Fatal().Msg("AMQP_URL not set")
+		}
+
+		if err := webhooks.New().Serve(amqpURL, "webhooks-deliveries"); err != nil {
+			log.Fatal().Err(err).Msg("failed to start rabbitmq consumer")
+		}
+	},
+}
+
+func init() {
+	jobsCmd.AddCommand(webhooksCmd)
+
+	// Here you will define your flags and configuration settings.
+
+	// Cobra supports Persistent Flags which will work for this command
+	// and all subcommands, e.g.:
+	// webhooksCmd.PersistentFlags().String("foo", "", "A help for foo")
+
+	// Cobra supports local flags which will only run when this command
+	// is called directly, e.g.:
+	// webhooksCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
+}

deploy/app/crawler-locations/base/cronjob.yaml

@@ -4,7 +4,7 @@ metadata:
   name: crawler-locations
 spec:
   # Each minute
-  schedule: "* * * * *"
+  schedule: "* */12 * * *"
   successfulJobsHistoryLimit: 2
   failedJobsHistoryLimit: 5
   concurrencyPolicy: Forbid

deploy/app/webhooked/base/kustomization.yaml

@@ -5,7 +5,10 @@ resources:
 - virtual-service.yaml
 
 commonLabels:
+  app: webhooked
+  version: '0.6'
   kubernetes.io/name: webhooked
+  app.kubernetes.io/version: '0.6'
   app.kubernetes.io/component: micro-service
   app.kubernetes.io/part-of: s42-app
   app.kubernetes.io/managed-by: kustomize

deploy/app/webhooks-processor/base/deployment.yaml

@@ -0,0 +1,73 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: webhooks-processor
+spec:
+  selector: {}
+  revisionHistoryLimit: 1
+  template:
+    spec:
+      imagePullSecrets:
+      - name: ghcr-creds
+      containers:
+      - name: service
+        image: app
+        args: [ '--config', '/config/stud42.yaml', jobs, webhooks ]
+        command: [ stud42cli ]
+        env:
+        - name: GO_ENV
+          value: production
+        - name: APP_VERSION
+          value: latest
+        - name: SENTRY_DSN
+          valueFrom:
+            secretKeyRef:
+              key: 'WEBHOOKS_PROCESSOR_DSN'
+              name: 'sentry-dsns'
+        - name: DATABASE_URL
+          value: postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST):5432/$(DATABASE_NAME)?sslmode=disable
+        - name: DATABASE_HOST
+          value: primary-postgres
+        - name: DATABASE_USERNAME
+          value: postgres
+        - name: DATABASE_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              key: POSTGRES_PASSWORD
+              name: primary-postgres-credentials
+        - name: DATABASE_NAME
+          value: s42
+        - name: AMQP_URL
+          value: amqp://$(AMQP_USERNAME):$(AMQP_PASSWORD)@$(AMQP_HOST):$(AMQP_PORT)/
+        - name: AMQP_USERNAME
+          valueFrom:
+            secretKeyRef:
+              key: username
+              name: prod-primary-rabbitmq-default-user
+        - name: AMQP_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              key: password
+              name: prod-primary-rabbitmq-default-user
+        - name: AMQP_HOST
+          valueFrom:
+            secretKeyRef:
+              key: host
+              name: prod-primary-rabbitmq-default-user
+        - name: AMQP_PORT
+          valueFrom:
+            secretKeyRef:
+              key: port
+              name: prod-primary-rabbitmq-default-user
+        volumeMounts:
+          - name: config
+            mountPath: /config
+            readOnly: true
+        resources:
+          limits:
+            memory: "42Mi"
+            cpu: "200m"
+      volumes:
+      - name: config
+        configMap:
+          name: stud42-config

deploy/app/webhooks-processor/base/kustomization.yaml

@@ -0,0 +1,10 @@
+resources:
+- deployment.yaml
+
+commonLabels:
+  kubernetes.io/name: webhooks-processor
+  app.kubernetes.io/version: '0.1'
+  app.kubernetes.io/component: micro-service
+  app.kubernetes.io/part-of: s42-app
+  app.kubernetes.io/managed-by: kustomize
+  app.kubernetes.io/created-by: github-actions

deploy/app/webhooks-processor/overlays/live/kustomization.yaml

@@ -0,0 +1,17 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namePrefix: prod-
+resources:
+- ../../base
+namespace: production
+
+commonAnnotations:
+  sidecar.istio.io/inject: "false"
+
+images:
+- name: app
+  newName: ghcr.io/42atomys/stud42
+  newTag: latest
+
+patchesStrategicMerge:
+- ./use-live-primary-postgres.yaml

deploy/app/webhooks-processor/overlays/live/use-live-primary-postgres.yaml

@@ -0,0 +1,14 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: webhooks-processor
+spec:
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: service
+            env:
+            - name: DATABASE_HOST
+              value: prod-primary-postgres

go.mod

@@ -19,6 +19,7 @@ require (
 	github.com/shurcooL/githubv4 v0.0.0-20220115235240-a14260e6f8a2
 	github.com/spf13/cobra v1.4.0
 	github.com/spf13/viper v1.10.1
+	github.com/streadway/amqp v1.0.0
 	github.com/stretchr/testify v1.7.1
 	github.com/vektah/gqlparser/v2 v2.4.1
 	github.com/vmihailenco/msgpack/v5 v5.0.0-beta.9

go.sum

@@ -403,6 +403,8 @@ github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5q
 github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk=
 github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU=
 github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
+github.com/streadway/amqp v1.0.0 h1:kuuDrUJFZL1QYL9hUNuCxNObNzB0bV/ZG5jV3RWAQgo=
+github.com/streadway/amqp v1.0.0/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=

internal/api/api.resolvers.go

@@ -7,19 +7,20 @@ import (
 	"context"
 	"os"
 
+	"github.com/bwmarrin/discordgo"
+	"github.com/google/uuid"
+	"github.com/rs/zerolog/log"
+	"github.com/shurcooL/githubv4"
+	"github.com/spf13/viper"
+	"golang.org/x/oauth2"
+
 	apigen "atomys.codes/stud42/internal/api/generated"
 	typesgen "atomys.codes/stud42/internal/api/generated/types"
 	"atomys.codes/stud42/internal/models/generated"
 	"atomys.codes/stud42/internal/models/generated/account"
 	"atomys.codes/stud42/internal/models/generated/campus"
 	"atomys.codes/stud42/internal/models/generated/location"
 	"atomys.codes/stud42/internal/models/generated/user"
-	"github.com/bwmarrin/discordgo"
-	"github.com/google/uuid"
-	"github.com/rs/zerolog/log"
-	"github.com/shurcooL/githubv4"
-	"github.com/spf13/viper"
-	"golang.org/x/oauth2"
 )
 
 func (r *mutationResolver) CreateFriendship(ctx context.Context, userID uuid.UUID) (bool, error) {