mongo: Hedged Reads (#237)

* mongo: Hedged Reads * add config node * remove rmq & s3 from health check
SevenTV · Dec 6, 2023 · 5676fd5 · 5676fd5
1 parent e3c4189
commit 5676fd5
Show file tree

Hide file tree

Showing 9 changed files with 29 additions and 114 deletions.
diff --git a/cmd/main.go b/cmd/main.go
@@ -111,11 +111,12 @@ func main() {
 
 	{
 		gctx.Inst().Mongo, err = mongo.Setup(gctx, mongo.SetupOptions{
-			URI:      config.Mongo.URI,
-			DB:       config.Mongo.DB,
-			Direct:   config.Mongo.Direct,
-			Username: config.Mongo.Username,
-			Password: config.Mongo.Password,
+			URI:         config.Mongo.URI,
+			DB:          config.Mongo.DB,
+			Direct:      config.Mongo.Direct,
+			Username:    config.Mongo.Username,
+			Password:    config.Mongo.Password,
+			HedgedReads: config.Mongo.HedgedReads,
 		})
 		if err != nil {
 			zap.S().Fatalw("failed to setup mongo handler",
@@ -153,7 +154,7 @@ func main() {
 			})
 		}
 		if err != nil {
-			zap.S().Fatalw("failed to setup mq handler",
+			zap.S().Errorw("failed to setup mq handler",
 				"error", err,
 			)
 		}
@@ -192,14 +193,14 @@ func main() {
 
 	nc, err := nats.Connect(config.Nats.Url)
 	if err != nil {
-		zap.S().Fatalw("failed to connect to nats",
+		zap.S().Errorw("failed to connect to nats",
 			"error", err,
 		)
 	}
 
 	defer func() {
 		err = nc.Drain()
-		zap.S().Fatalw("failed to drain nats, is connection failing?",
+		zap.S().Errorw("failed to drain nats, is connection failing?",
 			"error", err,
 		)
 	}()

diff --git a/go.mod b/go.mod
@@ -17,7 +17,7 @@ require (
 	github.com/nats-io/nats.go v1.28.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/prometheus/client_golang v1.14.0
-	github.com/seventv/common v0.0.0-20231202052307-1d4dd4479600
+	github.com/seventv/common v0.0.0-20231206031419-1ba3f2c093f6
 	github.com/seventv/compactdisc v0.0.0-20221006190906-ccfe99954e48
 	github.com/seventv/image-processor/go v0.0.0-20221128171540-d050701ac324
 	github.com/seventv/message-queue/go v0.0.0-20231201171845-1bb9d5db6881

diff --git a/go.sum b/go.sum
@@ -354,6 +354,8 @@ github.com/seventv/common v0.0.0-20231109022220-2f3ccd557f7d h1:zMhDCupeb6lCR7nV
 github.com/seventv/common v0.0.0-20231109022220-2f3ccd557f7d/go.mod h1:jHHFe3uNMyzb/ReDqMvaI/A1euvV/PW/G+2PhcWQqWw=
 github.com/seventv/common v0.0.0-20231202052307-1d4dd4479600 h1:qwu6BlOxmKPBe9S8YXFVPXFUnaAVkN77k7Bcqn+fbNc=
 github.com/seventv/common v0.0.0-20231202052307-1d4dd4479600/go.mod h1:jHHFe3uNMyzb/ReDqMvaI/A1euvV/PW/G+2PhcWQqWw=
+github.com/seventv/common v0.0.0-20231206031419-1ba3f2c093f6 h1:A1aHPZxG5XU+rqpuWFm5oeUaV/CwcjVfFmdzdw7Kpzw=
+github.com/seventv/common v0.0.0-20231206031419-1ba3f2c093f6/go.mod h1:jHHFe3uNMyzb/ReDqMvaI/A1euvV/PW/G+2PhcWQqWw=
 github.com/seventv/compactdisc v0.0.0-20221006190906-ccfe99954e48 h1:IqWrtt/yob45YnOQ5Wwkbf8qP22eKVtg0WzfyEkGnFg=
 github.com/seventv/compactdisc v0.0.0-20221006190906-ccfe99954e48/go.mod h1:T+ldp0YQe03s44+A5HHHI/jB3ZmWqOIaNouEqAS+1Dk=
 github.com/seventv/image-processor/go v0.0.0-20221128171540-d050701ac324 h1:iU3wWepRTbkNoTAPR23m6TAW6Yb9pOMCYVr0K++OBAw=

diff --git a/internal/configure/config.go b/internal/configure/config.go
@@ -146,11 +146,12 @@ type Config struct {
 	} `mapstructure:"nats" json:"nats"`
 
 	Mongo struct {
-		URI      string `mapstructure:"uri" json:"uri"`
-		Username string `mapstructure:"username" json:"username"`
-		Password string `mapstructure:"password" json:"password"`
-		DB       string `mapstructure:"db" json:"db"`
-		Direct   bool   `mapstructure:"direct" json:"direct"`
+		URI         string `mapstructure:"uri" json:"uri"`
+		Username    string `mapstructure:"username" json:"username"`
+		Password    string `mapstructure:"password" json:"password"`
+		DB          string `mapstructure:"db" json:"db"`
+		Direct      bool   `mapstructure:"direct" json:"direct"`
+		HedgedReads bool   `mapstructure:"hedged_reads" json:"hedged_reads"`
 	} `mapstructure:"mongo" json:"mongo"`
 
 	Meilisearch struct {

diff --git a/internal/svc/health/health.go b/internal/svc/health/health.go
@@ -40,24 +40,6 @@ func New(gctx global.Context) <-chan struct{} {
 				cancel()
 			}
 
-			lCtx, cancel := context.WithTimeout(context.Background(), time.Second*5)
-			mqDown = gctx.Inst().MessageQueue != nil && !gctx.Inst().MessageQueue.Connected(lCtx)
-			cancel()
-			if mqDown {
-				zap.S().Warnw("mq is not connected")
-			}
-
-			if gctx.Config().S3.Enabled && gctx.Inst().S3 != nil {
-				lCtx, cancel := context.WithTimeout(context.Background(), time.Second*5)
-				if _, err := gctx.Inst().S3.ListBuckets(lCtx); err != nil {
-					s3Down = true
-					zap.S().Warnw("s3 is not responding",
-						"error", err,
-					)
-				}
-				cancel()
-			}
-
 			if gctx.Inst().Mongo != nil {
 				lCtx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 				if err := gctx.Inst().Mongo.Ping(lCtx); err != nil {

diff --git a/internal/svc/health/health_test.go b/internal/svc/health/health_test.go
diff --git a/terraform/config.template.yaml b/terraform/config.template.yaml
@@ -57,6 +57,7 @@ mongo:
   db: ${mongo_database}
   username: ${mongo_username}
   password: ${mongo_password}
+  hedged_reads: ${mongo_hedged_reads}
 
 meilisearch:
   host: ${meili_url}

diff --git a/terraform/deployment.tf b/terraform/deployment.tf
@@ -33,6 +33,7 @@ resource "kubernetes_secret" "app" {
       mongo_username        = local.infra.mongodb_user_app.username
       mongo_password        = local.infra.mongodb_user_app.password
       mongo_database        = "7tv"
+      mongo_hedged_reads    = var.mongo_use_hedged_reads
       meili_url             = "http://meilisearch.database.svc.cluster.local:7700"
       meili_key             = var.meilisearch_key
       meili_index           = "emotes"
@@ -56,15 +57,11 @@ resource "kubernetes_deployment" "app" {
   metadata {
     name      = "api"
     namespace = data.kubernetes_namespace.app.metadata[0].name
-    labels    = {
+    labels = {
       app = "api"
     }
   }
 
-  lifecycle {
-    replace_triggered_by = [kubernetes_secret.app]
-  }
-
   timeouts {
     create = "4m"
     update = "5m"
@@ -261,8 +258,8 @@ resource "kubernetes_service" "app" {
 
 resource "kubernetes_ingress_v1" "app" {
   metadata {
-    name        = "api"
-    namespace   = data.kubernetes_namespace.app.metadata[0].name
+    name      = "api"
+    namespace = data.kubernetes_namespace.app.metadata[0].name
     annotations = {
       "kubernetes.io/ingress.class"                         = "nginx"
       "external-dns.alpha.kubernetes.io/target"             = local.infra.cloudflare_tunnel_hostname.regular
@@ -335,7 +332,7 @@ resource "kubernetes_horizontal_pod_autoscaler_v2" "api" {
       name        = kubernetes_deployment.app.metadata[0].name
     }
 
-    min_replicas = 2
+    min_replicas = local.infra.production ? 5 : 1
     max_replicas = 14
 
     metric {

diff --git a/terraform/variables.tf b/terraform/variables.tf
@@ -85,6 +85,11 @@ variable "kick_challenge_token" {
   default = ""
 }
 
+variable "mongo_use_hedged_reads" {
+  type = bool
+  default = true
+}
+
 variable "s3" {
   type = object({
     endpoint        = string