Disable logging of SQL queries (#981)

Logging every SQL query that is executed or transformer has some performance penalty. This PR adds a flag to enable it. We should not log queries by default. Tracing can be enabled via conf: ``` logging: enableSQLTracing: true ```
QuesmaOrg · Nov 14, 2024 · d7ac5fa · d7ac5fa
1 parent c250a90
commit d7ac5fa
Show file tree

Hide file tree

Showing 4 changed files with 15 additions and 8 deletions.
diff --git a/quesma/ingest/processor.go b/quesma/ingest/processor.go
@@ -820,8 +820,10 @@ func (ip *IngestProcessor) execute(ctx context.Context, query string) error {
 	span := ip.phoneHomeAgent.ClickHouseInsertDuration().Begin()
 
 	// We log every DDL query
-	if strings.HasPrefix(query, "ALTER") || strings.HasPrefix(query, "CREATE") {
-		logger.InfoWithCtx(ctx).Msgf("DDL query execution: %s", query)
+	if ip.cfg.Logging.EnableSQLTracing {
+		if strings.HasPrefix(query, "ALTER") || strings.HasPrefix(query, "CREATE") {
+			logger.InfoWithCtx(ctx).Msgf("DDL query execution: %s", query)
+		}
 	}
 
 	_, err := ip.chDb.ExecContext(ctx, query)
@@ -831,9 +833,7 @@ func (ip *IngestProcessor) execute(ctx context.Context, query string) error {
 
 func (ip *IngestProcessor) executeStatements(ctx context.Context, queries []string) error {
 	for _, q := range queries {
-		if strings.HasPrefix("ALTER", q) || strings.HasPrefix("CREATE", q) {
-			logger.InfoWithCtx(ctx).Msgf("DDL query execution: %s", q)
-		}
+
 		err := ip.execute(ctx, q)
 		if err != nil {
 			logger.ErrorWithCtx(ctx).Msgf("error executing query: %v", err)

diff --git a/quesma/quesma/config/config_v2.go b/quesma/quesma/config/config_v2.go
@@ -55,6 +55,7 @@ type LoggingConfiguration struct {
 	Level             *zerolog.Level `koanf:"level"`
 	FileLogging       bool           `koanf:"fileLogging"`
 	RemoteLogDrainUrl *Url
+	EnableSQLTracing  bool `koanf:"enableSQLTracing"`
 }
 
 type Pipeline struct {

diff --git a/quesma/quesma/schema_transformer.go b/quesma/quesma/schema_transformer.go
@@ -770,8 +770,10 @@ func (s *SchemaCheckPass) Transform(queries []*model.Query) ([]*model.Query, err
 
 				query.TransformationHistory.SchemaTransformers = append(query.TransformationHistory.SchemaTransformers, transformation.TransformationName)
 
-				logger.Info().Msgf(transformation.TransformationName+" triggered, input query: %s", inputQuery)
-				logger.Info().Msgf(transformation.TransformationName+" triggered, output query: %s", query.SelectCommand.String())
+				if s.cfg.Logging.EnableSQLTracing {
+					logger.Info().Msgf(transformation.TransformationName+" triggered, input query: %s", inputQuery)
+					logger.Info().Msgf(transformation.TransformationName+" triggered, output query: %s", query.SelectCommand.String())
+				}
 			}
 
 		}

diff --git a/quesma/quesma/search.go b/quesma/quesma/search.go
@@ -718,7 +718,11 @@ func (q *QueryRunner) searchWorkerCommon(
 
 	for i, query := range queries {
 		sql := query.SelectCommand.String()
-		logger.InfoWithCtx(ctx).Msgf("SQL: %s", sql)
+
+		if q.cfg.Logging.EnableSQLTracing {
+			logger.InfoWithCtx(ctx).Msgf("SQL: %s", sql)
+		}
+
 		translatedQueryBody[i].Query = []byte(sql)
 		if query.OptimizeHints != nil {
 			translatedQueryBody[i].PerformedOptimizations = query.OptimizeHints.OptimizationsPerformed