feat(logs): add thresholding, misc cleanup (#2773)

datahub-project · Jun 25, 2021 · 942196e · 942196e
1 parent 29be543
commit 942196e
Show file tree

Hide file tree

Showing 6 changed files with 40 additions and 20 deletions.
diff --git a/datahub-frontend/app/react/controllers/GraphQLController.java b/datahub-frontend/app/react/controllers/GraphQLController.java
@@ -105,9 +105,9 @@ public Result execute() throws Exception {
                 executionResult.toSpecification(),
                 executionResult.getErrors()));
         } else {
-            _logger.debug(String.format("Executed graphQL query: %s, result: %s"),
+            _logger.debug(String.format("Executed graphQL query: %s, result: %s",
                 queryJson,
-                executionResult.toSpecification());
+                executionResult.toSpecification()));
         }
 
 

diff --git a/datahub-frontend/conf/logback.xml b/datahub-frontend/conf/logback.xml
@@ -7,6 +7,9 @@
     <layout class="ch.qos.logback.classic.PatternLayout">
       <pattern>%d{HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n</pattern>
     </layout>
+    <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+      <level>INFO</level>
+    </filter>
   </appender>
 
   <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
@@ -20,6 +23,9 @@
       <!-- 30 days to keep -->
       <maxHistory>30</maxHistory>
     </rollingPolicy>
+    <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+      <level>INFO</level>
+    </filter>
     <encoder>
       <pattern>%d{HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n</pattern>
     </encoder>
@@ -38,28 +44,29 @@
     </rollingPolicy>
     <filter class="ch.qos.logback.classic.filter.LevelFilter">
       <level>DEBUG</level>
+      <onMatch>ACCEPT</onMatch>
+      <onMismatch>DENY</onMismatch>
     </filter>
     <encoder>
       <pattern>%d{HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n</pattern>
     </encoder>
   </appender>
 
   <!-- Change this to disable logging debug -->
-  <logger name="com.linkedin" level="debug">
+  <logger name="com.linkedin" level="DEBUG">
     <appender-ref ref="DEBUG_FILE"/>
   </logger>
-  <logger name="controller" level="debug">
+  <logger name="controller" level="DEBUG">
     <appender-ref ref="DEBUG_FILE"/>
   </logger>
-  <logger name="graphql" level="debug">
+  <logger name="graphql" level="DEBUG">
     <appender-ref ref="DEBUG_FILE"/>
   </logger>
-  <logger name="react" level="debug">
+  <logger name="react" level="DEBUG">
     <appender-ref ref="DEBUG_FILE"/>
   </logger>
 
-  <!-- Change this to disable logging debug -->
-  <root level="info">
+  <root level="INFO">
     <appender-ref ref="STDOUT"/>
     <appender-ref ref="FILE"/>
   </root>

diff --git a/datahub-frontend/run/logback.xml b/datahub-frontend/run/logback.xml
@@ -7,6 +7,9 @@
     <layout class="ch.qos.logback.classic.PatternLayout">
       <pattern>%d{HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n</pattern>
     </layout>
+    <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+      <level>INFO</level>
+    </filter>
   </appender>
 
   <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
@@ -20,6 +23,9 @@
       <!-- 30 days to keep -->
       <maxHistory>30</maxHistory>
     </rollingPolicy>
+    <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+      <level>INFO</level>
+    </filter>
     <encoder>
       <pattern>%d{HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n</pattern>
     </encoder>
@@ -38,27 +44,29 @@
     </rollingPolicy>
     <filter class="ch.qos.logback.classic.filter.LevelFilter">
       <level>DEBUG</level>
+      <onMatch>ACCEPT</onMatch>
+      <onMismatch>DENY</onMismatch>
     </filter>
     <encoder>
       <pattern>%d{HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n</pattern>
     </encoder>
   </appender>
 
   <!-- Change this to disable logging debug -->
-  <logger name="com.linkedin" level="debug">
+  <logger name="com.linkedin" level="DEBUG">
     <appender-ref ref="DEBUG_FILE"/>
   </logger>
-  <logger name="controller" level="debug">
+  <logger name="controller" level="DEBUG">
     <appender-ref ref="DEBUG_FILE"/>
   </logger>
-  <logger name="graphql" level="debug">
+  <logger name="graphql" level="DEBUG">
     <appender-ref ref="DEBUG_FILE"/>
   </logger>
-  <logger name="react" level="debug">
+  <logger name="react" level="DEBUG">
     <appender-ref ref="DEBUG_FILE"/>
   </logger>
 
-  <root level="info">
+  <root level="INFO">
     <appender-ref ref="STDOUT"/>
     <appender-ref ref="FILE"/>
   </root>

diff --git a/gms/impl/src/main/java/com/linkedin/metadata/filter/LoggingFilter.java b/gms/impl/src/main/java/com/linkedin/metadata/filter/LoggingFilter.java
@@ -52,7 +52,7 @@ private void logResponse(
     String method = requestContext.getMethod().getName();
     String uri = requestContext.getRequestURI().toString();
 
-    String logStr = String.format("%s %s - %s - %s - %sms", httpMethod, uri, method, status, duration);
+    String logStr = String.format("%s %s - %s - %s - %sms", httpMethod, uri, method, status.getCode(), duration);
     log.info(logStr);
   }
 

diff --git a/gms/impl/src/main/java/com/linkedin/metadata/resources/lineage/Relationships.java b/gms/impl/src/main/java/com/linkedin/metadata/resources/lineage/Relationships.java
@@ -25,7 +25,6 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.stream.Collectors;
-import lombok.extern.slf4j.Slf4j;
 
 import static com.linkedin.metadata.dao.Neo4jUtil.*;
 import static com.linkedin.metadata.dao.utils.QueryUtils.newFilter;
@@ -34,7 +33,6 @@
 /**
  * Rest.li entry point: /relationships?type={entityType}&direction={direction}&types={types}
  */
-@Slf4j
 @RestLiSimpleResource(name = "relationships", namespace = "com.linkedin.lineage")
 public final class Relationships extends SimpleResourceTemplate<EntityRelationships> {
 

diff --git a/gms/war/src/main/resources/logback.xml b/gms/war/src/main/resources/logback.xml
@@ -5,6 +5,9 @@
         <encoder>
             <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
         </encoder>
+        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+            <level>INFO</level>
+        </filter>
     </appender>
 
     <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
@@ -18,7 +21,9 @@
             <!-- 30 days to keep -->
             <maxHistory>30</maxHistory>
         </rollingPolicy>
-
+        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+            <level>INFO</level>
+        </filter>
         <encoder>
             <pattern>%d{HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n</pattern>
         </encoder>
@@ -37,22 +42,24 @@
         </rollingPolicy>
         <filter class="ch.qos.logback.classic.filter.LevelFilter">
             <level>DEBUG</level>
+            <onMatch>ACCEPT</onMatch>
+            <onMismatch>DENY</onMismatch>
         </filter>
         <encoder>
             <pattern>%d{HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n</pattern>
         </encoder>
     </appender>
 
     <!-- Remove this to disable logging debug -->
-    <logger name="com.linkedin" level="debug">
+    <logger name="com.linkedin" level="DEBUG">
         <appender-ref ref="DEBUG_FILE"/>
     </logger>
 
-    <logger name="org.apache.kafka.clients" level="info">
+    <logger name="org.apache.kafka.clients" level="INFO">
         <appender-ref ref="STDOUT" />
     </logger>
 
-    <root level="info">
+    <root level="INFO">
         <appender-ref ref="STDOUT" />
         <appender-ref ref="FILE"/>
     </root>