Merge branch 'tstack:master' into log-formats

tstack · Aug 20, 2024 · c44e3be · c44e3be
2 parents ce88dce + 2f0fb22
commit c44e3be
Show file tree

Hide file tree

Showing 15 changed files with 97 additions and 31 deletions.
diff --git a/docs/source/faq.rst b/docs/source/faq.rst
@@ -83,3 +83,12 @@ Q: Why isn't a file being displayed?
 
 :Details: If a file being monitored by lnav does not match a known log file
   format, it is treated as plaintext and will be displayed in the TEXT view.
+
+Q: How can I search for an exact word?
+--------------------------------------
+
+:Solution: Surround the word to search for with :code:`\b`.
+
+:Details: The :code:`\b` means "word break" and matches a position where a
+  "word" ends.  That is, right before a space character, punctuation, etc,
+  or at EOL.
diff --git a/src/base/time_util.cc b/src/base/time_util.cc
@@ -104,18 +104,26 @@ get_posix_zone(const char* name)
     }
 }
 
-static const date::time_zone*
+static std::optional<const date::time_zone*>
 get_date_zone(const char* name)
 {
     if (name == nullptr) {
-        return date::current_zone();
+        try {
+            return date::current_zone();
+        } catch (const std::runtime_error& e) {
+            return std::nullopt;
+        }
     }
 
     try {
         return date::locate_zone(name);
     } catch (const std::runtime_error& e) {
         log_error("invalid TZ value: %s -- %s", name, e.what());
-        return date::current_zone();
+        try {
+            return date::current_zone();
+        } catch (const std::runtime_error& e) {
+            return std::nullopt;
+        }
     }
 }
 
@@ -124,57 +132,82 @@ to_sys_time(date::local_seconds secs)
 {
     static const auto* TZ = getenv("TZ");
     static const auto TZ_POSIX_ZONE = get_posix_zone(TZ);
-    static const auto* TZ_DATE_ZONE = get_date_zone(TZ);
-
     if (TZ_POSIX_ZONE) {
         return TZ_POSIX_ZONE.value().to_sys(secs);
     }
 
-    auto inf = TZ_DATE_ZONE->get_info(secs);
+    static const auto TZ_DATE_ZONE = get_date_zone(TZ);
 
-    return TZ_DATE_ZONE->to_sys(secs);
+    if (TZ_DATE_ZONE) {
+        auto inf = TZ_DATE_ZONE.value()->get_info(secs);
+
+        return TZ_DATE_ZONE.value()->to_sys(secs);
+    }
+
+    static const auto TZ_POSIX_UTC = get_posix_zone("UTC0");
+
+    return TZ_POSIX_UTC.value().to_sys(secs);
 }
 
 date::local_seconds
 to_local_time(date::sys_seconds secs)
 {
     static const auto* TZ = getenv("TZ");
     static const auto TZ_POSIX_ZONE = get_posix_zone(TZ);
-    static const auto* TZ_DATE_ZONE = get_date_zone(TZ);
 
     if (TZ_POSIX_ZONE) {
         return TZ_POSIX_ZONE.value().to_local(secs);
     }
 
-    return TZ_DATE_ZONE->to_local(secs);
+    static const auto TZ_DATE_ZONE = get_date_zone(TZ);
+
+    if (TZ_DATE_ZONE) {
+        return TZ_DATE_ZONE.value()->to_local(secs);
+    }
+
+    static const auto TZ_POSIX_UTC = get_posix_zone("UTC0");
+
+    return TZ_POSIX_UTC.value().to_local(secs);
 }
 
 date::sys_info
 sys_time_to_info(date::sys_seconds secs)
 {
     static const auto* TZ = getenv("TZ");
     static const auto TZ_POSIX_ZONE = get_posix_zone(TZ);
-    static const auto* TZ_DATE_ZONE = get_date_zone(TZ);
 
     if (TZ_POSIX_ZONE) {
         return TZ_POSIX_ZONE.value().get_info(secs);
     }
 
-    return TZ_DATE_ZONE->get_info(secs);
+    static const auto TZ_DATE_ZONE = get_date_zone(TZ);
+    if (TZ_DATE_ZONE) {
+        return TZ_DATE_ZONE.value()->get_info(secs);
+    }
+
+    static const auto TZ_POSIX_UTC = get_posix_zone("UTC0");
+
+    return TZ_POSIX_UTC.value().get_info(secs);
 }
 
 date::local_info
 local_time_to_info(date::local_seconds secs)
 {
     static const auto* TZ = getenv("TZ");
     static const auto TZ_POSIX_ZONE = get_posix_zone(TZ);
-    static const auto* TZ_DATE_ZONE = get_date_zone(TZ);
 
     if (TZ_POSIX_ZONE) {
         return TZ_POSIX_ZONE.value().get_info(secs);
     }
 
-    return TZ_DATE_ZONE->get_info(secs);
+    static const auto TZ_DATE_ZONE = get_date_zone(TZ);
+    if (TZ_DATE_ZONE) {
+        return TZ_DATE_ZONE.value()->get_info(secs);
+    }
+
+    static const auto TZ_POSIX_UTC = get_posix_zone("UTC0");
+
+    return TZ_POSIX_UTC.value().get_info(secs);
 }
 
 }  // namespace lnav

diff --git a/src/formats/java_log.json b/src/formats/java_log.json
@@ -39,10 +39,13 @@
                 "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}( |T)\\d{2}:\\d{2}:\\d{2}(,|\\.)\\d{3}(?:Z|[-+]\\d{2}:?\\d{2})?)\\s+\\[(?<thread>[^\\]]+)\\]\\s+(?<level>ERROR|WARN|INFO|DEBUG|TRACE)\\s+opId=(?<opid>\\S*)\\s+(?<class>\\S+)\\s+-\\s+(?<body>.*)$"
             },
             "level-thread-class": {
-                "pattern": "(?<timestamp>\\d{4}-\\d{2}-\\d{2}[ T]\\d{2}:\\d{2}:\\d{2}[,\\.]\\d{3}) (?<level>\\w+)\\s+\\[(?<thread>[^\\]]+)\\] (?<class>[^:]+): (?<body>.*)"
+                "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}[ T]\\d{2}:\\d{2}:\\d{2}[,\\.]\\d{1,3}) (?<level>\\w+)\\s+\\[(?<thread>[^\\]]+)\\] (?<class>[^:]+): (?<body>.*)"
             },
             "level-class-src": {
-                "pattern": "(?<timestamp>\\d{4}-\\d{2}-\\d{2}[ T]\\d{2}:\\d{2}:\\d{2}[,\\.]\\d{3}) (?<level>\\w+)\\s+(?<class>[\\.\\w]+) \\((?<srcfile>[^:]+):(?<function>[^(]+)\\((?<srcline>\\d+)\\)\\) - (?<body>.*)"
+                "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}[ T]\\d{2}:\\d{2}:\\d{2}[,\\.]\\d{1,3}) (?<level>\\w+)\\s+(?<class>[\\.\\w]+) \\((?<srcfile>[^:]+):(?<function>[^(]+)\\((?<srcline>\\d+)\\)\\) - (?<body>.*)"
+            },
+            "repeated": {
+                "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}[ T]\\d{2}:\\d{2}:\\d{2}[,\\.]\\d{1,3}) \\^\\^\\^ \\d+ similar messages? omitted \\^\\^\\^$"
             }
         },
         "level-field": "level",
@@ -172,6 +175,9 @@
             },
             {
                 "line": "2024-04-23 09:17:33,457 INFO  namenode.NameNode (LogAdapter.java:info(51)) - registered UNIX signal handlers for [TERM, HUP, INT]"
+            },
+            {
+                "line": "2024-07-18 14:06:40.554 ^^^ 1 similar message omitted ^^^"
             }
         ]
     }

diff --git a/src/log_format.cc b/src/log_format.cc
@@ -1523,7 +1523,7 @@ external_log_format::scan(logfile& lf,
             opid = opid_cap->hash();
         }
 
-        if (mod_cap) {
+        if (mod_cap && body_cap) {
             intern_string_t mod_name = intern_string::lookup(mod_cap.value());
             auto mod_iter = MODULE_FORMATS.find(mod_name);
 
@@ -1539,7 +1539,7 @@ external_log_format::scan(logfile& lf,
                     mod_iter->second.mf_mod_format);
 
                 if (mod_elf) {
-                    static thread_local auto mod_md
+                    thread_local auto mod_md
                         = lnav::pcre2pp::match_data::unitialized();
 
                     shared_buffer_ref body_ref;

diff --git a/src/logfile.cc b/src/logfile.cc
@@ -182,8 +182,14 @@ logfile::open(std::filesystem::path filename,
                 if (!phdr.h_timezone.empty()) {
                     log_info("setting default time zone from piper header: %s",
                              phdr.h_timezone.c_str());
-                    fo.fo_default_zone.pp_value
-                        = date::locate_zone(phdr.h_timezone);
+                    try {
+                        fo.fo_default_zone.pp_value
+                            = date::locate_zone(phdr.h_timezone);
+                    } catch (const std::runtime_error& e) {
+                        log_error("unable to get tz from piper header %s -- %s",
+                                  phdr.h_timezone.c_str(),
+                                  e.what());
+                    }
                 }
                 if (!fo.empty()) {
                     safe::WriteAccess<lnav::safe_file_options_hier>

diff --git a/src/readline_possibilities.cc b/src/readline_possibilities.cc
@@ -560,8 +560,12 @@ add_tz_possibilities(ln_mode_t context)
     auto* rc = lnav_data.ld_rl_view;
 
     rc->clear_possibilities(context, "timezone");
-    for (const auto& tz : date::get_tzdb().zones) {
-        rc->add_possibility(context, "timezone", tz.name());
+    try {
+        for (const auto& tz : date::get_tzdb().zones) {
+            rc->add_possibility(context, "timezone", tz.name());
+        }
+    } catch (const std::runtime_error& e) {
+        log_error("unable to get tzdb -- %s", e.what());
     }
 
     {

diff --git a/src/time_formats.am b/src/time_formats.am
@@ -25,6 +25,8 @@ TIME_FORMATS = \
     "%y-%m-%dT%H:%M:%S.%f%z" \
     "%Y-%m-%dT%H:%M:%S.%L%z" \
     "%y-%m-%dT%H:%M:%S.%L%z" \
+    "%Y-%m-%dT%H:%M:%S.%L%Z" \
+    "%y-%m-%dT%H:%M:%S.%L%Z" \
     "%Y-%m-%dT%H:%M:%S%z" \
     "%Y-%m-%dT%H:%M:%S%z" \
     "%Y-%m-%dT%H:%M:%S" \

diff --git a/src/vtab_module.hh b/src/vtab_module.hh
@@ -582,7 +582,7 @@ struct vtab_module : public vtab_module_base {
     struct vtab {
         explicit vtab(sqlite3* db, T& impl) : v_db(db), v_impl(impl) {}
 
-        explicit operator sqlite3_vtab*() { return &this->base; }
+        explicit operator sqlite3_vtab*() { return &this->v_base; }
 
         sqlite3_vtab v_base{};
         sqlite3* v_db;

diff --git a/test/expected/expected.am b/test/expected/expected.am
@@ -1138,6 +1138,8 @@ EXPECTED_FILES = \
     $(srcdir)/%reldir%/test_sql_time_func.sh_123c85ff1178743f5cb78efeaf98b637bcbe55ff.out \
     $(srcdir)/%reldir%/test_sql_time_func.sh_14737ee9597b7d22519d23fbe34c0eb7d6c09ff2.err \
     $(srcdir)/%reldir%/test_sql_time_func.sh_14737ee9597b7d22519d23fbe34c0eb7d6c09ff2.out \
+    $(srcdir)/%reldir%/test_sql_time_func.sh_1804a2824c5e7dee903ef7500a3b60a123b7de01.err \
+    $(srcdir)/%reldir%/test_sql_time_func.sh_1804a2824c5e7dee903ef7500a3b60a123b7de01.out \
     $(srcdir)/%reldir%/test_sql_time_func.sh_1fbeb1ba69a95284eb1d4d052f5068ede7968704.err \
     $(srcdir)/%reldir%/test_sql_time_func.sh_1fbeb1ba69a95284eb1d4d052f5068ede7968704.out \
     $(srcdir)/%reldir%/test_sql_time_func.sh_20477acc218c96f1385dc97e4d28c80a05c93709.err \

diff --git a/test/expected/test_sql_time_func.sh_1804a2824c5e7dee903ef7500a3b60a123b7de01.err b/test/expected/test_sql_time_func.sh_1804a2824c5e7dee903ef7500a3b60a123b7de01.err
@@ -0,0 +1,8 @@
+✘ error: invalid timestamp: 2022-03-02T10:20:30+
+ reason: the leading part of the timestamp was matched, however, the trailing text “+” was not
+ --> command-option:1
+ | ;SELECT timezone('UTC', '2022-03-02T10:20:30+')
+ = note: input matched time format “%Y-%m-%dT%H:%M:%S”
+ = note: 2022-03-02T10:20:30+
+                            ^ unrecognized input
+ = help: fix the timestamp or remove the trailing text
diff --git a/test/expected/test_sql_time_func.sh_1804a2824c5e7dee903ef7500a3b60a123b7de01.out b/test/expected/test_sql_time_func.sh_1804a2824c5e7dee903ef7500a3b60a123b7de01.out
diff --git a/test/expected/test_sql_time_func.sh_c1d7dc8a4bd3b8cb86a2f893f58a56f0f6ea1bc3.err b/test/expected/test_sql_time_func.sh_c1d7dc8a4bd3b8cb86a2f893f58a56f0f6ea1bc3.err
@@ -1,8 +0,0 @@
-✘ error: invalid timestamp: 2022-03-02T10:20:30.400bad
- reason: the leading part of the timestamp was matched, however, the trailing text “bad” was not
- --> command-option:1
- | ;SELECT timezone('UTC', '2022-03-02T10:20:30.400bad')
- = note: input matched time format “%Y-%m-%dT%H:%M:%S”
- = note: 2022-03-02T10:20:30.400bad
-                                ^-^ unrecognized input
- = help: fix the timestamp or remove the trailing text

diff --git a/test/expected/test_sql_time_func.sh_c1d7dc8a4bd3b8cb86a2f893f58a56f0f6ea1bc3.out b/test/expected/test_sql_time_func.sh_c1d7dc8a4bd3b8cb86a2f893f58a56f0f6ea1bc3.out
@@ -0,0 +1,2 @@
+timezone('UTC', '2022-03-02T10:20:30.400bad') 
+2022-03-02T10:20:30.400000+0000               
diff --git a/test/logfile_invalid_tz.0 b/test/logfile_invalid_tz.0
@@ -0,0 +1,2 @@
+2015-04-24T21:08:10.313913FOO err rbd  [22968]lotuscreds:ERROR:Could not retrieve lotus account information from db
+2015-04-24T21:08:58.430632FOO err rbd  [24206]networkutil:ERROR:The configured address sg01-1-vc1.oc.vmware.com was invalid
diff --git a/test/test_sql_time_func.sh b/test/test_sql_time_func.sh
@@ -82,4 +82,4 @@ run_cap_test ./drive_sql "SELECT timezone('UTC', '2022-03-02T10:20:30.400-0700')
 
 run_cap_test ${lnav_test} -nN -c ";SELECT timezone('bad-zone', '2022-03-02T10:20:30.400-0700')"
 
-run_cap_test ${lnav_test} -nN -c ";SELECT timezone('UTC', '2022-03-02T10:20:30.400bad')"
+run_cap_test ${lnav_test} -nN -c ";SELECT timezone('UTC', '2022-03-02T10:20:30+')"