From ce97375280877f80e3dcebd20c21be44b13d8199 Mon Sep 17 00:00:00 2001 From: David Seifert Date: Sun, 21 Jul 2024 19:38:41 +0200 Subject: [PATCH 1/8] [build] fix for GCC 15 two-phase lookup * GCC 15 is more aggressive about checking dependent names Bug: https://bugs.gentoo.org/936409 --- src/vtab_module.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vtab_module.hh b/src/vtab_module.hh index 1278b9c157..538dae212f 100644 --- a/src/vtab_module.hh +++ b/src/vtab_module.hh @@ -582,7 +582,7 @@ struct vtab_module : public vtab_module_base { struct vtab { explicit vtab(sqlite3* db, T& impl) : v_db(db), v_impl(impl) {} - explicit operator sqlite3_vtab*() { return &this->base; } + explicit operator sqlite3_vtab*() { return &this->v_base; } sqlite3_vtab v_base{}; sqlite3* v_db; From 586b5073ed93cdb323ccd35f5f924067a025388a Mon Sep 17 00:00:00 2001 From: Tim Stack Date: Wed, 31 Jul 2024 16:36:06 -0700 Subject: [PATCH 2/8] [tz] handle invalid timezone Related to #1291 --- src/base/time_util.cc | 59 +++++++++++++++---- src/logfile.cc | 10 +++- src/readline_possibilities.cc | 8 ++- src/time_formats.am | 2 + test/expected/expected.am | 2 + ...04a2824c5e7dee903ef7500a3b60a123b7de01.err | 8 +++ ...04a2824c5e7dee903ef7500a3b60a123b7de01.out | 0 ...d7dc8a4bd3b8cb86a2f893f58a56f0f6ea1bc3.err | 8 --- ...d7dc8a4bd3b8cb86a2f893f58a56f0f6ea1bc3.out | 2 + test/logfile_invalid_tz.0 | 2 + test/test_sql_time_func.sh | 2 +- 11 files changed, 77 insertions(+), 26 deletions(-) create mode 100644 test/expected/test_sql_time_func.sh_1804a2824c5e7dee903ef7500a3b60a123b7de01.err create mode 100644 test/expected/test_sql_time_func.sh_1804a2824c5e7dee903ef7500a3b60a123b7de01.out create mode 100644 test/logfile_invalid_tz.0 diff --git a/src/base/time_util.cc b/src/base/time_util.cc index 95fddd436c..49a7c329e4 100644 --- a/src/base/time_util.cc +++ b/src/base/time_util.cc @@ -104,18 +104,26 @@ get_posix_zone(const char* name) } } -static const date::time_zone* +static std::optional get_date_zone(const char* name) { if (name == nullptr) { - return date::current_zone(); + try { + return date::current_zone(); + } catch (const std::runtime_error& e) { + return std::nullopt; + } } try { return date::locate_zone(name); } catch (const std::runtime_error& e) { log_error("invalid TZ value: %s -- %s", name, e.what()); - return date::current_zone(); + try { + return date::current_zone(); + } catch (const std::runtime_error& e) { + return std::nullopt; + } } } @@ -124,15 +132,21 @@ to_sys_time(date::local_seconds secs) { static const auto* TZ = getenv("TZ"); static const auto TZ_POSIX_ZONE = get_posix_zone(TZ); - static const auto* TZ_DATE_ZONE = get_date_zone(TZ); - if (TZ_POSIX_ZONE) { return TZ_POSIX_ZONE.value().to_sys(secs); } - auto inf = TZ_DATE_ZONE->get_info(secs); + static const auto TZ_DATE_ZONE = get_date_zone(TZ); - return TZ_DATE_ZONE->to_sys(secs); + if (TZ_DATE_ZONE) { + auto inf = TZ_DATE_ZONE.value()->get_info(secs); + + return TZ_DATE_ZONE.value()->to_sys(secs); + } + + static const auto TZ_POSIX_UTC = get_posix_zone("UTC0"); + + return TZ_POSIX_UTC.value().to_sys(secs); } date::local_seconds @@ -140,13 +154,20 @@ to_local_time(date::sys_seconds secs) { static const auto* TZ = getenv("TZ"); static const auto TZ_POSIX_ZONE = get_posix_zone(TZ); - static const auto* TZ_DATE_ZONE = get_date_zone(TZ); if (TZ_POSIX_ZONE) { return TZ_POSIX_ZONE.value().to_local(secs); } - return TZ_DATE_ZONE->to_local(secs); + static const auto TZ_DATE_ZONE = get_date_zone(TZ); + + if (TZ_DATE_ZONE) { + return TZ_DATE_ZONE.value()->to_local(secs); + } + + static const auto TZ_POSIX_UTC = get_posix_zone("UTC0"); + + return TZ_POSIX_UTC.value().to_local(secs); } date::sys_info @@ -154,13 +175,19 @@ sys_time_to_info(date::sys_seconds secs) { static const auto* TZ = getenv("TZ"); static const auto TZ_POSIX_ZONE = get_posix_zone(TZ); - static const auto* TZ_DATE_ZONE = get_date_zone(TZ); if (TZ_POSIX_ZONE) { return TZ_POSIX_ZONE.value().get_info(secs); } - return TZ_DATE_ZONE->get_info(secs); + static const auto TZ_DATE_ZONE = get_date_zone(TZ); + if (TZ_DATE_ZONE) { + return TZ_DATE_ZONE.value()->get_info(secs); + } + + static const auto TZ_POSIX_UTC = get_posix_zone("UTC0"); + + return TZ_POSIX_UTC.value().get_info(secs); } date::local_info @@ -168,13 +195,19 @@ local_time_to_info(date::local_seconds secs) { static const auto* TZ = getenv("TZ"); static const auto TZ_POSIX_ZONE = get_posix_zone(TZ); - static const auto* TZ_DATE_ZONE = get_date_zone(TZ); if (TZ_POSIX_ZONE) { return TZ_POSIX_ZONE.value().get_info(secs); } - return TZ_DATE_ZONE->get_info(secs); + static const auto TZ_DATE_ZONE = get_date_zone(TZ); + if (TZ_DATE_ZONE) { + return TZ_DATE_ZONE.value()->get_info(secs); + } + + static const auto TZ_POSIX_UTC = get_posix_zone("UTC0"); + + return TZ_POSIX_UTC.value().get_info(secs); } } // namespace lnav diff --git a/src/logfile.cc b/src/logfile.cc index 8d5991ab65..9a53c25c4e 100644 --- a/src/logfile.cc +++ b/src/logfile.cc @@ -182,8 +182,14 @@ logfile::open(std::filesystem::path filename, if (!phdr.h_timezone.empty()) { log_info("setting default time zone from piper header: %s", phdr.h_timezone.c_str()); - fo.fo_default_zone.pp_value - = date::locate_zone(phdr.h_timezone); + try { + fo.fo_default_zone.pp_value + = date::locate_zone(phdr.h_timezone); + } catch (const std::runtime_error& e) { + log_error("unable to get tz from piper header %s -- %s", + phdr.h_timezone.c_str(), + e.what()); + } } if (!fo.empty()) { safe::WriteAccess diff --git a/src/readline_possibilities.cc b/src/readline_possibilities.cc index 3d942e4609..1d04a34c91 100644 --- a/src/readline_possibilities.cc +++ b/src/readline_possibilities.cc @@ -560,8 +560,12 @@ add_tz_possibilities(ln_mode_t context) auto* rc = lnav_data.ld_rl_view; rc->clear_possibilities(context, "timezone"); - for (const auto& tz : date::get_tzdb().zones) { - rc->add_possibility(context, "timezone", tz.name()); + try { + for (const auto& tz : date::get_tzdb().zones) { + rc->add_possibility(context, "timezone", tz.name()); + } + } catch (const std::runtime_error& e) { + log_error("unable to get tzdb -- %s", e.what()); } { diff --git a/src/time_formats.am b/src/time_formats.am index 47b2a0048c..29253d45bd 100644 --- a/src/time_formats.am +++ b/src/time_formats.am @@ -25,6 +25,8 @@ TIME_FORMATS = \ "%y-%m-%dT%H:%M:%S.%f%z" \ "%Y-%m-%dT%H:%M:%S.%L%z" \ "%y-%m-%dT%H:%M:%S.%L%z" \ + "%Y-%m-%dT%H:%M:%S.%L%Z" \ + "%y-%m-%dT%H:%M:%S.%L%Z" \ "%Y-%m-%dT%H:%M:%S%z" \ "%Y-%m-%dT%H:%M:%S%z" \ "%Y-%m-%dT%H:%M:%S" \ diff --git a/test/expected/expected.am b/test/expected/expected.am index 35bf491489..598843d19a 100644 --- a/test/expected/expected.am +++ b/test/expected/expected.am @@ -1138,6 +1138,8 @@ EXPECTED_FILES = \ $(srcdir)/%reldir%/test_sql_time_func.sh_123c85ff1178743f5cb78efeaf98b637bcbe55ff.out \ $(srcdir)/%reldir%/test_sql_time_func.sh_14737ee9597b7d22519d23fbe34c0eb7d6c09ff2.err \ $(srcdir)/%reldir%/test_sql_time_func.sh_14737ee9597b7d22519d23fbe34c0eb7d6c09ff2.out \ + $(srcdir)/%reldir%/test_sql_time_func.sh_1804a2824c5e7dee903ef7500a3b60a123b7de01.err \ + $(srcdir)/%reldir%/test_sql_time_func.sh_1804a2824c5e7dee903ef7500a3b60a123b7de01.out \ $(srcdir)/%reldir%/test_sql_time_func.sh_1fbeb1ba69a95284eb1d4d052f5068ede7968704.err \ $(srcdir)/%reldir%/test_sql_time_func.sh_1fbeb1ba69a95284eb1d4d052f5068ede7968704.out \ $(srcdir)/%reldir%/test_sql_time_func.sh_20477acc218c96f1385dc97e4d28c80a05c93709.err \ diff --git a/test/expected/test_sql_time_func.sh_1804a2824c5e7dee903ef7500a3b60a123b7de01.err b/test/expected/test_sql_time_func.sh_1804a2824c5e7dee903ef7500a3b60a123b7de01.err new file mode 100644 index 0000000000..3ec027fd09 --- /dev/null +++ b/test/expected/test_sql_time_func.sh_1804a2824c5e7dee903ef7500a3b60a123b7de01.err @@ -0,0 +1,8 @@ +✘ error: invalid timestamp: 2022-03-02T10:20:30+ + reason: the leading part of the timestamp was matched, however, the trailing text “+” was not + --> command-option:1 + | ;SELECT timezone('UTC', '2022-03-02T10:20:30+') + = note: input matched time format “%Y-%m-%dT%H:%M:%S” + = note: 2022-03-02T10:20:30+ + ^ unrecognized input + = help: fix the timestamp or remove the trailing text diff --git a/test/expected/test_sql_time_func.sh_1804a2824c5e7dee903ef7500a3b60a123b7de01.out b/test/expected/test_sql_time_func.sh_1804a2824c5e7dee903ef7500a3b60a123b7de01.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/expected/test_sql_time_func.sh_c1d7dc8a4bd3b8cb86a2f893f58a56f0f6ea1bc3.err b/test/expected/test_sql_time_func.sh_c1d7dc8a4bd3b8cb86a2f893f58a56f0f6ea1bc3.err index ad8b26b8dd..e69de29bb2 100644 --- a/test/expected/test_sql_time_func.sh_c1d7dc8a4bd3b8cb86a2f893f58a56f0f6ea1bc3.err +++ b/test/expected/test_sql_time_func.sh_c1d7dc8a4bd3b8cb86a2f893f58a56f0f6ea1bc3.err @@ -1,8 +0,0 @@ -✘ error: invalid timestamp: 2022-03-02T10:20:30.400bad - reason: the leading part of the timestamp was matched, however, the trailing text “bad” was not - --> command-option:1 - | ;SELECT timezone('UTC', '2022-03-02T10:20:30.400bad') - = note: input matched time format “%Y-%m-%dT%H:%M:%S” - = note: 2022-03-02T10:20:30.400bad - ^-^ unrecognized input - = help: fix the timestamp or remove the trailing text diff --git a/test/expected/test_sql_time_func.sh_c1d7dc8a4bd3b8cb86a2f893f58a56f0f6ea1bc3.out b/test/expected/test_sql_time_func.sh_c1d7dc8a4bd3b8cb86a2f893f58a56f0f6ea1bc3.out index e69de29bb2..95df169cab 100644 --- a/test/expected/test_sql_time_func.sh_c1d7dc8a4bd3b8cb86a2f893f58a56f0f6ea1bc3.out +++ b/test/expected/test_sql_time_func.sh_c1d7dc8a4bd3b8cb86a2f893f58a56f0f6ea1bc3.out @@ -0,0 +1,2 @@ +timezone('UTC', '2022-03-02T10:20:30.400bad') +2022-03-02T10:20:30.400000+0000 diff --git a/test/logfile_invalid_tz.0 b/test/logfile_invalid_tz.0 new file mode 100644 index 0000000000..cbe55e6094 --- /dev/null +++ b/test/logfile_invalid_tz.0 @@ -0,0 +1,2 @@ +2015-04-24T21:08:10.313913FOO err rbd [22968]lotuscreds:ERROR:Could not retrieve lotus account information from db +2015-04-24T21:08:58.430632FOO err rbd [24206]networkutil:ERROR:The configured address sg01-1-vc1.oc.vmware.com was invalid diff --git a/test/test_sql_time_func.sh b/test/test_sql_time_func.sh index 1e35bd96c0..e16a9b74cd 100644 --- a/test/test_sql_time_func.sh +++ b/test/test_sql_time_func.sh @@ -82,4 +82,4 @@ run_cap_test ./drive_sql "SELECT timezone('UTC', '2022-03-02T10:20:30.400-0700') run_cap_test ${lnav_test} -nN -c ";SELECT timezone('bad-zone', '2022-03-02T10:20:30.400-0700')" -run_cap_test ${lnav_test} -nN -c ";SELECT timezone('UTC', '2022-03-02T10:20:30.400bad')" +run_cap_test ${lnav_test} -nN -c ";SELECT timezone('UTC', '2022-03-02T10:20:30+')" From 94dd0daa8c486ec3e2b2165ae6f2768415f2a27e Mon Sep 17 00:00:00 2001 From: Tim Stack Date: Fri, 2 Aug 2024 10:05:07 -0700 Subject: [PATCH 3/8] [log_format] check that a message with a module has a body --- src/log_format.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/log_format.cc b/src/log_format.cc index 7a2a66883f..a75fd586c6 100644 --- a/src/log_format.cc +++ b/src/log_format.cc @@ -1539,7 +1539,7 @@ external_log_format::scan(logfile& lf, mod_iter->second.mf_mod_format); if (mod_elf) { - static thread_local auto mod_md + thread_local auto mod_md = lnav::pcre2pp::match_data::unitialized(); shared_buffer_ref body_ref; From 05521cd5cd430f9c8f178ac4d85b8edccb6bc171 Mon Sep 17 00:00:00 2001 From: Tim Stack Date: Fri, 2 Aug 2024 10:20:09 -0700 Subject: [PATCH 4/8] sigh --- src/log_format.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/log_format.cc b/src/log_format.cc index a75fd586c6..b4e9b5f8b1 100644 --- a/src/log_format.cc +++ b/src/log_format.cc @@ -1523,7 +1523,7 @@ external_log_format::scan(logfile& lf, opid = opid_cap->hash(); } - if (mod_cap) { + if (mod_cap && body_cap) { intern_string_t mod_name = intern_string::lookup(mod_cap.value()); auto mod_iter = MODULE_FORMATS.find(mod_name); From f9f3caca0e282bd9f54c9ace5c599dd50b54197f Mon Sep 17 00:00:00 2001 From: Tim Stack Date: Fri, 2 Aug 2024 21:57:10 -0700 Subject: [PATCH 5/8] [java_log] allow truncated millis Related to #1286 --- src/formats/java_log.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/formats/java_log.json b/src/formats/java_log.json index 4a2338cecd..59b8c1565f 100644 --- a/src/formats/java_log.json +++ b/src/formats/java_log.json @@ -39,10 +39,10 @@ "pattern": "^(?\\d{4}-\\d{2}-\\d{2}( |T)\\d{2}:\\d{2}:\\d{2}(,|\\.)\\d{3}(?:Z|[-+]\\d{2}:?\\d{2})?)\\s+\\[(?[^\\]]+)\\]\\s+(?ERROR|WARN|INFO|DEBUG|TRACE)\\s+opId=(?\\S*)\\s+(?\\S+)\\s+-\\s+(?.*)$" }, "level-thread-class": { - "pattern": "(?\\d{4}-\\d{2}-\\d{2}[ T]\\d{2}:\\d{2}:\\d{2}[,\\.]\\d{3}) (?\\w+)\\s+\\[(?[^\\]]+)\\] (?[^:]+): (?.*)" + "pattern": "^(?\\d{4}-\\d{2}-\\d{2}[ T]\\d{2}:\\d{2}:\\d{2}[,\\.]\\d{1,3}) (?\\w+)\\s+\\[(?[^\\]]+)\\] (?[^:]+): (?.*)" }, "level-class-src": { - "pattern": "(?\\d{4}-\\d{2}-\\d{2}[ T]\\d{2}:\\d{2}:\\d{2}[,\\.]\\d{3}) (?\\w+)\\s+(?[\\.\\w]+) \\((?[^:]+):(?[^(]+)\\((?\\d+)\\)\\) - (?.*)" + "pattern": "^(?\\d{4}-\\d{2}-\\d{2}[ T]\\d{2}:\\d{2}:\\d{2}[,\\.]\\d{1,3}) (?\\w+)\\s+(?[\\.\\w]+) \\((?[^:]+):(?[^(]+)\\((?\\d+)\\)\\) - (?.*)" } }, "level-field": "level", From 6ce3c58ea4a83c080f255ce9181c2c1de6c86421 Mon Sep 17 00:00:00 2001 From: Tim Stack Date: Sun, 4 Aug 2024 09:07:02 -0700 Subject: [PATCH 6/8] [faq] add Q about searching Related to #1290 --- docs/source/faq.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/source/faq.rst b/docs/source/faq.rst index 013d1c9617..dd3ae727a2 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -83,3 +83,12 @@ Q: Why isn't a file being displayed? :Details: If a file being monitored by lnav does not match a known log file format, it is treated as plaintext and will be displayed in the TEXT view. + +Q: How can I search for an exact word? +-------------------------------------- + +:Solution: Surround the word to search for with :code:`\b`. + +:Details: The :code:`\b` means "word break" and matches a position where a + "word" ends. That is, right before a space character, punctuation, etc, + or at EOL. From e68182e0bc27bda6b25cf7347c8feca1da9405db Mon Sep 17 00:00:00 2001 From: Tim Stack Date: Wed, 14 Aug 2024 15:39:30 -0700 Subject: [PATCH 7/8] [java_log] add omitted message pattern Related to #1286 --- src/formats/java_log.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/formats/java_log.json b/src/formats/java_log.json index 59b8c1565f..318b8c0d7c 100644 --- a/src/formats/java_log.json +++ b/src/formats/java_log.json @@ -43,6 +43,9 @@ }, "level-class-src": { "pattern": "^(?\\d{4}-\\d{2}-\\d{2}[ T]\\d{2}:\\d{2}:\\d{2}[,\\.]\\d{1,3}) (?\\w+)\\s+(?[\\.\\w]+) \\((?[^:]+):(?[^(]+)\\((?\\d+)\\)\\) - (?.*)" + }, + "repeated": { + "pattern": "^(?\\d{4}-\\d{2}-\\d{2}[ T]\\d{2}:\\d{2}:\\d{2}[,\\.]\\d{1,3}) \\^\\^\\^ \\d+ similar messages? omitted" } }, "level-field": "level", From 2f0fb22a12d284479fc0b88c9391674342c576eb Mon Sep 17 00:00:00 2001 From: Tim Stack Date: Wed, 14 Aug 2024 15:49:12 -0700 Subject: [PATCH 8/8] [java_log] missed some stuff --- src/formats/java_log.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/formats/java_log.json b/src/formats/java_log.json index 318b8c0d7c..88d9d8fe44 100644 --- a/src/formats/java_log.json +++ b/src/formats/java_log.json @@ -45,7 +45,7 @@ "pattern": "^(?\\d{4}-\\d{2}-\\d{2}[ T]\\d{2}:\\d{2}:\\d{2}[,\\.]\\d{1,3}) (?\\w+)\\s+(?[\\.\\w]+) \\((?[^:]+):(?[^(]+)\\((?\\d+)\\)\\) - (?.*)" }, "repeated": { - "pattern": "^(?\\d{4}-\\d{2}-\\d{2}[ T]\\d{2}:\\d{2}:\\d{2}[,\\.]\\d{1,3}) \\^\\^\\^ \\d+ similar messages? omitted" + "pattern": "^(?\\d{4}-\\d{2}-\\d{2}[ T]\\d{2}:\\d{2}:\\d{2}[,\\.]\\d{1,3}) \\^\\^\\^ \\d+ similar messages? omitted \\^\\^\\^$" } }, "level-field": "level", @@ -175,6 +175,9 @@ }, { "line": "2024-04-23 09:17:33,457 INFO namenode.NameNode (LogAdapter.java:info(51)) - registered UNIX signal handlers for [TERM, HUP, INT]" + }, + { + "line": "2024-07-18 14:06:40.554 ^^^ 1 similar message omitted ^^^" } ] }