diff --git a/Cargo.lock b/Cargo.lock index dbf0f2b60..967c7c946 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -117,18 +117,6 @@ dependencies = [ "syn", ] -[[package]] -name = "async-broadcast" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bbd92a9bd0e9c1298118ecf8a2f825e86b12c3ec9e411573e34aaf3a0c03cdd" -dependencies = [ - "easy-parallel", - "event-listener", - "futures-core", - "parking_lot 0.11.2", -] - [[package]] name = "async-channel" version = "1.6.1" @@ -208,17 +196,6 @@ dependencies = [ "event-listener", ] -[[package]] -name = "async-recursion" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7d78656ba01f1b93024b7c3a0467f1608e4be67d725749fdcd7d2c7678fd7a2" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "async-std" version = "1.11.0" @@ -303,8 +280,8 @@ version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58aeb089fb698e06db8089971c7ee317ab9644bade33383f63631437b03aafb6" dependencies = [ - "glib-sys 0.15.10", - "gobject-sys 0.15.10", + "glib-sys", + "gobject-sys", "libc", "system-deps 6.0.2", ] @@ -516,7 +493,7 @@ version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c55d429bef56ac9172d25fecb85dc8068307d17acd74b377866b7a1ef25d3c8" dependencies = [ - "glib-sys 0.15.10", + "glib-sys", "libc", "system-deps 6.0.2", ] @@ -560,15 +537,6 @@ dependencies = [ "uuid 0.8.2", ] -[[package]] -name = "cfg-expr" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b412e83326147c2bb881f8b40edfbf9905b9b8abaebd0e47ca190ba62fda8f0e" -dependencies = [ - "smallvec", -] - [[package]] name = "cfg-expr" version = "0.9.1" @@ -977,6 +945,17 @@ dependencies = [ "lock_api", ] +[[package]] +name = "dbus" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de0a745c25b32caa56b82a3950f5fec7893a960f4c10ca3b02060b0c38d8c2ce" +dependencies = [ + "libc", + "libdbus-sys", + "winapi", +] + [[package]] name = "deflate" version = "0.7.20" @@ -996,17 +975,6 @@ dependencies = [ "adler32", ] -[[package]] -name = "derivative" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "derive_more" version = "0.99.17" @@ -1125,12 +1093,6 @@ dependencies = [ "dtoa", ] -[[package]] -name = "easy-parallel" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6907e25393cdcc1f4f3f513d9aac1e840eb1cc341a0fccb01171f7d14d10b946" - [[package]] name = "ego-tree" version = "0.6.2" @@ -1165,6 +1127,7 @@ dependencies = [ "anyhow", "chrono", "log", + "regex", "sea-orm", "serde", "shared", @@ -1172,27 +1135,6 @@ dependencies = [ "url 2.2.2", ] -[[package]] -name = "enumflags2" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e75d4cd21b95383444831539909fbb14b9dc3fdceb2a6f5d36577329a1f55ccb" -dependencies = [ - "enumflags2_derive", - "serde", -] - -[[package]] -name = "enumflags2_derive" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f58dc3c5e468259f19f2d46304a6b28f1c3d034442e14b322d2b850e36f6d5ae" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "event-listener" version = "2.5.2" @@ -1322,12 +1264,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "fuchsia-cprng" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" - [[package]] name = "futf" version = "0.1.5" @@ -1505,9 +1441,9 @@ version = "0.15.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "140b2f5378256527150350a8346dbdb08fadc13453a7a2d73aecd5fab3c402a7" dependencies = [ - "gio-sys 0.15.10", - "glib-sys 0.15.10", - "gobject-sys 0.15.10", + "gio-sys", + "glib-sys", + "gobject-sys", "libc", "system-deps 6.0.2", ] @@ -1520,9 +1456,9 @@ checksum = "32e7a08c1e8f06f4177fb7e51a777b8c1689f743a7bc11ea91d44d2226073a88" dependencies = [ "cairo-sys-rs", "gdk-pixbuf-sys", - "gio-sys 0.15.10", - "glib-sys 0.15.10", - "gobject-sys 0.15.10", + "gio-sys", + "glib-sys", + "gobject-sys", "libc", "pango-sys", "pkg-config", @@ -1536,7 +1472,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4b7f8c7a84b407aa9b143877e267e848ff34106578b64d1e0a24bf550716178" dependencies = [ "gdk-sys", - "glib-sys 0.15.10", + "glib-sys", "libc", "system-deps 6.0.2", "x11", @@ -1597,34 +1533,21 @@ dependencies = [ "futures-channel", "futures-core", "futures-io", - "gio-sys 0.15.10", + "gio-sys", "glib", "libc", "once_cell", "thiserror", ] -[[package]] -name = "gio-sys" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0a41df66e57fcc287c4bcf74fc26b884f31901ea9792ec75607289b456f48fa" -dependencies = [ - "glib-sys 0.14.0", - "gobject-sys 0.14.0", - "libc", - "system-deps 3.2.0", - "winapi", -] - [[package]] name = "gio-sys" version = "0.15.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32157a475271e2c4a023382e9cab31c4584ee30a97da41d3c4e9fdd605abcf8d" dependencies = [ - "glib-sys 0.15.10", - "gobject-sys 0.15.10", + "glib-sys", + "gobject-sys", "libc", "system-deps 6.0.2", "winapi", @@ -1642,8 +1565,8 @@ dependencies = [ "futures-executor", "futures-task", "glib-macros", - "glib-sys 0.15.10", - "gobject-sys 0.15.10", + "glib-sys", + "gobject-sys", "libc", "once_cell", "smallvec", @@ -1665,16 +1588,6 @@ dependencies = [ "syn", ] -[[package]] -name = "glib-sys" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c1d60554a212445e2a858e42a0e48cece1bd57b311a19a9468f70376cf554ae" -dependencies = [ - "libc", - "system-deps 3.2.0", -] - [[package]] name = "glib-sys" version = "0.15.10" @@ -1884,24 +1797,13 @@ dependencies = [ "web-sys", ] -[[package]] -name = "gobject-sys" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa92cae29759dae34ab5921d73fff5ad54b3d794ab842c117e36cafc7994c3f5" -dependencies = [ - "glib-sys 0.14.0", - "libc", - "system-deps 3.2.0", -] - [[package]] name = "gobject-sys" version = "0.15.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d57ce44246becd17153bd035ab4d32cfee096a657fc01f2231c9278378d1e0a" dependencies = [ - "glib-sys 0.15.10", + "glib-sys", "libc", "system-deps 6.0.2", ] @@ -1939,9 +1841,9 @@ dependencies = [ "cairo-sys-rs", "gdk-pixbuf-sys", "gdk-sys", - "gio-sys 0.15.10", - "glib-sys 0.15.10", - "gobject-sys 0.15.10", + "gio-sys", + "glib-sys", + "gobject-sys", "libc", "pango-sys", "system-deps 6.0.2", @@ -1976,7 +1878,7 @@ dependencies = [ "indexmap", "slab", "tokio", - "tokio-util 0.7.1", + "tokio-util 0.7.2", "tracing", ] @@ -2065,7 +1967,7 @@ checksum = "ff8670570af52249509a86f5e3e18a08c60b177071826898fde8997cf5f6bfbb" dependencies = [ "bytes", "fnv", - "itoa 1.0.1", + "itoa 1.0.2", ] [[package]] @@ -2112,7 +2014,7 @@ dependencies = [ "http-body", "httparse", "httpdate", - "itoa 1.0.1", + "itoa 1.0.2", "pin-project-lite", "socket2", "tokio", @@ -2253,9 +2155,9 @@ checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" [[package]] name = "itoa" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" +checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" [[package]] name = "javascriptcore-rs" @@ -2274,8 +2176,8 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "905fbb87419c5cde6e3269537e4ea7d46431f3008c5d057e915ef3f115e7793c" dependencies = [ - "glib-sys 0.15.10", - "gobject-sys 0.15.10", + "glib-sys", + "gobject-sys", "libc", "system-deps 5.0.0", ] @@ -2420,7 +2322,7 @@ dependencies = [ "log", "tokio", "tokio-stream", - "tokio-util 0.6.9", + "tokio-util 0.6.10", "unicase", ] @@ -2475,6 +2377,15 @@ version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5916d2ae698f6de9bfb891ad7a8d65c09d232dc58cc4ac433c7da3b2fd84bc2b" +[[package]] +name = "libdbus-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c185b5b7ad900923ef3a8ff594083d4d9b5aea80bb4f32b8342363138c0d456b" +dependencies = [ + "pkg-config", +] + [[package]] name = "libsqlite3-sys" version = "0.24.2" @@ -2508,9 +2419,9 @@ dependencies = [ [[package]] name = "loom" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edc5c7d328e32cc4954e8e01193d7f0ef5ab257b5090b70a964e099a36034309" +checksum = "85eb735cf3c8ebac6cc3655c5da2f4a088b6a19133aa482471a21ba0eb5d83ab" dependencies = [ "cfg-if", "generator", @@ -2635,7 +2546,7 @@ name = "migration" version = "0.1.0" dependencies = [ "entities", - "sea-schema", + "sea-orm-migration", ] [[package]] @@ -2661,25 +2572,14 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9" +checksum = "713d550d9b44d89174e066b7a6217ae06234c10cb47819a88290d2b353c31799" dependencies = [ "libc", "log", - "miow", - "ntapi", "wasi 0.11.0+wasi-snapshot-preview1", - "winapi", -] - -[[package]] -name = "miow" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" -dependencies = [ - "winapi", + "windows-sys", ] [[package]] @@ -2768,19 +2668,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54" -[[package]] -name = "nix" -version = "0.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f866317acbd3a240710c63f065ffb1e4fd466259045ccb504130b7f668f35c6" -dependencies = [ - "bitflags", - "cc", - "cfg-if", - "libc", - "memoffset", -] - [[package]] name = "nodrop" version = "0.1.14" @@ -2803,21 +2690,9 @@ version = "4.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a995a3d2834cefa389218e7a35156e8ce544bc95f836900da01ee0b26a07e9d4" dependencies = [ + "dbus", "mac-notification-sys", - "serde", "winrt-notification", - "zbus", - "zvariant", - "zvariant_derive", -] - -[[package]] -name = "ntapi" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f" -dependencies = [ - "winapi", ] [[package]] @@ -3011,16 +2886,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "ordered-stream" -version = "0.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44630c059eacfd6e08bdaa51b1db2ce33119caa4ddc1235e923109aa5f25ccb1" -dependencies = [ - "futures-core", - "pin-project-lite", -] - [[package]] name = "os_info" version = "3.3.0" @@ -3103,8 +2968,8 @@ version = "0.15.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2a00081cde4661982ed91d80ef437c20eacaf6aa1a5962c0279ae194662c3aa" dependencies = [ - "glib-sys 0.15.10", - "gobject-sys 0.15.10", + "glib-sys", + "gobject-sys", "libc", "system-deps 6.0.2", ] @@ -3458,18 +3323,18 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" [[package]] name = "proc-macro2" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec757218438d5fda206afc041538b2f6d889286160d649a86a24d37e1235afd1" +checksum = "9027b48e9d4c9175fa2218adf3557f91c1137021739951d4932f5f8268ac48aa" dependencies = [ "unicode-xid", ] [[package]] name = "psl" -version = "2.0.87" +version = "2.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "676a055de12d26776790188a69ec2969f9272f85e00ab0555df81f74db867434" +checksum = "027244bd67547f5897fa12d991182bf3af92fcfd55860410fbb5ccbebd80b579" dependencies = [ "psl-types", ] @@ -3489,19 +3354,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "rand" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" -dependencies = [ - "fuchsia-cprng", - "libc", - "rand_core 0.3.1", - "rdrand", - "winapi", -] - [[package]] name = "rand" version = "0.7.3" @@ -3547,21 +3399,6 @@ dependencies = [ "rand_core 0.6.3", ] -[[package]] -name = "rand_core" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" -dependencies = [ - "rand_core 0.4.2", -] - -[[package]] -name = "rand_core" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" - [[package]] name = "rand_core" version = "0.5.1" @@ -3609,9 +3446,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.5.2" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd249e82c21598a9a426a4e00dd7adc1d640b22445ec8545feef801d1a74c221" +checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d" dependencies = [ "autocfg", "crossbeam-deque", @@ -3621,9 +3458,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.9.2" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f51245e1e62e1f1629cbfec37b5793bbabcaeb90f30e94d2ba03564687353e4" +checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f" dependencies = [ "crossbeam-channel", "crossbeam-deque", @@ -3631,15 +3468,6 @@ dependencies = [ "num_cpus", ] -[[package]] -name = "rdrand" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" -dependencies = [ - "rand_core 0.3.1", -] - [[package]] name = "redox_syscall" version = "0.2.13" @@ -3758,8 +3586,8 @@ checksum = "92e3107b2e81967df7c0617e978dc656795583a73ad0ddbf645ce60109caf8c2" dependencies = [ "block", "dispatch", - "glib-sys 0.15.10", - "gobject-sys 0.15.10", + "glib-sys", + "gobject-sys", "gtk-sys", "js-sys", "lazy_static", @@ -3869,9 +3697,9 @@ checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f" [[package]] name = "ryu" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" +checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695" [[package]] name = "same-file" @@ -3922,9 +3750,9 @@ dependencies = [ [[package]] name = "sea-orm" -version = "0.7.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27dbb8a742003f8dbf2ba290d128134d4275a6b55fd02f4d728683b6b55ea9bf" +checksum = "51de529763804dd4f74c133055f53eccdda2221bdded94351009be28cc80d2fb" dependencies = [ "async-stream", "async-trait", @@ -3935,7 +3763,7 @@ dependencies = [ "ouroboros", "rust_decimal", "sea-orm-macros", - "sea-query 0.23.0", + "sea-query", "sea-strum", "serde", "serde_json", @@ -3946,11 +3774,28 @@ dependencies = [ "uuid 0.8.2", ] +[[package]] +name = "sea-orm-cli" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fca862fdba12c753bffba9c9adf95d3d3f5dcc82fd589b12faeee7068bb173d5" +dependencies = [ + "async-std", + "chrono", + "clap", + "dotenv", + "regex", + "sea-schema", + "tracing", + "tracing-subscriber", + "url 2.2.2", +] + [[package]] name = "sea-orm-macros" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953bf5fb9f6ec985c139c4a98550b600c2f7c97bea74e2acc4025438469cb5a2" +checksum = "9f9378e21366b119d281489013c8170c49972fd3709c2155eb4504a913715d2d" dependencies = [ "bae", "heck 0.3.3", @@ -3960,23 +3805,32 @@ dependencies = [ ] [[package]] -name = "sea-query" -version = "0.22.0" +name = "sea-orm-migration" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "727090e8d1e61edd07305d237664315226748ad559e16daa6293fa26c4e7a3c3" +checksum = "6be15a93dc65a67bb3615fc0fe45c22581b25b27123520e18fb66cd87f429f4c" dependencies = [ - "sea-query-derive", + "async-std", + "async-trait", + "clap", + "dotenv", + "sea-orm", + "sea-orm-cli", + "sea-schema", + "tracing", + "tracing-subscriber", ] [[package]] name = "sea-query" -version = "0.23.0" +version = "0.24.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bf24fc03259e206d8cd4c957ce7446fe54ab00ba5ada4cdb028aa3513e26231" +checksum = "6b0fa62db5ae33dfc61e805b0b0c9d579c3733f1ed90326b3779f5b38f30fa2a" dependencies = [ "chrono", "rust_decimal", "sea-query-derive", + "sea-query-driver", "serde_json", "time 0.2.27", "uuid 0.8.2", @@ -3995,22 +3849,26 @@ dependencies = [ "thiserror", ] +[[package]] +name = "sea-query-driver" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3953baee94dcb90f0e19e8b4b91b91e9394867b0fc1886d0221cfc6d0439f5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "sea-schema" -version = "0.7.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d492f4550a428d2be29e4df8e43b2e9e46717424c9603fafa3365ae6079bd73" +checksum = "09fea4b9dccc8b0667f108de2d09bdabd42a137b8437de092374a4e36de8c12f" dependencies = [ - "async-std", - "async-trait", - "clap", - "dotenv", - "log", - "sea-orm", - "sea-query 0.22.0", + "futures 0.3.21", + "sea-query", "sea-schema-derive", - "tracing", - "tracing-subscriber", ] [[package]] @@ -4167,7 +4025,7 @@ version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" dependencies = [ - "itoa 1.0.1", + "itoa 1.0.2", "ryu", "serde", ] @@ -4190,7 +4048,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", - "itoa 1.0.1", + "itoa 1.0.2", "ryu", "serde", ] @@ -4348,16 +4206,30 @@ dependencies = [ "winapi", ] +[[package]] +name = "soup2" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b4d76501d8ba387cf0fefbe055c3e0a59891d09f0f995ae4e4b16f6b60f3c0" +dependencies = [ + "bitflags", + "gio", + "glib", + "libc", + "once_cell", + "soup2-sys", +] + [[package]] name = "soup2-sys" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f056675eda9a7417163e5f742bb119e8e1d385edd2ada8f7031a7230a3ec10a" +checksum = "009ef427103fcb17f802871647a7fa6c60cbb654b4c4e4c0ac60a31c5f6dc9cf" dependencies = [ "bitflags", - "gio-sys 0.14.0", - "glib-sys 0.14.0", - "gobject-sys 0.14.0", + "gio-sys", + "glib-sys", + "gobject-sys", "libc", "system-deps 5.0.0", ] @@ -4502,7 +4374,7 @@ dependencies = [ "hashlink", "hex", "indexmap", - "itoa 1.0.1", + "itoa 1.0.2", "libc", "libsqlite3-sys", "log", @@ -4693,31 +4565,13 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" -[[package]] -name = "strum" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf86bbcfd1fa9670b7a129f64fc0c9fcbbfe4f1bc4210e9e98fe71ffc12cde2" - [[package]] name = "strum" version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7ac893c7d471c8a21f31cfe213ec4f6d9afeed25537c772e08ef3f005f8729e" dependencies = [ - "strum_macros 0.22.0", -] - -[[package]] -name = "strum_macros" -version = "0.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d06aaeeee809dbc59eb4556183dd927df67db1540de5be8d3ec0b6636358a5ec" -dependencies = [ - "heck 0.3.3", - "proc-macro2", - "quote", - "syn", + "strum_macros", ] [[package]] @@ -4734,33 +4588,15 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.92" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ff7c592601f11445996a06f8ad0c27f094a58857c2f89e97974ab9235b92c52" +checksum = "a07e33e919ebcd69113d5be0e4d70c5707004ff45188910106854f38b960df4a" dependencies = [ "proc-macro2", "quote", "unicode-xid", ] -[[package]] -name = "system-deps" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "480c269f870722b3b08d2f13053ce0c2ab722839f472863c3e2d61ff3a1c2fa6" -dependencies = [ - "anyhow", - "cfg-expr 0.8.1", - "heck 0.3.3", - "itertools", - "pkg-config", - "strum 0.21.0", - "strum_macros 0.21.1", - "thiserror", - "toml", - "version-compare 0.0.11", -] - [[package]] name = "system-deps" version = "5.0.0" @@ -4903,7 +4739,7 @@ dependencies = [ "gdkx11-sys", "gio", "glib", - "glib-sys 0.15.10", + "glib-sys", "gtk", "instant", "lazy_static", @@ -4950,13 +4786,14 @@ dependencies = [ [[package]] name = "tauri" -version = "1.0.0-rc.8" +version = "1.0.0-rc.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "537978045ca229b9c1bb51ea85bc807b9d109a119721134fc5da24f94fd3074a" +checksum = "6d5ab16d343563684af482b1cb87acc515a1e359515b70c525c709372134c257" dependencies = [ "anyhow", "attohttpc", "bincode", + "cocoa", "dirs-next", "embed_plist", "flate2", @@ -4969,6 +4806,7 @@ dependencies = [ "http", "ignore", "notify-rust", + "objc", "once_cell", "open", "os_info", @@ -4995,17 +4833,20 @@ dependencies = [ "tokio", "url 2.2.2", "uuid 1.0.0", + "webkit2gtk", + "webview2-com", "windows 0.30.0", ] [[package]] name = "tauri-build" -version = "1.0.0-rc.7" +version = "1.0.0-rc.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e6448e80778032b4f9dd86b5efc8214d5bfc81a11efa502bb5211b05d422b14" +checksum = "3f85528e1a51b1d79761f56a0af8fb639ffa282e6bb01b012cdd552673e45be6" dependencies = [ "anyhow", "cargo_toml", + "semver 1.0.9", "serde_json", "tauri-utils", "winres", @@ -5013,9 +4854,9 @@ dependencies = [ [[package]] name = "tauri-codegen" -version = "1.0.0-rc.5" +version = "1.0.0-rc.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4c2e553c2ceaf30f1feabc76abebbd5f9eddb99b643de0078e38037e43e3c2f" +checksum = "7b6273cb4ba4210d48cba182415dd8ed2f23b29f9f89801f77a09fcbb6a6d92b" dependencies = [ "base64", "brotli", @@ -5035,9 +4876,9 @@ dependencies = [ [[package]] name = "tauri-macros" -version = "1.0.0-rc.5" +version = "1.0.0-rc.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3e8af1367b1e1224edfa4117c88fe19717970fabfbc2555e957e077f0469248" +checksum = "07d91657771bb36eca42d86afc80aa05e10f6b7328f8f40630e52c10ae84ddf3" dependencies = [ "heck 0.4.0", "proc-macro2", @@ -5049,9 +4890,9 @@ dependencies = [ [[package]] name = "tauri-runtime" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27653d24a0d7e2c8e04838e975acbf7a5628746d8d60a916d33a9ccf8a06c4ea" +checksum = "86b266b563439a4c300a524edc695541d72cb5ba55cd41f27adc6944c9c88d6e" dependencies = [ "gtk", "http", @@ -5068,15 +4909,18 @@ dependencies = [ [[package]] name = "tauri-runtime-wry" -version = "0.4.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d9b0922c27ea8a1430a2bbf666fe7789645dffb9d317f8d2dfca1a5dff2271" +checksum = "31a5558141be2d58e9f8071ed29d49ccf56076d0b12d40b6b21c6d3f6786b8d2" dependencies = [ + "cocoa", "gtk", + "percent-encoding 2.1.0", "rand 0.8.5", "tauri-runtime", "tauri-utils", "uuid 1.0.0", + "webkit2gtk", "webview2-com", "windows 0.30.0", "wry", @@ -5084,9 +4928,9 @@ dependencies = [ [[package]] name = "tauri-utils" -version = "1.0.0-rc.5" +version = "1.0.0-rc.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a485f9fc0f381d3da0818c4260b3a04be86dc1844a12edaff68afb07bc55d735" +checksum = "ed2d8266063ac4d696560d4ff4e695c3c238d8d96ef1120d1eab9ba7482f59a3" dependencies = [ "brotli", "ctor", @@ -5107,16 +4951,6 @@ dependencies = [ "walkdir", ] -[[package]] -name = "tempdir" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" -dependencies = [ - "rand 0.4.6", - "remove_dir_all", -] - [[package]] name = "tempfile" version = "3.3.0" @@ -5218,7 +5052,7 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2702e08a7a860f005826c6815dcac101b19b5eb330c27fe4a5928fec1d20ddd" dependencies = [ - "itoa 1.0.1", + "itoa 1.0.2", "libc", "num_threads", ] @@ -5263,9 +5097,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.18.1" +version = "1.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dce653fb475565de9f6fb0614b28bca8df2c430c0cf84bcd9c843f15de5414cc" +checksum = "4903bf0427cf68dddd5aa6a93220756f8be0c34fcfa9f5e6191e103e15a31395" dependencies = [ "bytes", "libc", @@ -5337,9 +5171,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.6.9" +version = "0.6.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0" +checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" dependencies = [ "bytes", "futures-core", @@ -5351,9 +5185,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0edfdeb067411dba2044da6d1cb2df793dd35add7888d73c16e3381ded401764" +checksum = "f988a1a1adc2fb21f9c12aa96441da33a1728193ae0b95d2be22dbd17fcb4e5c" dependencies = [ "bytes", "futures-core", @@ -5489,16 +5323,6 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c" -[[package]] -name = "uds_windows" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "486992108df0fe0160680af1941fe856c521be931d5a5ecccefe0de86dc47e4a" -dependencies = [ - "tempdir", - "winapi", -] - [[package]] name = "unicase" version = "2.6.0" @@ -5790,48 +5614,49 @@ dependencies = [ [[package]] name = "webkit2gtk" -version = "0.17.1" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cbd39499e917de9dad36eb11c09f665eb984d432638ae7971feed98eb96df88" +checksum = "29952969fb5e10fe834a52eb29ad0814ccdfd8387159b0933edf1344a1c9cdcc" dependencies = [ "bitflags", "cairo-rs", "gdk", "gdk-sys", "gio", - "gio-sys 0.15.10", + "gio-sys", "glib", - "glib-sys 0.15.10", - "gobject-sys 0.15.10", + "glib-sys", + "gobject-sys", "gtk", "gtk-sys", "javascriptcore-rs", "libc", "once_cell", + "soup2", "webkit2gtk-sys", ] [[package]] name = "webkit2gtk-sys" -version = "0.17.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddcce6f1e0fc7715d651dba29875741509f5fc12f4e2976907272a74405f2b01" +checksum = "4d76ca6ecc47aeba01ec61e480139dda143796abcae6f83bcddf50d6b5b1dcf3" dependencies = [ "atk-sys", "bitflags", "cairo-sys-rs", "gdk-pixbuf-sys", "gdk-sys", - "gio-sys 0.15.10", - "glib-sys 0.15.10", - "gobject-sys 0.15.10", + "gio-sys", + "glib-sys", + "gobject-sys", "gtk-sys", "javascriptcore-rs-sys", "libc", "pango-sys", "pkg-config", "soup2-sys", - "system-deps 5.0.0", + "system-deps 6.0.2", ] [[package]] @@ -6169,16 +5994,16 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "007a0353840b23e0c6dc73e5b962ff58ed7f6bc9ceff3ce7fe6fbad8d496edf4" dependencies = [ - "strum 0.22.0", + "strum", "windows 0.24.0", "xml-rs", ] [[package]] name = "wry" -version = "0.15.1" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20b69cff9f50bab10b42e51bac9c2cf695484059f1b19e911754477ae703ef42" +checksum = "a5676092e1a33448ed0f268717bcbb2e928354b78f4c1f60f3d43641eedea0d9" dependencies = [ "block", "cocoa", @@ -6274,91 +6099,3 @@ dependencies = [ "quote", "syn", ] - -[[package]] -name = "zbus" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53819092b9db813b2c6168b097b4b13ad284d81c9f2b0165a0a1b190e505a1f3" -dependencies = [ - "async-broadcast", - "async-channel", - "async-executor", - "async-io", - "async-lock", - "async-recursion", - "async-task", - "async-trait", - "byteorder", - "derivative", - "enumflags2", - "event-listener", - "futures-core", - "futures-sink", - "futures-util", - "hex", - "lazy_static", - "nix", - "once_cell", - "ordered-stream", - "rand 0.8.5", - "serde", - "serde_repr", - "sha1", - "static_assertions", - "uds_windows", - "winapi", - "zbus_macros", - "zbus_names", - "zvariant", -] - -[[package]] -name = "zbus_macros" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7174ebe6722c280d6d132d694bb5664ce50a788cb70eeb518e7fc1ca095a114" -dependencies = [ - "proc-macro-crate 1.1.3", - "proc-macro2", - "quote", - "regex", - "syn", -] - -[[package]] -name = "zbus_names" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45dfcdcf87b71dad505d30cc27b1b7b88a64b6d1c435648f48f9dbc1fdc4b7e1" -dependencies = [ - "serde", - "static_assertions", - "zvariant", -] - -[[package]] -name = "zvariant" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e18ba99d71e03af262953f476071607da0c44e225236cf9b5b9f7f11f1d0b6b0" -dependencies = [ - "byteorder", - "enumflags2", - "libc", - "serde", - "static_assertions", - "zvariant_derive", -] - -[[package]] -name = "zvariant_derive" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9042892ebdca35261951a83d17bcbfd4d3d528cb3bde828498a9b306b50d05c0" -dependencies = [ - "proc-macro-crate 1.1.3", - "proc-macro2", - "quote", - "syn", -] diff --git a/README.md b/README.md index aa8bf2995..f54fad890 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,13 @@ # Spyglass -> tl; dr: Spyglass is a search platform that lives on your device, indexing what -> you want, exposing it to you in a super simple & fast interface. +## tl; dr; Spyglass indexes what you want exposing it to you in a super simple & fast interface -> ⚠️ Spyglass is very much in its early stages, but it’s in a place where it's functional -> and can be used to replace basic searches. ⚠️ +⚠️ Spyglass is very much in its early stages, but it’s in a place where it's functional and can be used to replace basic searches. ⚠️ + +Download now: [Mac](https://github.com/a5huynh/spyglass/releases/download/v2022.5.11/Spyglass_22.5.11_x64.dmg) | [Windows](https://github.com/a5huynh/spyglass/releases/download/v2022.5.11/Spyglass_22.5.11_x64_en-US.msi) | [Linux (AppImage)](https://github.com/a5huynh/spyglass/releases/download/v2022.5.11/spyglass_22.5.11_amd64.AppImage) + +--- ## Table of Contents @@ -20,6 +22,7 @@ * [Settings](#settings) * [Updating the shorcut](#updating-the-shortcut) +--- ## Installation @@ -35,11 +38,15 @@ make build-release ## Spyglass in action -Once launched, press **`Cmd + Shift + /`** to open Spyglass. Queries prefixed with `/` -will search through your installed lenses, otherwise it'll search through your index. +Once launched, press **`Cmd + Shift + /`** to open Spyglass. If the app has been +successfully launched, you'll see a little menubar icon like the following: + +![Menubar icon and menu](docs/menubar-menu.png) + -Use the arrow keys to select the result you want and hit `Enter` to open the link in the -browser of your choice! +Queries prefixed with `/` will search through your installed lenses, otherwise it'll +search through your index. Use the arrow keys to select the result you want and hit +`Enter` to open the link in the browser of your choice! [![Spyglass in action!](docs/spyglass-poc.gif)](https://www.youtube.com/embed/OzNrxtM3s_8) @@ -72,6 +79,8 @@ curated set of websites with high quality recipes. ``` rust ( version: "1", + // Be proud of your creation :). Maybe soon we can share these ;) + author: "Andrew Huynh", name: "recipes", description: Some(r#" A curated collection of websites with useful, high-quality recipes. @@ -88,9 +97,13 @@ curated set of websites with high quality recipes. "www.vickypham.com", ], - // Not yet supported but ideally more ways to filter URLs within a domain urls: [ - "www.reddit.com/r/recipes/*", + // URLs are considered prefixed, i.e. anything that starts w/ the following + // will be matched and crawled. + // + // https://www.reddit.com/r/recipes/ -> matches + // https://www.reddit.com/r/recipes_not/ -> does not matche, notice the end slash. + "https://www.reddit.com/r/recipes/", ] ) ``` @@ -105,6 +118,7 @@ programming language and not the Rust game / The Rust Belt / oxidation / etc. ``` rust ( version: "1", + author: "Andrew Huynh", name: "rustlang", description: Some("Rustlang targeted websites"), domains: [ @@ -117,12 +131,9 @@ programming language and not the Rust game / The Rust Belt / oxidation / etc. ... ], - // Again not yet supported but an example of indexing specific communities that - // are relevant to the topic urls: [ - "www.reddit.com/r/rust", - "www.reddit.com/r/rust_gamedev", - "https://github.com/topics/rust" + "https://www.reddit.com/r/rust/", + "https://www.reddit.com/r/rust_gamedev/", ] ) ``` @@ -145,13 +156,19 @@ file found in their directory on startup, a default one will be created. run_wizard: false, // Not used... yet! allow_list: [], - // Domains to completely ignore. + // Domains to completely ignore, regardless of the lenses you have installed. block_list: [ "web.archive.org", "w3schools.com" ], // Shortcut to launch the search bar shortcut: "CmdOrCtrl+Shift+/", + // Where to store your index and index metadata + // The exact default location is dependent on your OS + data_directory: "/Users//Library/Application Support/com.athlabs.spyglass", + // By default, Spyglass will only crawl things as specified in your lenses. If you want + // to follow links without regard to those rules, set this to true. + crawl_external_links: false, ) ``` diff --git a/crates/client/src/components.rs b/crates/client/src/components.rs index 6e78dae8a..65c647f2e 100644 --- a/crates/client/src/components.rs +++ b/crates/client/src/components.rs @@ -73,7 +73,10 @@ pub fn search_result_component(res: &ResultListData, is_selected: bool) -> Html match res.result_type { ResultListType::DocSearch => { let url_link = if res.url.is_some() { - let domain = res.domain.clone().unwrap_or_else(||"example.com".to_string()); + let domain = res + .domain + .clone() + .unwrap_or_else(|| "example.com".to_string()); let url = res.url.clone().unwrap(); let path = url diff --git a/crates/entities/Cargo.toml b/crates/entities/Cargo.toml index 4f717a77e..b146b838e 100644 --- a/crates/entities/Cargo.toml +++ b/crates/entities/Cargo.toml @@ -9,7 +9,8 @@ edition = "2021" anyhow = "1.0" chrono = { version = "0.4", features = ["serde"] } log = "0.4" -sea-orm = { version = "^0", features = ["macros", "sqlx-sqlite", "runtime-tokio-rustls"], default-features = false } +regex = "1" +sea-orm = { version = "^0.8", features = ["macros", "sqlx-sqlite", "runtime-tokio-rustls"], default-features = false } serde = { version = "1.0", features = ["derive"] } shared = { path = "../shared" } tokio = { version = "1", features = ["full"] } diff --git a/crates/entities/src/lib.rs b/crates/entities/src/lib.rs index 5aa2a4494..8caa8f425 100644 --- a/crates/entities/src/lib.rs +++ b/crates/entities/src/lib.rs @@ -1,4 +1,5 @@ pub mod models; +pub mod regex; pub mod test; pub use sea_orm; diff --git a/crates/entities/src/models/crawl_queue.rs b/crates/entities/src/models/crawl_queue.rs index 42be19f60..c69f3fd1a 100644 --- a/crates/entities/src/models/crawl_queue.rs +++ b/crates/entities/src/models/crawl_queue.rs @@ -1,13 +1,15 @@ use std::collections::HashSet; use std::fmt; +use regex::RegexSet; use sea_orm::entity::prelude::*; use sea_orm::{sea_query, DbBackend, FromQueryResult, QuerySelect, Set, Statement}; use serde::Serialize; use url::Url; use super::indexed_document; -use shared::config::{Limit, UserSettings}; +use crate::regex::{regex_for_domain, regex_for_prefix}; +use shared::config::{Lens, Limit, UserSettings}; const MAX_RETRIES: u8 = 5; @@ -247,22 +249,38 @@ pub enum SkipReason { #[derive(Default)] pub struct EnqueueSettings { pub skip_blocklist: bool, + pub skip_lenses: bool, pub crawl_type: CrawlType, } pub async fn enqueue_all( db: &DatabaseConnection, urls: &[String], + lenses: &[Lens], settings: &UserSettings, overrides: &EnqueueSettings, ) -> anyhow::Result<(), sea_orm::DbErr> { + let mut allow_list: Vec = Vec::new(); + for lens in lenses { + // Build regex from domain + for domain in lens.domains.iter() { + allow_list.push(regex_for_domain(domain)); + } + + // Build regex from url rules + for prefix in lens.urls.iter() { + allow_list.push(regex_for_prefix(prefix)); + } + } + + let allow_list = RegexSet::new(allow_list).unwrap(); let block_list: HashSet = HashSet::from_iter(settings.block_list.iter().cloned()); // Ignore invalid URLs let urls: Vec = urls .iter() - .filter_map(|x| { - if let Ok(mut parsed) = Url::parse(x) { + .filter_map(|url| { + if let Ok(mut parsed) = Url::parse(url) { // Always ignore fragments, otherwise crawling // https://wikipedia.org/Rust#Blah would be considered different than // https://wikipedia.org/Rust @@ -270,12 +288,23 @@ pub async fn enqueue_all( // Ignore URLs w/ no domain/host strings let domain = parsed.host_str()?; + let normalized = parsed.to_string(); // Ignore domains on blacklist if !overrides.skip_blocklist && block_list.contains(&domain.to_string()) { return None; } + // Check lense rules? + if !overrides.skip_lenses + // Should we crawl external links? + && !settings.crawl_external_links + // Only allow crawls specified in our lenses + && !allow_list.is_match(&normalized) + { + return None; + } + Some(parsed.as_str().to_string()) } else { None @@ -361,7 +390,7 @@ mod test { use crate::test::setup_test_db; use shared::config::{Limit, UserSettings}; - use super::{gen_priority_sql, gen_priority_values}; + use super::{gen_priority_sql, gen_priority_values, EnqueueSettings}; #[tokio::test] async fn test_insert() { @@ -397,7 +426,7 @@ mod test { let sql = gen_priority_sql(&p_domains, &p_prefixes, settings); assert_eq!( sql.to_string(), - "WITH\n p_domain(domain, priority) AS (values (\"en.wikipedia.org\", 1)),\n p_prefix(prefix, priority) AS (values (\"https://roll20.net/compendium/dnd5e%\", 1)), indexed AS (\n SELECT\n domain,\n count(*) as count\n FROM indexed_document\n GROUP BY domain\n),\ninflight AS (\n SELECT\n domain,\n count(*) as count\n FROM crawl_queue\n WHERE status = \"Processing\"\n GROUP BY domain\n)\nSELECT\n cq.*\nFROM crawl_queue cq\nLEFT JOIN p_domain ON cq.domain like p_domain.domain\nLEFT JOIN p_prefix ON cq.url like p_prefix.prefix\nLEFT JOIN indexed ON indexed.domain = cq.domain\nLEFT JOIN inflight ON inflight.domain = cq.domain\nWHERE\n COALESCE(indexed.count, 0) < 1000 AND\n COALESCE(inflight.count, 0) < 2 AND\n status = \"Queued\"\nORDER BY\n p_prefix.priority DESC,\n p_domain.priority DESC,\n cq.updated_at ASC" + "WITH\n p_domain(domain, priority) AS (values (\"en.wikipedia.org\", 1)),\n p_prefix(prefix, priority) AS (values (\"https://roll20.net/compendium/dnd5e%\", 1)), indexed AS (\n SELECT\n domain,\n count(*) as count\n FROM indexed_document\n GROUP BY domain\n),\ninflight AS (\n SELECT\n domain,\n count(*) as count\n FROM crawl_queue\n WHERE status = \"Processing\"\n GROUP BY domain\n)\nSELECT\n cq.*\nFROM crawl_queue cq\nLEFT JOIN p_domain ON cq.domain like p_domain.domain\nLEFT JOIN p_prefix ON cq.url like p_prefix.prefix\nLEFT JOIN indexed ON indexed.domain = cq.domain\nLEFT JOIN inflight ON inflight.domain = cq.domain\nWHERE\n COALESCE(indexed.count, 0) < 10000 AND\n COALESCE(inflight.count, 0) < 2 AND\n status = \"Queued\"\nORDER BY\n p_prefix.priority DESC,\n p_domain.priority DESC,\n cq.updated_at ASC" ); } @@ -406,7 +435,12 @@ mod test { let settings = UserSettings::default(); let db = setup_test_db().await; let url = vec!["https://oldschool.runescape.wiki/".into()]; - crawl_queue::enqueue_all(&db, &url, &settings, &Default::default()) + + let overrides = EnqueueSettings { + skip_lenses: true, + ..Default::default() + }; + crawl_queue::enqueue_all(&db, &url, &[], &settings, &overrides) .await .unwrap(); @@ -426,7 +460,12 @@ mod test { let url = vec!["https://oldschool.runescape.wiki/".into()]; let prioritized = vec![]; - crawl_queue::enqueue_all(&db, &url, &settings, &Default::default()) + let overrides = EnqueueSettings { + skip_lenses: true, + ..Default::default() + }; + + crawl_queue::enqueue_all(&db, &url, &[], &settings, &overrides) .await .unwrap(); @@ -448,8 +487,12 @@ mod test { let url: Vec = vec!["https://oldschool.runescape.wiki/".into()]; let parsed = Url::parse(&url[0]).unwrap(); let prioritized = vec![]; + let overrides = EnqueueSettings { + skip_lenses: true, + ..Default::default() + }; - crawl_queue::enqueue_all(&db, &url, &settings, &Default::default()) + crawl_queue::enqueue_all(&db, &url, &[], &settings, &overrides) .await .unwrap(); let doc = indexed_document::ActiveModel { diff --git a/crates/entities/src/models/mod.rs b/crates/entities/src/models/mod.rs index 5ece365b4..252ed488c 100644 --- a/crates/entities/src/models/mod.rs +++ b/crates/entities/src/models/mod.rs @@ -9,13 +9,16 @@ pub mod resource_rule; use shared::config::Config; -pub async fn create_connection(is_test: bool) -> anyhow::Result { +pub async fn create_connection( + config: &Config, + is_test: bool, +) -> anyhow::Result { let db_uri: String = if is_test { "sqlite::memory:".to_string() } else { format!( "sqlite://{}?mode=rwc", - Config::data_dir().join("db.sqlite").to_str().unwrap() + config.data_dir().join("db.sqlite").to_str().unwrap() ) }; @@ -30,10 +33,12 @@ pub async fn create_connection(is_test: bool) -> anyhow::Result String { + let mut regex = String::new(); + for ch in domain.chars() { + match ch { + '*' => regex.push_str(".*"), + _ => regex.push_str(®ex::escape(&ch.to_string())), + } + } + + format!("(http://|https://){}.*", regex) +} + +pub fn regex_for_prefix(prefix: &str) -> String { + format!("{}.*", prefix) +} + +/// Convert a robots.txt rule into a proper regex string +pub fn regex_for_robots(rule: &str) -> Option { + if rule.is_empty() { + return None; + } + + let mut regex = String::new(); + let mut has_end = false; + for ch in rule.chars() { + match ch { + '*' => regex.push_str(".*"), + '^' => { + regex.push('^'); + has_end = true; + } + _ => regex.push_str(®ex::escape(&ch.to_string())), + } + } + + if !has_end { + regex.push_str(".*"); + } + + Some(regex) +} + +#[cfg(test)] +mod test { + use super::{regex_for_domain, regex_for_prefix}; + use regex::Regex; + + #[test] + fn test_regex_for_domain() { + // Baseline check + let regex = Regex::new(®ex_for_domain("en.wikipedia.org")).unwrap(); + assert!(regex.is_match("https://en.wikipedia.org/wiki/Rust")); + + // Should match http OR https + let regex = Regex::new(®ex_for_domain("en.wikipedia.org")).unwrap(); + assert!(regex.is_match("http://en.wikipedia.org/wiki/Rust")); + + // Wildcard should match anything + let regex = Regex::new(®ex_for_domain("*.wikipedia.org")).unwrap(); + for test in [ + "https://en.wikipedia.org/wiki/Rust", + "http://sub.sub.wikipedia.org/wiki/blah", + ] { + assert!(regex.is_match(test)); + } + } + + #[test] + fn test_regex_for_prefix() { + let prefix = "https://roll20.net/compendium/dnd5e"; + let regex = Regex::new(®ex_for_prefix(prefix)).unwrap(); + // Successes + for test in [ + "https://roll20.net/compendium/dnd5e", + "https://roll20.net/compendium/dnd5e/monsters", + "https://roll20.net/compendium/dnd5e.html", + ] { + assert!(regex.is_match(test)); + } + + // Failures + for test in [ + "https://sub.roll20.net", + "https://en.wikipedia.org", + "https://localhost", + ] { + assert!(!regex.is_match(test)); + } + } +} diff --git a/crates/entities/src/test.rs b/crates/entities/src/test.rs index b6f683db2..ca37e1249 100644 --- a/crates/entities/src/test.rs +++ b/crates/entities/src/test.rs @@ -1,4 +1,5 @@ use sea_orm::{ConnectionTrait, DatabaseConnection, Schema}; +use shared::config::Config; use crate::models::{ crawl_queue, create_connection, fetch_history, indexed_document, lens, link, resource_rule, @@ -6,7 +7,8 @@ use crate::models::{ #[allow(dead_code)] pub async fn setup_test_db() -> DatabaseConnection { - let db = create_connection(true).await.unwrap(); + let config = Config::default(); + let db = create_connection(&config, true).await.unwrap(); setup_schema(&db).await.expect("Unable to create tables"); db diff --git a/crates/migrations/Cargo.toml b/crates/migrations/Cargo.toml index 8144d93cb..c7d66b7a5 100644 --- a/crates/migrations/Cargo.toml +++ b/crates/migrations/Cargo.toml @@ -9,5 +9,5 @@ name = "migration" path = "src/lib.rs" [dependencies] -sea-schema = { version = "^0.7.0", default-features = false, features = [ "migration", "debug-print" ] } +sea-orm-migration = { version = "^0" } entities = { path = "../entities" } \ No newline at end of file diff --git a/crates/migrations/src/lib.rs b/crates/migrations/src/lib.rs index 0418ce835..035479455 100644 --- a/crates/migrations/src/lib.rs +++ b/crates/migrations/src/lib.rs @@ -1,4 +1,4 @@ -pub use sea_schema::migration::prelude::*; +pub use sea_orm_migration::prelude::*; mod m20220505_000001_create_table; mod m20220508_000001_lens_and_crawl_queue_update; diff --git a/crates/migrations/src/m20220505_000001_create_table.rs b/crates/migrations/src/m20220505_000001_create_table.rs index 4e18b3a74..56b0e16f7 100644 --- a/crates/migrations/src/m20220505_000001_create_table.rs +++ b/crates/migrations/src/m20220505_000001_create_table.rs @@ -1,5 +1,5 @@ use entities::sea_orm::{ConnectionTrait, Statement}; -use sea_schema::migration::prelude::*; +use sea_orm_migration::prelude::*; pub struct Migration; diff --git a/crates/migrations/src/m20220508_000001_lens_and_crawl_queue_update.rs b/crates/migrations/src/m20220508_000001_lens_and_crawl_queue_update.rs index e1a6fe854..a5565c394 100644 --- a/crates/migrations/src/m20220508_000001_lens_and_crawl_queue_update.rs +++ b/crates/migrations/src/m20220508_000001_lens_and_crawl_queue_update.rs @@ -2,7 +2,7 @@ use entities::{ models::crawl_queue, sea_orm::{ConnectionTrait, Statement}, }; -use sea_schema::migration::prelude::*; +use sea_orm_migration::prelude::*; pub struct Migration; diff --git a/crates/reindexer/src/main.rs b/crates/reindexer/src/main.rs index d9a366274..b68bae01d 100644 --- a/crates/reindexer/src/main.rs +++ b/crates/reindexer/src/main.rs @@ -4,12 +4,14 @@ use entities::sea_orm::{ActiveModelTrait, EntityTrait, PaginatorTrait, QueryOrde use libspyglass::crawler::Crawler; use libspyglass::search::Searcher; use libspyglass::state::AppState; +use shared::config::{Config, Lens}; use url::Url; #[tokio::main] async fn main() -> Result<(), anyhow::Error> { + let config = Config::new(); // Load app configuration - let state = AppState::new().await; + let state = AppState::new(&config).await; let crawler = Crawler::new(); let fields = Searcher::doc_fields(); @@ -71,9 +73,15 @@ async fn main() -> Result<(), anyhow::Error> { // Update parsed links let to_add: Vec = scrape.links.into_iter().collect(); + let lenses: Vec = state + .lenses + .iter() + .map(|entry| entry.value().clone()) + .collect(); crawl_queue::enqueue_all( &state.db, &to_add, + &lenses, &state.user_settings, &Default::default(), ) diff --git a/crates/shared/src/config.rs b/crates/shared/src/config.rs index 522928d2a..d701eb0bd 100644 --- a/crates/shared/src/config.rs +++ b/crates/shared/src/config.rs @@ -75,11 +75,22 @@ pub struct UserSettings { pub allow_list: Vec, /// Domains explicitly blocked from crawling. pub block_list: Vec, + /// Search bar activation hot key #[serde(default = "UserSettings::default_shortcut")] + /// Directory for metadata & index pub shortcut: String, + #[serde(default = "UserSettings::default_data_dir")] + pub data_directory: PathBuf, + /// Should we crawl links that don't match our lens rules? + #[serde(default)] + pub crawl_external_links: bool, } impl UserSettings { + fn default_data_dir() -> PathBuf { + Config::default_data_dir() + } + fn default_shortcut() -> String { "CmdOrCtrl+Shift+/".to_string() } @@ -105,7 +116,7 @@ impl UserSettings { impl Default for UserSettings { fn default() -> Self { UserSettings { - domain_crawl_limit: Limit::Finite(1000), + domain_crawl_limit: Limit::Finite(10000), // 10 total crawlers at a time inflight_crawl_limit: Limit::Finite(10), // Limit to 2 crawlers for a domain @@ -114,7 +125,11 @@ impl Default for UserSettings { run_wizard: false, allow_list: Vec::new(), block_list: vec!["web.archive.org".to_string()], + // Activation shortcut shortcut: UserSettings::default_shortcut(), + // Where to store the metadata & index + data_directory: UserSettings::default_data_dir(), + crawl_external_links: false, } } } @@ -144,10 +159,8 @@ impl Config { } } - fn load_lenses() -> anyhow::Result> { - let mut lenses = HashMap::new(); - - let lense_dir = Self::lenses_dir(); + fn load_lenses(&mut self) -> anyhow::Result<()> { + let lense_dir = self.lenses_dir(); for entry in (fs::read_dir(lense_dir)?).flatten() { let path = entry.path(); if path.is_file() && path.extension().unwrap_or_default() == "ron" { @@ -156,14 +169,14 @@ impl Config { Err(err) => log::error!("Unable to load lens {:?}: {}", entry.path(), err), Ok(lens) => { log::info!("Loaded lens {}", lens.name); - lenses.insert(lens.name.clone(), lens); + self.lenses.insert(lens.name.clone(), lens); } } } } } - if lenses.is_empty() { + if self.lenses.is_empty() { // Create a default lens as an example. let lens = Lens { author: "Spyglass".to_string(), @@ -173,22 +186,22 @@ impl Config { "Search through official user-supported wikis for knowledge, games, and more." .to_string(), ), - domains: vec![ - "en.wikipedia.org".to_string(), - "oldschool.runescape.wiki".to_string(), - "wiki.factorio.com".to_string(), + domains: vec!["blog.rust-lang.org".into(), "wiki.factorio.com".into()], + urls: vec![ + "https://https://en.wikipedia.org/wiki/Portal:".into(), + "https://doc.rust-lang.org/book/".into(), + "https://oldschool.runescape.wiki/w/".into(), ], - urls: Vec::new(), }; fs::write( - Self::lenses_dir().join("wiki.ron"), + self.lenses_dir().join("wiki.ron"), ron::ser::to_string_pretty(&lens, Default::default()).unwrap(), ) .expect("Unable to save default lens file."); } - Ok(lenses) + Ok(()) } pub fn app_identifier() -> String { @@ -199,17 +212,25 @@ impl Config { } } - pub fn data_dir() -> PathBuf { + pub fn default_data_dir() -> PathBuf { let proj_dirs = ProjectDirs::from("com", "athlabs", &Config::app_identifier()).unwrap(); proj_dirs.data_dir().to_path_buf() } - pub fn index_dir() -> PathBuf { - Self::data_dir().join("index") + pub fn data_dir(&self) -> PathBuf { + if self.user_settings.data_directory != Self::default_data_dir() { + self.user_settings.data_directory.clone() + } else { + Self::default_data_dir() + } + } + + pub fn index_dir(&self) -> PathBuf { + self.data_dir().join("index") } - pub fn logs_dir() -> PathBuf { - Self::data_dir().join("logs") + pub fn logs_dir(&self) -> PathBuf { + self.data_dir().join("logs") } pub fn prefs_dir() -> PathBuf { @@ -223,52 +244,59 @@ impl Config { Self::prefs_dir().join("settings.ron") } - pub fn lenses_dir() -> PathBuf { - Self::data_dir().join("lenses") + pub fn lenses_dir(&self) -> PathBuf { + self.data_dir().join("lenses") } pub fn new() -> Self { - let data_dir = Config::data_dir(); - fs::create_dir_all(&data_dir).expect("Unable to create data folder"); - - let index_dir = Config::index_dir(); - fs::create_dir_all(&index_dir).expect("Unable to create index folder"); - - let logs_dir = Config::logs_dir(); - fs::create_dir_all(&logs_dir).expect("Unable to create logs folder"); - let prefs_dir = Config::prefs_dir(); fs::create_dir_all(&prefs_dir).expect("Unable to create config folder"); - let lenses_dir = Config::lenses_dir(); - fs::create_dir_all(&lenses_dir).expect("Unable to create `lenses` folder"); - // Gracefully handle issues loading user settings/lenses let user_settings = Self::load_user_settings().unwrap_or_else(|err| { log::warn!("Invalid user settings file! Reason: {}", err); Default::default() }); - let lenses = Self::load_lenses().unwrap_or_else(|err| { + let mut config = Config { + lenses: HashMap::new(), + user_settings, + }; + + let data_dir = config.data_dir(); + fs::create_dir_all(&data_dir).expect("Unable to create data folder"); + + let index_dir = config.index_dir(); + fs::create_dir_all(&index_dir).expect("Unable to create index folder"); + + let logs_dir = config.logs_dir(); + fs::create_dir_all(&logs_dir).expect("Unable to create logs folder"); + + let lenses_dir = config.lenses_dir(); + fs::create_dir_all(&lenses_dir).expect("Unable to create `lenses` folder"); + + config.load_lenses().unwrap_or_else(|err| { log::warn!("Unable to load lenses! Reason: {}", err); Default::default() }); - Config { - lenses, - user_settings, - } + config } } #[cfg(test)] mod test { use crate::config::Config; + use std::collections::HashMap; #[test] #[ignore] pub fn test_load_lenses() { - let res = Config::load_lenses(); + let mut config = Config { + lenses: HashMap::new(), + user_settings: Default::default(), + }; + let res = config.load_lenses(); assert!(!res.is_err()); } } diff --git a/crates/spyglass/src/crawler/bootstrap.rs b/crates/spyglass/src/crawler/bootstrap.rs index 3e35cabae..90c4cecd7 100644 --- a/crates/spyglass/src/crawler/bootstrap.rs +++ b/crates/spyglass/src/crawler/bootstrap.rs @@ -125,6 +125,7 @@ pub async fn bootstrap( let mut count: usize = 0; let overrides = crawl_queue::EnqueueSettings { skip_blocklist: true, + skip_lenses: true, crawl_type: crawl_queue::CrawlType::Bootstrap, }; @@ -143,7 +144,7 @@ pub async fn bootstrap( // Add URLs to crawl queue log::info!("enqueing {} urls", urls.len()); let urls: Vec = urls.into_iter().collect(); - crawl_queue::enqueue_all(db, &urls, settings, &overrides).await?; + crawl_queue::enqueue_all(db, &urls, &Vec::new(), settings, &overrides).await?; count += urls.len(); if resume.is_none() { diff --git a/crates/spyglass/src/crawler/robots.rs b/crates/spyglass/src/crawler/robots.rs index 26124b841..d2d931974 100644 --- a/crates/spyglass/src/crawler/robots.rs +++ b/crates/spyglass/src/crawler/robots.rs @@ -3,6 +3,7 @@ /// - https://developers.google.com/search/docs/advanced/robots/intro /// - https://www.robotstxt.org/robotstxt.html use entities::models::resource_rule; +use entities::regex::regex_for_robots; use entities::sea_orm::prelude::*; use entities::sea_orm::{DatabaseConnection, Set}; @@ -30,32 +31,6 @@ impl From for ParsedRule { const BOT_AGENT_NAME: &str = "spyglass"; -/// Convert a robots.txt rule into a proper regex string -fn rule_to_regex(rule: &str) -> Option { - if rule.is_empty() { - return None; - } - - let mut regex = String::new(); - let mut has_end = false; - for ch in rule.chars() { - match ch { - '*' => regex.push_str(".*"), - '^' => { - regex.push('^'); - has_end = true; - } - _ => regex.push_str(®ex::escape(&ch.to_string())), - } - } - - if !has_end { - regex.push_str(".*"); - } - - Some(regex) -} - /// Convert a set of rules into a regex set for matching pub fn filter_set(rules: &[ParsedRule], allow: bool) -> RegexSet { let rules: Vec = rules @@ -93,7 +68,7 @@ pub fn parse(domain: &str, txt: &str) -> Vec { } if prefix.starts_with("disallow") || prefix.starts_with("allow") { - let regex = rule_to_regex(end.trim()); + let regex = regex_for_robots(end.trim()); if let Some(regex) = regex { rules.push(ParsedRule { domain: domain.to_string(), @@ -104,7 +79,7 @@ pub fn parse(domain: &str, txt: &str) -> Vec { } else if regex.is_none() && prefix.starts_with("disallow") { rules.push(ParsedRule { domain: domain.to_string(), - regex: rule_to_regex("/").unwrap(), + regex: regex_for_robots("/").unwrap(), allow_crawl: true, }); } @@ -210,10 +185,11 @@ pub async fn check_resource_rules( #[cfg(test)] mod test { - use super::{check_resource_rules, filter_set, parse, rule_to_regex, ParsedRule}; + use super::{check_resource_rules, filter_set, parse, ParsedRule}; use crate::crawler::Crawler; use entities::models::resource_rule; + use entities::regex::regex_for_robots; use entities::sea_orm::{ActiveModelTrait, Set}; use entities::test::setup_test_db; use regex::Regex; @@ -244,7 +220,7 @@ mod test { #[test] fn test_rule_to_regex() { - let regex = rule_to_regex("/*?title=Property:").unwrap(); + let regex = regex_for_robots("/*?title=Property:").unwrap(); assert_eq!(regex, "/.*\\?title=Property:.*"); let re = Regex::new(®ex).unwrap(); diff --git a/crates/spyglass/src/importer.rs b/crates/spyglass/src/importer.rs index 083a34c80..db3fcafa6 100644 --- a/crates/spyglass/src/importer.rs +++ b/crates/spyglass/src/importer.rs @@ -5,6 +5,7 @@ use std::{env, fs, path::PathBuf}; use entities::models::crawl_queue; use libspyglass::state::AppState; +use shared::config::Lens; #[allow(dead_code)] pub struct FirefoxImporter { @@ -35,7 +36,7 @@ impl FirefoxImporter { profile_path = Some(path); } - let imported_path = Config::data_dir().join("firefox.sqlite"); + let imported_path = config.data_dir().join("firefox.sqlite"); FirefoxImporter { profile_path, imported_path, @@ -83,10 +84,11 @@ impl FirefoxImporter { let mut count = 0; let to_add: Vec = rows.into_iter().map(|(_, x)| x).collect(); - + let lenses: Vec = self.config.lenses.clone().into_values().collect(); crawl_queue::enqueue_all( &state.db, &to_add, + &lenses, &self.config.user_settings, &Default::default(), ) diff --git a/crates/spyglass/src/main.rs b/crates/spyglass/src/main.rs index 502a4d953..d9f3e8e6f 100644 --- a/crates/spyglass/src/main.rs +++ b/crates/spyglass/src/main.rs @@ -1,4 +1,5 @@ use std::io; +use std::time::Duration; use tokio::signal; use tokio::sync::{broadcast, mpsc}; use tracing_log::LogTracer; @@ -17,7 +18,9 @@ mod importer; use crate::api::start_api_ipc; fn main() -> Result<(), Box> { - let file_appender = tracing_appender::rolling::daily(Config::logs_dir(), "server.log"); + let config = Config::new(); + + let file_appender = tracing_appender::rolling::daily(config.logs_dir(), "server.log"); let (non_blocking, _guard) = tracing_appender::non_blocking(file_appender); let subscriber = tracing_subscriber::registry() @@ -39,7 +42,7 @@ fn main() -> Result<(), Box> { .unwrap(); // Initialize/Load user preferences - let state = rt.block_on(AppState::new()); + let state = rt.block_on(AppState::new(&config)); // Run any migrations match rt.block_on(Migrator::up(&state.db, None)) { @@ -152,6 +155,21 @@ async fn start_backend(state: &AppState) { shutdown_tx.subscribe(), )); + // Clean up crew. Commit anything added to the index in the last 10s + { + let state = state.clone(); + let _ = tokio::spawn(async move { + let mut interval = tokio::time::interval(Duration::from_secs(10)); + + loop { + interval.tick().await; + if let Err(err) = state.index.writer.lock().unwrap().commit() { + log::error!("{:?}", err); + } + } + }); + } + // Gracefully handle shutdowns match signal::ctrl_c().await { Ok(()) => { diff --git a/crates/spyglass/src/search/mod.rs b/crates/spyglass/src/search/mod.rs index da94f0d1a..69bb032c9 100644 --- a/crates/spyglass/src/search/mod.rs +++ b/crates/spyglass/src/search/mod.rs @@ -176,8 +176,6 @@ impl Searcher { doc.add_text(fields.url, url); writer.add_document(doc)?; - writer.commit()?; - Ok(doc_id) } @@ -306,8 +304,13 @@ mod test { ) .expect("Unable to add doc"); + let res = writer.commit(); + if let Err(err) = res { + println!("{:?}", err); + } + // add a small delay so that the documents can be properly committed - std::thread::sleep(std::time::Duration::from_millis(100)); + std::thread::sleep(std::time::Duration::from_millis(1000)); } #[test] diff --git a/crates/spyglass/src/state.rs b/crates/spyglass/src/state.rs index b3240ecf7..01b90491d 100644 --- a/crates/spyglass/src/state.rs +++ b/crates/spyglass/src/state.rs @@ -16,14 +16,12 @@ pub struct AppState { } impl AppState { - pub async fn new() -> Self { - let config = Config::new(); - - let db = create_connection(false) + pub async fn new(config: &Config) -> Self { + let db = create_connection(config, false) .await .expect("Unable to connect to database"); - let index = Searcher::with_index(&IndexPath::LocalPath(Config::index_dir())); + let index = Searcher::with_index(&IndexPath::LocalPath(config.index_dir())); // TODO: Load from saved preferences let app_state = DashMap::new(); @@ -31,14 +29,14 @@ impl AppState { // Convert into dashmap let lenses = DashMap::new(); - for (key, value) in config.lenses.into_iter() { - lenses.insert(key, value); + for (key, value) in config.lenses.iter() { + lenses.insert(key.clone(), value.clone()); } AppState { db, app_state: Arc::new(app_state), - user_settings: config.user_settings, + user_settings: config.user_settings.clone(), lenses: Arc::new(lenses), index, } diff --git a/crates/spyglass/src/task.rs b/crates/spyglass/src/task.rs index 7704f39a3..4c5e1be59 100644 --- a/crates/spyglass/src/task.rs +++ b/crates/spyglass/src/task.rs @@ -6,7 +6,9 @@ use url::Url; use crate::crawler::Crawler; use crate::search::Searcher; use crate::state::AppState; + use entities::models::{crawl_queue, indexed_document}; +use shared::config::Lens; #[derive(Debug, Clone)] pub struct CrawlTask { @@ -50,26 +52,31 @@ pub async fn manager_task( state.user_settings.clone(), &prioritized_domains, &prioritized_prefixes, - ) => res.unwrap(), + ) => res, _ = shutdown_rx.recv() => { log::info!("🛑 Shutting down manager"); return; } }; - if let Some(task) = next_url { - // Mark in progress - let task_id = task.id; - let mut update: crawl_queue::ActiveModel = task.into(); - update.status = Set(crawl_queue::CrawlStatus::Processing); - update.update(&state.db).await.unwrap(); - - // Send to worker - let cmd = Command::Fetch(CrawlTask { id: task_id }); - if queue.send(cmd).await.is_err() { - eprintln!("unable to send command to worker"); - return; + match next_url { + Err(err) => log::error!("Unable to dequeue: {}", err), + Ok(Some(task)) => { + // Mark in progress + let task_id = task.id; + let mut update: crawl_queue::ActiveModel = task.into(); + update.status = Set(crawl_queue::CrawlStatus::Processing); + let _ = update.update(&state.db).await; + + // Send to worker + let cmd = Command::Fetch(CrawlTask { id: task_id }); + if queue.send(cmd).await.is_err() { + eprintln!("unable to send command to worker"); + return; + } } + // ignore everything else + _ => {} } } } @@ -89,15 +96,19 @@ async fn _handle_fetch(state: AppState, crawler: Crawler, task: CrawlTask) { crawl_queue::CrawlStatus::Failed }; - crawl_queue::mark_done(&state.db, task.id, cq_status) - .await - .unwrap(); + let _ = crawl_queue::mark_done(&state.db, task.id, cq_status).await; // Add all valid, non-duplicate, non-indexed links found to crawl queue let to_enqueue: Vec = crawl_result.links.into_iter().collect(); + let lenses: Vec = state + .lenses + .iter() + .map(|entry| entry.value().clone()) + .collect(); if let Err(err) = crawl_queue::enqueue_all( &state.db, &to_enqueue, + &lenses, &state.user_settings, &Default::default(), ) diff --git a/crates/tauri/src/cmd.rs b/crates/tauri/src/cmd.rs index 302f5c1b9..9bd321ae0 100644 --- a/crates/tauri/src/cmd.rs +++ b/crates/tauri/src/cmd.rs @@ -23,7 +23,7 @@ pub async fn resize_window(window: tauri::Window, height: f64) { #[tauri::command] pub async fn search_docs<'r>( _: tauri::Window, - rpc: State<'r, rpc::RpcClient>, + rpc: State<'r, rpc::RpcMutex>, lenses: Vec, query: &str, ) -> Result, String> { @@ -32,6 +32,7 @@ pub async fn search_docs<'r>( query: query.to_string(), }; + let rpc = rpc.lock().await; match rpc .client .call_method::<(request::SearchParam,), response::SearchResults>("search_docs", "", (data,)) @@ -48,13 +49,14 @@ pub async fn search_docs<'r>( #[tauri::command] pub async fn search_lenses<'r>( _: tauri::Window, - rpc: State<'r, rpc::RpcClient>, + rpc: State<'r, rpc::RpcMutex>, query: &str, ) -> Result, String> { let data = request::SearchLensesParam { query: query.to_string(), }; + let rpc = rpc.lock().await; match rpc .client .call_method::<(request::SearchLensesParam,), response::SearchLensesResp>( diff --git a/crates/tauri/src/main.rs b/crates/tauri/src/main.rs index 74baf3a4c..81a8df819 100644 --- a/crates/tauri/src/main.rs +++ b/crates/tauri/src/main.rs @@ -2,19 +2,24 @@ all(not(debug_assertions), target_os = "windows"), windows_subsystem = "windows" )] + use std::io; use std::path::PathBuf; +use std::sync::Arc; use std::time::Duration; use jsonrpc_core::Value; use num_format::{Locale, ToFormattedString}; +use rpc::RpcMutex; use tauri::{AppHandle, GlobalShortcutManager, Manager, SystemTray, SystemTrayEvent}; +use tokio::sync::Mutex; use tokio::time; use tracing_log::LogTracer; use tracing_subscriber::{fmt, layer::SubscriberExt, EnvFilter}; use shared::config::Config; use shared::response; +use shared::response::AppStatus; mod cmd; mod constants; @@ -23,7 +28,9 @@ mod rpc; mod window; fn main() -> Result<(), Box> { - let file_appender = tracing_appender::rolling::daily(Config::logs_dir(), "client.log"); + let config = Config::new(); + + let file_appender = tracing_appender::rolling::daily(config.logs_dir(), "client.log"); let (non_blocking, _guard) = tracing_appender::non_blocking(file_appender); let subscriber = tracing_subscriber::registry() @@ -39,7 +46,6 @@ fn main() -> Result<(), Box> { LogTracer::init()?; let ctx = tauri::generate_context!(); - let config = Config::new(); tauri::Builder::default() .invoke_handler(tauri::generate_handler![ @@ -64,7 +70,8 @@ fn main() -> Result<(), Box> { let _ = window.set_skip_taskbar(true); // Wait for the server to boot up - app.manage(tauri::async_runtime::block_on(rpc::RpcClient::new())); + let rpc = tauri::async_runtime::block_on(rpc::RpcClient::new()); + app.manage(Arc::new(Mutex::new(rpc))); // Load user settings app.manage(config.clone()); @@ -113,13 +120,13 @@ fn main() -> Result<(), Box> { }) .on_system_tray_event(|app, event| { if let SystemTrayEvent::MenuItemClick { id, .. } = event { + let config = app.state::(); let item_handle = app.tray_handle().get_item(&id); let window = app.get_window("main").unwrap(); match id.as_str() { menu::CRAWL_STATUS_MENU_ITEM => { - let rpc = app.state::().inner(); - + let rpc = app.state::().inner(); let is_paused = tauri::async_runtime::block_on(pause_crawler(rpc)); let new_label = if is_paused { "▶️ Resume indexing" @@ -129,8 +136,8 @@ fn main() -> Result<(), Box> { item_handle.set_title(new_label).unwrap(); } - menu::OPEN_LENSES_FOLDER => open_folder(Config::lenses_dir()), - menu::OPEN_LOGS_FOLDER => open_folder(Config::logs_dir()), + menu::OPEN_LENSES_FOLDER => open_folder(config.lenses_dir()), + menu::OPEN_LOGS_FOLDER => open_folder(config.logs_dir()), menu::OPEN_SETTINGS_FOLDER => open_folder(Config::prefs_dir()), menu::SHOW_SEARCHBAR => { if !window.is_visible().unwrap() { @@ -150,7 +157,8 @@ fn main() -> Result<(), Box> { Ok(()) } -async fn app_status(rpc: &rpc::RpcClient) -> Option { +async fn app_status(rpc: &rpc::RpcMutex) -> Option { + let mut rpc = rpc.lock().await; match rpc .client .call_method::("app_stats", "", Value::Null) @@ -158,13 +166,15 @@ async fn app_status(rpc: &rpc::RpcClient) -> Option { { Ok(resp) => Some(resp), Err(err) => { - log::error!("{}", err); + log::error!("Error sending RPC: {}", err); + rpc.reconnect().await; None } } } -async fn pause_crawler(rpc: &rpc::RpcClient) -> bool { +async fn pause_crawler(rpc: &rpc::RpcMutex) -> bool { + let mut rpc = rpc.lock().await; match rpc .client .call_method::("toggle_pause", "", Value::Null) @@ -172,7 +182,8 @@ async fn pause_crawler(rpc: &rpc::RpcClient) -> bool { { Ok(resp) => resp.is_paused, Err(err) => { - log::error!("{}", err); + log::error!("Error sending RPC: {}", err); + rpc.reconnect().await; false } } @@ -199,9 +210,8 @@ fn open_folder(folder: PathBuf) { } async fn update_tray_menu(app: &AppHandle) { - let rpc = app.state::().inner(); - - let app_status = app_status(rpc).await; + let rpc = app.state::().inner(); + let app_status: Option = app_status(rpc).await; let handle = app.tray_handle(); if let Some(app_status) = app_status { diff --git a/crates/tauri/src/rpc.rs b/crates/tauri/src/rpc.rs index 327221df7..ee09c19b4 100644 --- a/crates/tauri/src/rpc.rs +++ b/crates/tauri/src/rpc.rs @@ -1,45 +1,75 @@ +use std::sync::Arc; + use jsonrpc_core_client::{transports::ipc, TypedClient}; use shared::rpc::gen_ipc_path; -use tauri::api::process::Command; +use tauri::api::process::{Command, CommandEvent}; +use tokio::sync::Mutex; use tokio_retry::strategy::{jitter, ExponentialBackoff}; use tokio_retry::Retry; +pub type RpcMutex = Arc>; + pub struct RpcClient { pub client: TypedClient, pub endpoint: String, } -#[allow(dead_code)] pub fn check_and_start_backend() { - let _ = Command::new_sidecar("spyglass-server") - .expect("failed to create `spyglass-server` binary command") - .spawn() - .expect("Failed to spawn sidecar"); + tauri::async_runtime::spawn(async move { + let (mut rx, _) = Command::new_sidecar("spyglass-server") + .expect("failed to create `spyglass-server` binary command") + .spawn() + .expect("Failed to spawn sidecar"); + + while let Some(event) = rx.recv().await { + match event { + CommandEvent::Error(line) => log::error!("sidecar error: {}", line), + CommandEvent::Terminated(payload) => { + log::error!("sidecar terminated: {:?}", payload) + } + _ => {} + } + } + }); } -async fn connect(endpoint: String) -> Result { - if let Ok(client) = ipc::connect(endpoint.clone()).await { +async fn connect(endpoint: &str) -> Result { + if let Ok(client) = ipc::connect(endpoint).await { return Ok(client); } Err(()) } +async fn try_connect(endpoint: &str) -> Result { + let retry_strategy = ExponentialBackoff::from_millis(10) + .map(jitter) // add jitter to delays + .take(10); + + Retry::spawn(retry_strategy, || connect(endpoint)).await +} + impl RpcClient { pub async fn new() -> Self { let endpoint = gen_ipc_path(); - let retry_strategy = ExponentialBackoff::from_millis(10) - .map(jitter) // add jitter to delays - .take(10); - - let client: TypedClient = Retry::spawn(retry_strategy, || connect(endpoint.clone())) + let client = try_connect(&endpoint) .await - .unwrap(); + .expect("Unable to connect to spyglass backend!"); RpcClient { client, endpoint: endpoint.clone(), } } + + pub async fn reconnect(&mut self) { + log::info!("Attempting to restart backend"); + // Attempt to reconnect + check_and_start_backend(); + self.client = try_connect(&self.endpoint) + .await + .expect("Unable to connect to spyglass backend!"); + log::info!("restarted"); + } } diff --git a/crates/tauri/tauri.conf.json b/crates/tauri/tauri.conf.json index 41646f531..f2ad49a1c 100644 --- a/crates/tauri/tauri.conf.json +++ b/crates/tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "package": { "productName": "Spyglass", - "version": "22.5.11" + "version": "22.5.15" }, "build": { "distDir": "../client/dist", diff --git a/docs/menubar-menu.png b/docs/menubar-menu.png new file mode 100644 index 000000000..66bdb3b44 Binary files /dev/null and b/docs/menubar-menu.png differ