From 73f8cb3d49f52f02de7649c8f39caf47db901ecc Mon Sep 17 00:00:00 2001 From: Andrew Huynh Date: Tue, 8 Nov 2022 22:04:39 -0800 Subject: [PATCH 1/7] chore: bump versions for v2022.11.3 release --- README.md | 6 +++--- VERSION.json | 22 +++++++++++----------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index cca8eb7f5..e8d32e9d3 100644 --- a/README.md +++ b/README.md @@ -5,13 +5,13 @@

Download now: - + macOS | - + Windows | - + Linux (AppImage)
diff --git a/VERSION.json b/VERSION.json index edefb7228..b88e108cf 100644 --- a/VERSION.json +++ b/VERSION.json @@ -1,23 +1,23 @@ { - "version": "22.11.2", - "notes": "See full release notes here: https://github.com/a5huynh/spyglass/releases/tag/v2022.11.2", - "pub_date": "2022-11-08T21:19:50Z", + "version": "22.11.3", + "notes": "See full release notes here: https://github.com/a5huynh/spyglass/releases/tag/v2022.11.3", + "pub_date": "2022-11-09T06:02:07Z", "platforms": { "darwin-x86_64": { - "signature": "dW50cnVzdGVkIGNvbW1lbnQ6IHNpZ25hdHVyZSBmcm9tIHRhdXJpIHNlY3JldCBrZXkKUlVSRk9WYXdDUXphYmt1TFBFSWRNVFc4VmFna29UWCtNU1Bhd0FKbGd2dDlEcUEwNjZsT2ZmOWR6S1MxdHViQ1NIRFZEa1ZUUXJhUk9UM08xczJBZFNkdGhsNnRqbGtkZVFJPQp0cnVzdGVkIGNvbW1lbnQ6IHRpbWVzdGFtcDoxNjY3OTQxNjM0CWZpbGU6U3B5Z2xhc3MuYXBwLnRhci5negpTN3cxb1pucVZOUWcxeTBqdHJoMlhiNTkxOXh3QnV6WG1zSEZ3RGZrb1JPejRUU1FWQ1BLUUtVdHhFc2MvajgzSTFwa3ZKVzhST2RjNkZweW00ZEhEdz09Cg==", - "url": "https://github.com/a5huynh/spyglass/releases/download/v2022.11.2/Spyglass.app.tar.gz" + "signature": "dW50cnVzdGVkIGNvbW1lbnQ6IHNpZ25hdHVyZSBmcm9tIHRhdXJpIHNlY3JldCBrZXkKUlVSRk9WYXdDUXphYmxjRWRDcGFlbFBpcnRHNXpkbStENUx4Smc1bDloVWFYQ2wrQzlvZGZrOXgxaTVEZWprN2E5L0drT2VJZTl5clZTamtlQW9TcUlQNDZ1VFh4L3dLZ0FZPQp0cnVzdGVkIGNvbW1lbnQ6IHRpbWVzdGFtcDoxNjY3OTczMTQ1CWZpbGU6U3B5Z2xhc3MuYXBwLnRhci5negpsRW9TSHY2WnR1OXNHb1ZYMVFsUi9JWU9XbXN1c3pJaFVqM3BPNVd0SHhleHo2YktRUWZ1Rk1pa3ZtUGRLc1pManAxaVdSc2xVOWJmYjErWE52R3JEZz09Cg==", + "url": "https://github.com/a5huynh/spyglass/releases/download/v2022.11.3/Spyglass.app.tar.gz" }, "darwin-aarch64": { - "signature": "dW50cnVzdGVkIGNvbW1lbnQ6IHNpZ25hdHVyZSBmcm9tIHRhdXJpIHNlY3JldCBrZXkKUlVSRk9WYXdDUXphYmt1TFBFSWRNVFc4VmFna29UWCtNU1Bhd0FKbGd2dDlEcUEwNjZsT2ZmOWR6S1MxdHViQ1NIRFZEa1ZUUXJhUk9UM08xczJBZFNkdGhsNnRqbGtkZVFJPQp0cnVzdGVkIGNvbW1lbnQ6IHRpbWVzdGFtcDoxNjY3OTQxNjM0CWZpbGU6U3B5Z2xhc3MuYXBwLnRhci5negpTN3cxb1pucVZOUWcxeTBqdHJoMlhiNTkxOXh3QnV6WG1zSEZ3RGZrb1JPejRUU1FWQ1BLUUtVdHhFc2MvajgzSTFwa3ZKVzhST2RjNkZweW00ZEhEdz09Cg==", - "url": "https://github.com/a5huynh/spyglass/releases/download/v2022.11.2/Spyglass.app.tar.gz" + "signature": "dW50cnVzdGVkIGNvbW1lbnQ6IHNpZ25hdHVyZSBmcm9tIHRhdXJpIHNlY3JldCBrZXkKUlVSRk9WYXdDUXphYmxjRWRDcGFlbFBpcnRHNXpkbStENUx4Smc1bDloVWFYQ2wrQzlvZGZrOXgxaTVEZWprN2E5L0drT2VJZTl5clZTamtlQW9TcUlQNDZ1VFh4L3dLZ0FZPQp0cnVzdGVkIGNvbW1lbnQ6IHRpbWVzdGFtcDoxNjY3OTczMTQ1CWZpbGU6U3B5Z2xhc3MuYXBwLnRhci5negpsRW9TSHY2WnR1OXNHb1ZYMVFsUi9JWU9XbXN1c3pJaFVqM3BPNVd0SHhleHo2YktRUWZ1Rk1pa3ZtUGRLc1pManAxaVdSc2xVOWJmYjErWE52R3JEZz09Cg==", + "url": "https://github.com/a5huynh/spyglass/releases/download/v2022.11.3/Spyglass.app.tar.gz" }, "linux-x86_64": { - "signature": "dW50cnVzdGVkIGNvbW1lbnQ6IHNpZ25hdHVyZSBmcm9tIHRhdXJpIHNlY3JldCBrZXkKUlVSRk9WYXdDUXphYm96ZlYxVjNxR09ZMGJwR0NWMElkM2Nvd2UwbWxZc1F3WHhXTzN0QjdoV0VjSkhhRWljd1hERXdFN0dOQXdHdmZIckxFV2VFTzJYVy9MVVI0b015Wmc4PQp0cnVzdGVkIGNvbW1lbnQ6IHRpbWVzdGFtcDoxNjY3OTM5ODE2CWZpbGU6c3B5Z2xhc3NfMjIuMTEuMl9hbWQ2NC5BcHBJbWFnZS50YXIuZ3oKdm0wV0NYekIvemdablhSNERkdTJKKzNUTUNLdjVvSHhsdDhoTXNIWjFIWVNKTW1wQzZ0Q3lTbkxGanE4VTFCZHZZVFpKcmFGbUVwVmtBOXR0VzJWQXc9PQo=", - "url": "https://github.com/a5huynh/spyglass/releases/download/v2022.11.2/spyglass_22.11.2_amd64.AppImage.tar.gz" + "signature": "dW50cnVzdGVkIGNvbW1lbnQ6IHNpZ25hdHVyZSBmcm9tIHRhdXJpIHNlY3JldCBrZXkKUlVSRk9WYXdDUXphYnVXVXpHS3ZCUWROK3d3WlNSN1BLdjdOZkJMdUViZ3puODlmdnBFVHpJczRxUG9UYjdsZjhTNkQ4dnJuN1FNSFlZdkxPMHpBblpLT3kxZ0JvaE9VK1E0PQp0cnVzdGVkIGNvbW1lbnQ6IHRpbWVzdGFtcDoxNjY3OTcxODg3CWZpbGU6c3B5Z2xhc3NfMjIuMTEuM19hbWQ2NC5BcHBJbWFnZS50YXIuZ3oKSHcvTS9OdC9NSzdoM1hueHVNcHV3Wk0vWGNCVEY2QXpqRm5WQWUxR05Hc204RUNvc0FpcitPTkRQNVlDUjdTdk5mMVUzTGtVOThFWnZ1RW1JNmxDREE9PQo=", + "url": "https://github.com/a5huynh/spyglass/releases/download/v2022.11.3/spyglass_22.11.3_amd64.AppImage.tar.gz" }, "windows-x86_64": { - "signature": "dW50cnVzdGVkIGNvbW1lbnQ6IHNpZ25hdHVyZSBmcm9tIHRhdXJpIHNlY3JldCBrZXkKUlVSRk9WYXdDUXphYnJPaTY2ZG01TlBkSHlZNXNvZXNaUDMwanlXUDZNYjc2MnRZK0phditNVmNLU0RXdWswaEdUS3ozbFM1UVMxODlrYlNONVBPM0w2Z3pZZkw5SEZSc2dFPQp0cnVzdGVkIGNvbW1lbnQ6IHRpbWVzdGFtcDoxNjY3OTQwNDg1CWZpbGU6U3B5Z2xhc3NfMjIuMTEuMl94NjRfZW4tVVMubXNpLnppcAplVGRuYzhuYkxOcUV1RlhIQ21BQjlqNjhvQmJocC9ibDNBZzc0UmNHTDBPY3Fyc1FkL1dvRVFVS0pXVVZLNU0rV25IenZIaVFxcC9PakwxTUFRbWZCQT09Cg==", - "url": "https://github.com/a5huynh/spyglass/releases/download/v2022.11.2/Spyglass_22.11.2_x64_en-US.msi.zip" + "signature": "dW50cnVzdGVkIGNvbW1lbnQ6IHNpZ25hdHVyZSBmcm9tIHRhdXJpIHNlY3JldCBrZXkKUlVSRk9WYXdDUXphYmhiZ1VKUHh4R2Vadm5aVWFJL2lZcVoxVllqd090cWc5M3F0SllMRmJ4bE9jajZNRlMyWGJsWENxQmFmLzMrQmNqdTNXUHNQWjEveVdXTFdnaWF0cXdZPQp0cnVzdGVkIGNvbW1lbnQ6IHRpbWVzdGFtcDoxNjY3OTczNTIyCWZpbGU6U3B5Z2xhc3NfMjIuMTEuM194NjRfZW4tVVMubXNpLnppcApWWkRJZEpWRUNSUWx6Wldsd0xCMVpQRjZTUmc2Z3dRSHQvempuZkhBVmdlVTBxdDhvcTJlRzRSazd4b3ZJemFlM2lZVXZBaVlDY211ekQvNkkvMldDZz09Cg==", + "url": "https://github.com/a5huynh/spyglass/releases/download/v2022.11.3/Spyglass_22.11.3_x64_en-US.msi.zip" } } } \ No newline at end of file From 47b18732768fef8d38fd9ef2d9ec4d9850017d05 Mon Sep 17 00:00:00 2001 From: Andrew Huynh Date: Wed, 9 Nov 2022 09:33:03 -0800 Subject: [PATCH 2/7] tweak to remove applied lens when we clear search results --- crates/client/src/pages/search.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/client/src/pages/search.rs b/crates/client/src/pages/search.rs index 4d4fbf30e..6282c7fcc 100644 --- a/crates/client/src/pages/search.rs +++ b/crates/client/src/pages/search.rs @@ -173,6 +173,7 @@ impl Component for SearchPage { Msg::ClearResults => { self.selected_idx = 0; self.docs_results = Vec::new(); + self.lens = Vec::new(); self.lens_results = Vec::new(); self.search_meta = None; self.result_display = ResultDisplay::None; @@ -182,6 +183,7 @@ impl Component for SearchPage { Msg::ClearQuery => { self.selected_idx = 0; self.docs_results = Vec::new(); + self.lens = Vec::new(); self.lens_results = Vec::new(); self.search_meta = None; self.query = "".to_string(); From 5b38d223abfb062f5936afbdc0c6c7d4ee6b67f4 Mon Sep 17 00:00:00 2001 From: Andrew Huynh Date: Wed, 9 Nov 2022 12:19:34 -0800 Subject: [PATCH 3/7] bugfix: firefox plugin windows compatibility (#215) * wasm path can't check host, lets do it manually * move enqueue logic into host, plugin is being too finnicky * bugfix: use data_dir() instead of data_local_dir() so that windows default data dir correct * cargo fmt --- Cargo.lock | 126 +++++++++++++++++++++++--- crates/client/src/pages/search.rs | 1 - crates/spyglass-plugin/src/shims.rs | 5 +- crates/spyglass/Cargo.toml | 2 +- crates/spyglass/src/crawler/mod.rs | 1 - crates/spyglass/src/plugin/exports.rs | 15 ++- crates/spyglass/src/plugin/mod.rs | 2 +- plugins/firefox-importer/src/main.rs | 41 ++++----- 8 files changed, 147 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e5a401588..b0762acaa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -727,6 +727,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "chunked_transfer" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fff857943da45f546682664a79488be82e69e43c1a7a2307679ab9afb3a66d2e" + [[package]] name = "cipher" version = "0.3.0" @@ -5165,11 +5171,28 @@ checksum = "73642819e7fa63eb264abc818a2f65ac8764afbe4870b5ee25bcecc491be0d4c" dependencies = [ "httpdate", "reqwest", - "sentry-backtrace", - "sentry-contexts", - "sentry-core", - "sentry-panic", + "sentry-backtrace 0.27.0", + "sentry-contexts 0.27.0", + "sentry-core 0.27.0", + "sentry-panic 0.27.0", + "tokio", +] + +[[package]] +name = "sentry" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a120fb5e8b7975736bf1fc57de380531e617a6a8f5a55d037bcea25a7f5e8371" +dependencies = [ + "httpdate", + "native-tls", + "reqwest", + "sentry-backtrace 0.28.0", + "sentry-contexts 0.28.0", + "sentry-core 0.28.0", + "sentry-panic 0.28.0", "tokio", + "ureq", ] [[package]] @@ -5181,7 +5204,19 @@ dependencies = [ "backtrace", "once_cell", "regex", - "sentry-core", + "sentry-core 0.27.0", +] + +[[package]] +name = "sentry-backtrace" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac56ff9aae25b024a5aad4f0242808dfde29161c82d183adce778338c6822ef" +dependencies = [ + "backtrace", + "once_cell", + "regex", + "sentry-core 0.28.0", ] [[package]] @@ -5193,7 +5228,21 @@ dependencies = [ "hostname", "libc", "rustc_version 0.4.0", - "sentry-core", + "sentry-core 0.27.0", + "uname", +] + +[[package]] +name = "sentry-contexts" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "188506b08b5e64004c71b7a5edb34959083e6e1288fada3b8d18d0bc7449ce1e" +dependencies = [ + "hostname", + "libc", + "os_info", + "rustc_version 0.4.0", + "sentry-core 0.28.0", "uname", ] @@ -5205,7 +5254,20 @@ checksum = "5a4591a2d128af73b1b819ab95f143bc6a2fbe48cd23a4c45e1ee32177e66ae6" dependencies = [ "once_cell", "rand 0.8.5", - "sentry-types", + "sentry-types 0.27.0", + "serde", + "serde_json", +] + +[[package]] +name = "sentry-core" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff58433a7ad557b586a09c42c4298d5f3ddb0c777e1a79d950e510d7b93fce0e" +dependencies = [ + "once_cell", + "rand 0.8.5", + "sentry-types 0.28.0", "serde", "serde_json", ] @@ -5216,8 +5278,18 @@ version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "696c74c5882d5a0d5b4a31d0ff3989b04da49be7983b7f52a52c667da5b480bf" dependencies = [ - "sentry-backtrace", - "sentry-core", + "sentry-backtrace 0.27.0", + "sentry-core 0.27.0", +] + +[[package]] +name = "sentry-panic" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4145005d9b5c117132765c34e2cb33e9d24d16e73d7f3a357122b77fe3a3b815" +dependencies = [ + "sentry-backtrace 0.28.0", + "sentry-core 0.28.0", ] [[package]] @@ -5237,6 +5309,24 @@ dependencies = [ "uuid 1.2.1", ] +[[package]] +name = "sentry-types" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb30d75498a041005a774ec1b6b7d9589c5906d17ebaca338cb685dc92170f9b" +dependencies = [ + "chrono", + "debugid", + "getrandom 0.2.7", + "hex", + "serde", + "serde_json", + "thiserror", + "time 0.3.15", + "url", + "uuid 1.2.1", +] + [[package]] name = "serde" version = "1.0.145" @@ -5614,7 +5704,7 @@ dependencies = [ "reqwest", "ron", "rusqlite", - "sentry", + "sentry 0.28.0", "serde", "sha2", "shared", @@ -5649,7 +5739,7 @@ dependencies = [ "open", "reqwest", "ron", - "sentry", + "sentry 0.27.0", "serde", "serde_json", "shared", @@ -6840,6 +6930,20 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "ureq" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97acb4c28a254fd7a4aeec976c46a7fa404eac4d7c134b30c75144846d7cb8f" +dependencies = [ + "base64", + "chunked_transfer", + "log", + "native-tls", + "once_cell", + "url", +] + [[package]] name = "url" version = "2.3.1" diff --git a/crates/client/src/pages/search.rs b/crates/client/src/pages/search.rs index 6282c7fcc..f8f242d05 100644 --- a/crates/client/src/pages/search.rs +++ b/crates/client/src/pages/search.rs @@ -183,7 +183,6 @@ impl Component for SearchPage { Msg::ClearQuery => { self.selected_idx = 0; self.docs_results = Vec::new(); - self.lens = Vec::new(); self.lens_results = Vec::new(); self.search_meta = None; self.query = "".to_string(); diff --git a/crates/spyglass-plugin/src/shims.rs b/crates/spyglass-plugin/src/shims.rs index 62dd2b752..eb523ec20 100644 --- a/crates/spyglass-plugin/src/shims.rs +++ b/crates/spyglass-plugin/src/shims.rs @@ -78,7 +78,7 @@ pub fn log(msg: String) { /// Hacky workaround until rusqlite can compile to wasm easily. /// Path is expected to be rooted in the plugins data directory. -pub fn sqlite3_query(path: &str, query: &str) -> Result, ron::error::SpannedError> { +pub fn sqlite3_query(path: &str, query: &str) { if object_to_stdout(&PluginCommandRequest::SqliteQuery { path: path.to_string(), query: query.to_string(), @@ -86,10 +86,7 @@ pub fn sqlite3_query(path: &str, query: &str) -> Result, ron::error: .is_ok() { unsafe { plugin_cmd() }; - return object_from_stdin::>(); } - - Ok(Vec::new()) } /// Adds / updates a file in the plugin VFS from the host. diff --git a/crates/spyglass/Cargo.toml b/crates/spyglass/Cargo.toml index 14d697d3c..06c5e4bd7 100644 --- a/crates/spyglass/Cargo.toml +++ b/crates/spyglass/Cargo.toml @@ -33,7 +33,7 @@ regex = "1" reqwest = "0.11" ron = "0.8" rusqlite = { version = "*", features = ["bundled"] } -sentry = "0.27.0" +sentry = "0.28.0" serde = { version = "1.0", features = ["derive"] } sha2 = "0.10" shared = { path = "../shared" } diff --git a/crates/spyglass/src/crawler/mod.rs b/crates/spyglass/src/crawler/mod.rs index 17f27de06..4423276f2 100644 --- a/crates/spyglass/src/crawler/mod.rs +++ b/crates/spyglass/src/crawler/mod.rs @@ -397,7 +397,6 @@ impl Crawler { url: &Url, parse_results: bool, ) -> anyhow::Result, anyhow::Error> { - println!("Http fetch"); // Modify bootstrapped URLs to pull from the Internet Archive let url: Url = if crawl.crawl_type == crawl_queue::CrawlType::Bootstrap { Url::parse(&create_archive_url(url.as_ref())).expect("Unable to create archive URL") diff --git a/crates/spyglass/src/plugin/exports.rs b/crates/spyglass/src/plugin/exports.rs index 27c56b1fb..23d20aa31 100644 --- a/crates/spyglass/src/plugin/exports.rs +++ b/crates/spyglass/src/plugin/exports.rs @@ -58,7 +58,7 @@ async fn handle_plugin_cmd_request( // Enqueue a list of URLs to be crawled PluginCommandRequest::Enqueue { urls } => handle_plugin_enqueue(env, urls), PluginCommandRequest::ListDir { path } => { - log::info!("{} listing path: {}", env.name, path); + log::debug!("{} listing path: {}", env.name, path); let entries = std::fs::read_dir(path)? .flatten() .map(|entry| { @@ -95,16 +95,21 @@ async fn handle_plugin_cmd_request( Ok(row.get::(0).unwrap_or_default()) })?; - let collected: Vec = results + let urls: Vec = results .map(|x| x.unwrap_or_default()) .collect::>() .into_iter() .filter(|x| !x.is_empty()) .collect(); - wasi_write(&env.wasi_env, &collected)?; + log::debug!("PCR::SqliteQUery: found {} urls", urls.len()); + handle_plugin_enqueue(env, &urls); + } + PluginCommandRequest::SyncFile { dst, src } => { + handle_sync_file(env, dst, src); + // Sleep a little bit to let the copy complete. + tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; } - PluginCommandRequest::SyncFile { dst, src } => handle_sync_file(env, dst, src), // Walk through a path & enqueue matching files for indexing. PluginCommandRequest::WalkAndEnqueue { path, extensions } => { let dir_path = Path::new(&path); @@ -153,7 +158,7 @@ pub(crate) fn plugin_log(env: &PluginEnv) { /// Adds a file into the plugin data directory. Use this to copy files from elsewhere /// in the filesystem so that it can be processed by the plugin. fn handle_sync_file(env: &PluginEnv, dst: &str, src: &str) { - log::info!("<{}> requesting access to folder: {}", env.name, src); + log::info!("<{}> requesting access to file: {}", env.name, src); let dst = Path::new(dst.trim_start_matches('/')); let src = Path::new(&src); diff --git a/crates/spyglass/src/plugin/mod.rs b/crates/spyglass/src/plugin/mod.rs index 83776c35a..08a2cf478 100644 --- a/crates/spyglass/src/plugin/mod.rs +++ b/crates/spyglass/src/plugin/mod.rs @@ -460,7 +460,7 @@ pub async fn plugin_init( .map_or_else(|| "".to_string(), |dir| dir); let base_data_dir: String = directories::BaseDirs::new() - .map(|base| base.data_local_dir().display().to_string()) + .map(|base| base.data_dir().display().to_string()) .map_or_else(|| "".to_string(), |dir| dir); let home_dir: String = directories::BaseDirs::new() diff --git a/plugins/firefox-importer/src/main.rs b/plugins/firefox-importer/src/main.rs index 940b9ddcd..98975951e 100644 --- a/plugins/firefox-importer/src/main.rs +++ b/plugins/firefox-importer/src/main.rs @@ -7,15 +7,7 @@ const DB_FILE: &str = "places.sqlite"; // How often we want to sync w/ the firefox database const SYNC_INTERVAL_S: u64 = 60 * 5; // SQL query to find bookmarks -const BOOKMARK_QUERY: &str = " - SELECT - DISTINCT url - FROM moz_bookmarks - JOIN moz_places on moz_places.id = moz_bookmarks.fk - WHERE - moz_places.hidden = 0 - AND url like 'http%' -"; +const BOOKMARK_QUERY: &str = "SELECT DISTINCT url FROM moz_bookmarks JOIN moz_places on moz_places.id = moz_bookmarks.fk WHERE moz_places.hidden = 0 AND url like 'http%'"; struct Plugin { last_update: Instant, @@ -70,9 +62,12 @@ impl SpyglassPlugin for Plugin { } if path.exists() { - enqueue_all(&self.read_bookmarks()); + self.read_bookmarks(); } else { - log("Unable to find places.sqlite file".to_string()); + log(format!( + "Unable to find places.sqlite file @ {}", + path.to_string_lossy() + )); } } } @@ -85,7 +80,7 @@ impl Plugin { let base_data_res = std::env::var(consts::env::BASE_DATA_DIR); let profiles_dir = if let (Ok(host_os), Ok(home_dir), Ok(data_dir)) = - (host_os_res, host_home_res, base_data_res) + (host_os_res.clone(), host_home_res, base_data_res) { // Determined from https://support.mozilla.org/en-US/kb/profiles-where-firefox-stores-user-data match host_os.as_str() { @@ -93,7 +88,9 @@ impl Plugin { "macos" => { Some(Path::new(&home_dir).join("Library/Application Support/Firefox/Profiles")) } - "windows" => Some(Path::new(&data_dir).join("Mozilla\\Firefox\\Profile\\")), + "windows" => Some( + Path::new(&format!("{}\\Mozilla\\Firefox\\Profile\\", &data_dir)).to_path_buf(), + ), _ => None, } } else { @@ -102,14 +99,19 @@ impl Plugin { // Loop through profiles in the profile directory & find the default one. // A little hacky since Firefox prepends a random string to the profile name. - if let Some(profiles_dir) = profiles_dir { + if let (Ok(host_os), Some(profiles_dir)) = (host_os_res, profiles_dir) { if let Ok(entries) = list_dir(&profiles_dir.display().to_string()) { for entry in entries { if entry.is_dir && (entry.path.ends_with(".default") || entry.path.ends_with(".default-release")) { - return Some(Path::new(&entry.path).join(DB_FILE)); + return match host_os.as_str() { + "windows" => Some( + Path::new(&format!("{}\\{}", &entry.path, DB_FILE)).to_path_buf(), + ), + _ => Some(Path::new(&entry.path).join(DB_FILE)), + }; } } } @@ -118,12 +120,7 @@ impl Plugin { None } - fn read_bookmarks(&self) -> Vec { - let urls = sqlite3_query("places.sqlite", BOOKMARK_QUERY); - if let Ok(urls) = urls { - return urls; - } - - Vec::new() + fn read_bookmarks(&self) { + sqlite3_query("places.sqlite", BOOKMARK_QUERY); } } From a3945cdc4669a39f7ce472527fbf6418f3278971 Mon Sep 17 00:00:00 2001 From: Andrew Huynh Date: Wed, 9 Nov 2022 13:44:14 -0800 Subject: [PATCH 4/7] bugfix: fix lock contention w/ plugin manager (#216) * fix lock contention w/ plugin manager * cargo fmt + clippy --- crates/spyglass/src/plugin/mod.rs | 40 ++++++++++++++++--------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/crates/spyglass/src/plugin/mod.rs b/crates/spyglass/src/plugin/mod.rs index 08a2cf478..008efd065 100644 --- a/crates/spyglass/src/plugin/mod.rs +++ b/crates/spyglass/src/plugin/mod.rs @@ -6,11 +6,10 @@ use std::time::Duration; use dashmap::DashMap; use entities::sea_orm::{ColumnTrait, EntityTrait, QueryFilter}; -use notify::{event::ModifyKind, EventKind, RecommendedWatcher, RecursiveMode, Watcher}; +use notify::{event::ModifyKind, EventKind, RecursiveMode, Watcher}; use serde::de::DeserializeOwned; use serde::Serialize; use spyglass_plugin::SearchFilter; -use tokio::sync::mpsc::Receiver; use tokio::sync::{broadcast, mpsc}; use tokio::task::JoinHandle; use wasmer::{Instance, Module, Store, WasmerEnv}; @@ -115,9 +114,6 @@ pub struct PluginManager { check_update_subs: HashSet, file_watch_subs: DashMap, plugins: DashMap, - // For file watching subscribers - file_events: Receiver>, - file_watcher: RecommendedWatcher, } impl Default for PluginManager { @@ -146,20 +142,10 @@ impl PluginManager { } pub fn new() -> Self { - let (tx, rx) = tokio::sync::mpsc::channel(1); - let watcher = notify::recommended_watcher(move |res| { - futures::executor::block_on(async { - tx.send(res).await.expect("Unable to send FS event"); - }) - }) - .expect("Unable to watch lens directory"); - PluginManager { check_update_subs: Default::default(), file_watch_subs: Default::default(), plugins: Default::default(), - file_events: rx, - file_watcher: watcher, } } @@ -188,16 +174,24 @@ pub async fn plugin_event_loop( let mut config = config.clone(); plugin_load(&state, &mut config, &cmd_writer).await; + // For file watching subscribers + let (tx, mut file_events) = tokio::sync::mpsc::channel(1); + let mut watcher = notify::recommended_watcher(move |res| { + futures::executor::block_on(async { + tx.send(res).await.expect("Unable to send FS event"); + }) + }) + .expect("Unable to watch lens directory"); + // Subscribe plugins check for updates every 10 minutes let mut interval = tokio::time::interval(Duration::from_secs(10 * 60)); loop { - let mut manager = state.plugin_manager.lock().await; // Wait for next command / handle shutdown responses let next_cmd = tokio::select! { // Listen for plugin requests res = cmd_queue.recv() => res, // Listen for file change notifications - file_event = manager.file_events.recv() => { + file_event = file_events.recv() => { if let Some(Ok(file_event)) = file_event { Some(PluginCommand::QueueFileNotify(file_event)) } else { @@ -218,6 +212,7 @@ pub async fn plugin_event_loop( log::info!("disabling plugin <{}>", plugin_name); let mut disabled = Vec::new(); + let mut manager = state.plugin_manager.lock().await; if let Some(plugin) = manager.find_by_name(plugin_name) { if let Some(mut instance) = manager.plugins.get_mut(&plugin.id) { instance.config.is_enabled = false; @@ -231,6 +226,7 @@ pub async fn plugin_event_loop( } Some(PluginCommand::EnablePlugin(plugin_name)) => { log::info!("enabling plugin <{}>", plugin_name); + let manager = state.plugin_manager.lock().await; if let Some(plugin) = manager.find_by_name(plugin_name) { if let Some(mut instance) = manager.plugins.get_mut(&plugin.id) { instance.config.is_enabled = true; @@ -242,13 +238,15 @@ pub async fn plugin_event_loop( } } Some(PluginCommand::HandleUpdate { plugin_id, event }) => { + let manager = state.plugin_manager.lock().await; if let Some(mut plugin) = manager.plugins.get_mut(&plugin_id) { plugin.update(event); } else { log::error!("Unable to find plugin id: {}", plugin_id); - } + }; } Some(PluginCommand::Initialize(plugin)) => { + let manager = state.plugin_manager.lock().await; let plugin_id = manager.plugins.len(); match plugin_init(plugin_id, &state, &cmd_writer, &plugin).await { Ok((instance, env)) => { @@ -267,6 +265,7 @@ pub async fn plugin_event_loop( } Some(PluginCommand::Subscribe(plugin_id, event)) => match event { PluginSubscription::CheckUpdateInterval => { + let mut manager = state.plugin_manager.lock().await; manager.check_update_subs.insert(plugin_id); let _ = cmd_writer .send(PluginCommand::HandleUpdate { @@ -282,7 +281,7 @@ pub async fn plugin_event_loop( return; } - let _ = manager.file_watcher.watch( + let _ = watcher.watch( &path, if recurse { RecursiveMode::Recursive @@ -291,11 +290,13 @@ pub async fn plugin_event_loop( }, ); + let manager = state.plugin_manager.lock().await; manager.file_watch_subs.insert(plugin_id, path); } }, // Queue update checks for subscribed plugins Some(PluginCommand::QueueIntervalCheck) => { + let manager = state.plugin_manager.lock().await; for plugin_id in &manager.check_update_subs { let _ = cmd_writer .send(PluginCommand::HandleUpdate { @@ -341,6 +342,7 @@ pub async fn plugin_event_loop( }; if let Some(event) = event { + let manager = state.plugin_manager.lock().await; for entry in &manager.file_watch_subs { let watched_path = entry.value(); if updated_path.starts_with(watched_path) { From 6a0532ad5875643a2a5412e683aa8da3c1efd5a0 Mon Sep 17 00:00:00 2001 From: Andrew Huynh Date: Wed, 9 Nov 2022 14:02:05 -0800 Subject: [PATCH 5/7] tweak: don't filter domains in crawl stats --- crates/spyglass/src/api/route.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/spyglass/src/api/route.rs b/crates/spyglass/src/api/route.rs index 1f9c4237a..5017fdd3b 100644 --- a/crates/spyglass/src/api/route.rs +++ b/crates/spyglass/src/api/route.rs @@ -176,7 +176,6 @@ pub async fn crawl_stats(state: AppState) -> Result { let by_domain = by_domain .into_iter() - .filter(|(_, stats)| stats.total() >= 10) .collect(); Ok(CrawlStats { by_domain }) From 5977e899573c8431454b247af660422443a6864a Mon Sep 17 00:00:00 2001 From: Andrew Huynh Date: Wed, 9 Nov 2022 14:15:07 -0800 Subject: [PATCH 6/7] tweak: cleaning up os path compatibility checks --- crates/spyglass/src/api/route.rs | 4 +--- plugins/firefox-importer/src/main.rs | 34 +++++++++++++++------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/crates/spyglass/src/api/route.rs b/crates/spyglass/src/api/route.rs index 5017fdd3b..34e6c58ad 100644 --- a/crates/spyglass/src/api/route.rs +++ b/crates/spyglass/src/api/route.rs @@ -174,9 +174,7 @@ pub async fn crawl_stats(state: AppState) -> Result { entry.num_indexed += stat.count as u64; } - let by_domain = by_domain - .into_iter() - .collect(); + let by_domain = by_domain.into_iter().collect(); Ok(CrawlStats { by_domain }) } diff --git a/plugins/firefox-importer/src/main.rs b/plugins/firefox-importer/src/main.rs index 98975951e..c22acc043 100644 --- a/plugins/firefox-importer/src/main.rs +++ b/plugins/firefox-importer/src/main.rs @@ -25,6 +25,14 @@ impl Default for Plugin { register_plugin!(Plugin); +fn join_path(folder: &str, file: &str) -> PathBuf { + let host_os_res = std::env::var(consts::env::HOST_OS); + match host_os_res.unwrap_or_default().as_str() { + "windows" => Path::new(&format!("{}\\\\{}", folder, file)).to_path_buf(), + _ => Path::new(&folder).join(file), + } +} + impl SpyglassPlugin for Plugin { fn load(&mut self) { // Let the host know we want to check for updates on a regular interval. @@ -33,7 +41,7 @@ impl SpyglassPlugin for Plugin { let mut profile_path = None; if let Ok(folder) = std::env::var("FIREFOX_DATA_FOLDER") { if !folder.is_empty() { - profile_path = Some(Path::new(&folder).join(DB_FILE)) + profile_path = Some(join_path(&folder, DB_FILE)); } } @@ -80,17 +88,16 @@ impl Plugin { let base_data_res = std::env::var(consts::env::BASE_DATA_DIR); let profiles_dir = if let (Ok(host_os), Ok(home_dir), Ok(data_dir)) = - (host_os_res.clone(), host_home_res, base_data_res) + (host_os_res, host_home_res, base_data_res) { // Determined from https://support.mozilla.org/en-US/kb/profiles-where-firefox-stores-user-data match host_os.as_str() { - "linux" => Some(Path::new(&home_dir).join(".mozilla/firefox")), - "macos" => { - Some(Path::new(&home_dir).join("Library/Application Support/Firefox/Profiles")) - } - "windows" => Some( - Path::new(&format!("{}\\Mozilla\\Firefox\\Profile\\", &data_dir)).to_path_buf(), - ), + "linux" => Some(join_path(&home_dir, ".mozilla/firefox")), + "macos" => Some(join_path( + &home_dir, + "Library/Application Support/Firefox/Profiles", + )), + "windows" => Some(join_path(&data_dir, "Mozilla\\Firefox\\Profiles\\")), _ => None, } } else { @@ -99,19 +106,14 @@ impl Plugin { // Loop through profiles in the profile directory & find the default one. // A little hacky since Firefox prepends a random string to the profile name. - if let (Ok(host_os), Some(profiles_dir)) = (host_os_res, profiles_dir) { + if let Some(profiles_dir) = profiles_dir { if let Ok(entries) = list_dir(&profiles_dir.display().to_string()) { for entry in entries { if entry.is_dir && (entry.path.ends_with(".default") || entry.path.ends_with(".default-release")) { - return match host_os.as_str() { - "windows" => Some( - Path::new(&format!("{}\\{}", &entry.path, DB_FILE)).to_path_buf(), - ), - _ => Some(Path::new(&entry.path).join(DB_FILE)), - }; + return Some(join_path(&entry.path, DB_FILE)); } } } From ce897e920d4c15d8ead830fa07017eff1d70a3f7 Mon Sep 17 00:00:00 2001 From: Andrew Huynh Date: Wed, 9 Nov 2022 14:43:32 -0800 Subject: [PATCH 7/7] release: v2022.11.4 hotfix --- Cargo.lock | 2 +- crates/spyglass/Cargo.toml | 2 +- crates/tauri/tauri.conf.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b0762acaa..bd3dbf5c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5673,7 +5673,7 @@ dependencies = [ [[package]] name = "spyglass" -version = "22.11.3" +version = "22.11.4" dependencies = [ "addr", "anyhow", diff --git a/crates/spyglass/Cargo.toml b/crates/spyglass/Cargo.toml index 06c5e4bd7..11ad11706 100644 --- a/crates/spyglass/Cargo.toml +++ b/crates/spyglass/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spyglass" -version = "22.11.3" +version = "22.11.4" edition = "2021" [dependencies] diff --git a/crates/tauri/tauri.conf.json b/crates/tauri/tauri.conf.json index c3bbcb49b..e7ad4a5d6 100644 --- a/crates/tauri/tauri.conf.json +++ b/crates/tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "package": { "productName": "Spyglass", - "version": "22.11.3" + "version": "22.11.4" }, "build": { "distDir": "../client/dist",