release: v2022.11.4 hotfix

README.md

@@ -5,13 +5,13 @@
     <br />
     <br />
         Download now:
-        <a href="https://github.com/a5huynh/spyglass/releases/download/v2022.11.2/Spyglass_22.11.2_x64.dmg">
+        <a href="https://github.com/a5huynh/spyglass/releases/download/v2022.11.3/Spyglass_22.11.3_x64.dmg">
             <strong>macOS</strong>
         </a> |
-        <a href="https://github.com/a5huynh/spyglass/releases/download/v2022.11.2/Spyglass_22.11.2_x64_en-US.msi">
+        <a href="https://github.com/a5huynh/spyglass/releases/download/v2022.11.3/Spyglass_22.11.3_x64_en-US.msi">
             <strong>Windows</strong>
         </a> |
-        <a href="https://github.com/a5huynh/spyglass/releases/download/v2022.11.2/spyglass_22.11.2_amd64.AppImage">
+        <a href="https://github.com/a5huynh/spyglass/releases/download/v2022.11.3/spyglass_22.11.3_amd64.AppImage">
             <strong>Linux (AppImage)</strong>
         </a>
     <br />

VERSION.json

@@ -1,23 +1,23 @@
 {
-  "version": "22.11.2",
-  "notes": "See full release notes here: https://github.com/a5huynh/spyglass/releases/tag/v2022.11.2",
-  "pub_date": "2022-11-08T21:19:50Z",
+  "version": "22.11.3",
+  "notes": "See full release notes here: https://github.com/a5huynh/spyglass/releases/tag/v2022.11.3",
+  "pub_date": "2022-11-09T06:02:07Z",
   "platforms": {
     "darwin-x86_64": {
-      "signature": "dW50cnVzdGVkIGNvbW1lbnQ6IHNpZ25hdHVyZSBmcm9tIHRhdXJpIHNlY3JldCBrZXkKUlVSRk9WYXdDUXphYmt1TFBFSWRNVFc4VmFna29UWCtNU1Bhd0FKbGd2dDlEcUEwNjZsT2ZmOWR6S1MxdHViQ1NIRFZEa1ZUUXJhUk9UM08xczJBZFNkdGhsNnRqbGtkZVFJPQp0cnVzdGVkIGNvbW1lbnQ6IHRpbWVzdGFtcDoxNjY3OTQxNjM0CWZpbGU6U3B5Z2xhc3MuYXBwLnRhci5negpTN3cxb1pucVZOUWcxeTBqdHJoMlhiNTkxOXh3QnV6WG1zSEZ3RGZrb1JPejRUU1FWQ1BLUUtVdHhFc2MvajgzSTFwa3ZKVzhST2RjNkZweW00ZEhEdz09Cg==",
-      "url": "https://github.com/a5huynh/spyglass/releases/download/v2022.11.2/Spyglass.app.tar.gz"
+      "signature": "dW50cnVzdGVkIGNvbW1lbnQ6IHNpZ25hdHVyZSBmcm9tIHRhdXJpIHNlY3JldCBrZXkKUlVSRk9WYXdDUXphYmxjRWRDcGFlbFBpcnRHNXpkbStENUx4Smc1bDloVWFYQ2wrQzlvZGZrOXgxaTVEZWprN2E5L0drT2VJZTl5clZTamtlQW9TcUlQNDZ1VFh4L3dLZ0FZPQp0cnVzdGVkIGNvbW1lbnQ6IHRpbWVzdGFtcDoxNjY3OTczMTQ1CWZpbGU6U3B5Z2xhc3MuYXBwLnRhci5negpsRW9TSHY2WnR1OXNHb1ZYMVFsUi9JWU9XbXN1c3pJaFVqM3BPNVd0SHhleHo2YktRUWZ1Rk1pa3ZtUGRLc1pManAxaVdSc2xVOWJmYjErWE52R3JEZz09Cg==",
+      "url": "https://github.com/a5huynh/spyglass/releases/download/v2022.11.3/Spyglass.app.tar.gz"
     },
     "darwin-aarch64": {
-      "signature": "dW50cnVzdGVkIGNvbW1lbnQ6IHNpZ25hdHVyZSBmcm9tIHRhdXJpIHNlY3JldCBrZXkKUlVSRk9WYXdDUXphYmt1TFBFSWRNVFc4VmFna29UWCtNU1Bhd0FKbGd2dDlEcUEwNjZsT2ZmOWR6S1MxdHViQ1NIRFZEa1ZUUXJhUk9UM08xczJBZFNkdGhsNnRqbGtkZVFJPQp0cnVzdGVkIGNvbW1lbnQ6IHRpbWVzdGFtcDoxNjY3OTQxNjM0CWZpbGU6U3B5Z2xhc3MuYXBwLnRhci5negpTN3cxb1pucVZOUWcxeTBqdHJoMlhiNTkxOXh3QnV6WG1zSEZ3RGZrb1JPejRUU1FWQ1BLUUtVdHhFc2MvajgzSTFwa3ZKVzhST2RjNkZweW00ZEhEdz09Cg==",
-      "url": "https://github.com/a5huynh/spyglass/releases/download/v2022.11.2/Spyglass.app.tar.gz"
+      "signature": "dW50cnVzdGVkIGNvbW1lbnQ6IHNpZ25hdHVyZSBmcm9tIHRhdXJpIHNlY3JldCBrZXkKUlVSRk9WYXdDUXphYmxjRWRDcGFlbFBpcnRHNXpkbStENUx4Smc1bDloVWFYQ2wrQzlvZGZrOXgxaTVEZWprN2E5L0drT2VJZTl5clZTamtlQW9TcUlQNDZ1VFh4L3dLZ0FZPQp0cnVzdGVkIGNvbW1lbnQ6IHRpbWVzdGFtcDoxNjY3OTczMTQ1CWZpbGU6U3B5Z2xhc3MuYXBwLnRhci5negpsRW9TSHY2WnR1OXNHb1ZYMVFsUi9JWU9XbXN1c3pJaFVqM3BPNVd0SHhleHo2YktRUWZ1Rk1pa3ZtUGRLc1pManAxaVdSc2xVOWJmYjErWE52R3JEZz09Cg==",
+      "url": "https://github.com/a5huynh/spyglass/releases/download/v2022.11.3/Spyglass.app.tar.gz"
     },
     "linux-x86_64": {
-      "signature": "dW50cnVzdGVkIGNvbW1lbnQ6IHNpZ25hdHVyZSBmcm9tIHRhdXJpIHNlY3JldCBrZXkKUlVSRk9WYXdDUXphYm96ZlYxVjNxR09ZMGJwR0NWMElkM2Nvd2UwbWxZc1F3WHhXTzN0QjdoV0VjSkhhRWljd1hERXdFN0dOQXdHdmZIckxFV2VFTzJYVy9MVVI0b015Wmc4PQp0cnVzdGVkIGNvbW1lbnQ6IHRpbWVzdGFtcDoxNjY3OTM5ODE2CWZpbGU6c3B5Z2xhc3NfMjIuMTEuMl9hbWQ2NC5BcHBJbWFnZS50YXIuZ3oKdm0wV0NYekIvemdablhSNERkdTJKKzNUTUNLdjVvSHhsdDhoTXNIWjFIWVNKTW1wQzZ0Q3lTbkxGanE4VTFCZHZZVFpKcmFGbUVwVmtBOXR0VzJWQXc9PQo=",
-      "url": "https://github.com/a5huynh/spyglass/releases/download/v2022.11.2/spyglass_22.11.2_amd64.AppImage.tar.gz"
+      "signature": "dW50cnVzdGVkIGNvbW1lbnQ6IHNpZ25hdHVyZSBmcm9tIHRhdXJpIHNlY3JldCBrZXkKUlVSRk9WYXdDUXphYnVXVXpHS3ZCUWROK3d3WlNSN1BLdjdOZkJMdUViZ3puODlmdnBFVHpJczRxUG9UYjdsZjhTNkQ4dnJuN1FNSFlZdkxPMHpBblpLT3kxZ0JvaE9VK1E0PQp0cnVzdGVkIGNvbW1lbnQ6IHRpbWVzdGFtcDoxNjY3OTcxODg3CWZpbGU6c3B5Z2xhc3NfMjIuMTEuM19hbWQ2NC5BcHBJbWFnZS50YXIuZ3oKSHcvTS9OdC9NSzdoM1hueHVNcHV3Wk0vWGNCVEY2QXpqRm5WQWUxR05Hc204RUNvc0FpcitPTkRQNVlDUjdTdk5mMVUzTGtVOThFWnZ1RW1JNmxDREE9PQo=",
+      "url": "https://github.com/a5huynh/spyglass/releases/download/v2022.11.3/spyglass_22.11.3_amd64.AppImage.tar.gz"
     },
     "windows-x86_64": {
-      "signature": "dW50cnVzdGVkIGNvbW1lbnQ6IHNpZ25hdHVyZSBmcm9tIHRhdXJpIHNlY3JldCBrZXkKUlVSRk9WYXdDUXphYnJPaTY2ZG01TlBkSHlZNXNvZXNaUDMwanlXUDZNYjc2MnRZK0phditNVmNLU0RXdWswaEdUS3ozbFM1UVMxODlrYlNONVBPM0w2Z3pZZkw5SEZSc2dFPQp0cnVzdGVkIGNvbW1lbnQ6IHRpbWVzdGFtcDoxNjY3OTQwNDg1CWZpbGU6U3B5Z2xhc3NfMjIuMTEuMl94NjRfZW4tVVMubXNpLnppcAplVGRuYzhuYkxOcUV1RlhIQ21BQjlqNjhvQmJocC9ibDNBZzc0UmNHTDBPY3Fyc1FkL1dvRVFVS0pXVVZLNU0rV25IenZIaVFxcC9PakwxTUFRbWZCQT09Cg==",
-      "url": "https://github.com/a5huynh/spyglass/releases/download/v2022.11.2/Spyglass_22.11.2_x64_en-US.msi.zip"
+      "signature": "dW50cnVzdGVkIGNvbW1lbnQ6IHNpZ25hdHVyZSBmcm9tIHRhdXJpIHNlY3JldCBrZXkKUlVSRk9WYXdDUXphYmhiZ1VKUHh4R2Vadm5aVWFJL2lZcVoxVllqd090cWc5M3F0SllMRmJ4bE9jajZNRlMyWGJsWENxQmFmLzMrQmNqdTNXUHNQWjEveVdXTFdnaWF0cXdZPQp0cnVzdGVkIGNvbW1lbnQ6IHRpbWVzdGFtcDoxNjY3OTczNTIyCWZpbGU6U3B5Z2xhc3NfMjIuMTEuM194NjRfZW4tVVMubXNpLnppcApWWkRJZEpWRUNSUWx6Wldsd0xCMVpQRjZTUmc2Z3dRSHQvempuZkhBVmdlVTBxdDhvcTJlRzRSazd4b3ZJemFlM2lZVXZBaVlDY211ekQvNkkvMldDZz09Cg==",
+      "url": "https://github.com/a5huynh/spyglass/releases/download/v2022.11.3/Spyglass_22.11.3_x64_en-US.msi.zip"
     }
   }
 }

crates/client/src/pages/search.rs

@@ -173,6 +173,7 @@ impl Component for SearchPage {
             Msg::ClearResults => {
                 self.selected_idx = 0;
                 self.docs_results = Vec::new();
+                self.lens = Vec::new();
                 self.lens_results = Vec::new();
                 self.search_meta = None;
                 self.result_display = ResultDisplay::None;

crates/spyglass-plugin/src/shims.rs

@@ -78,18 +78,15 @@ pub fn log(msg: String) {
 
 /// Hacky workaround until rusqlite can compile to wasm easily.
 /// Path is expected to be rooted in the plugins data directory.
-pub fn sqlite3_query(path: &str, query: &str) -> Result<Vec<String>, ron::error::SpannedError> {
+pub fn sqlite3_query(path: &str, query: &str) {
     if object_to_stdout(&PluginCommandRequest::SqliteQuery {
         path: path.to_string(),
         query: query.to_string(),
     })
     .is_ok()
     {
         unsafe { plugin_cmd() };
-        return object_from_stdin::<Vec<String>>();
     }
-
-    Ok(Vec::new())
 }
 
 /// Adds / updates a file in the plugin VFS from the host.

crates/spyglass/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spyglass"
-version = "22.11.3"
+version = "22.11.4"
 edition = "2021"
 
 [dependencies]
@@ -33,7 +33,7 @@ regex = "1"
 reqwest = "0.11"
 ron = "0.8"
 rusqlite = { version = "*", features = ["bundled"] }
-sentry = "0.27.0"
+sentry = "0.28.0"
 serde = { version = "1.0", features = ["derive"] }
 sha2 = "0.10"
 shared = { path = "../shared" }

crates/spyglass/src/api/route.rs

@@ -174,10 +174,7 @@ pub async fn crawl_stats(state: AppState) -> Result<CrawlStats, Error> {
         entry.num_indexed += stat.count as u64;
     }
 
-    let by_domain = by_domain
-        .into_iter()
-        .filter(|(_, stats)| stats.total() >= 10)
-        .collect();
+    let by_domain = by_domain.into_iter().collect();
 
     Ok(CrawlStats { by_domain })
 }

crates/spyglass/src/crawler/mod.rs

@@ -397,7 +397,6 @@ impl Crawler {
         url: &Url,
         parse_results: bool,
     ) -> anyhow::Result<Option<CrawlResult>, anyhow::Error> {
-        println!("Http fetch");
         // Modify bootstrapped URLs to pull from the Internet Archive
         let url: Url = if crawl.crawl_type == crawl_queue::CrawlType::Bootstrap {
             Url::parse(&create_archive_url(url.as_ref())).expect("Unable to create archive URL")

crates/spyglass/src/plugin/exports.rs

@@ -58,7 +58,7 @@ async fn handle_plugin_cmd_request(
         // Enqueue a list of URLs to be crawled
         PluginCommandRequest::Enqueue { urls } => handle_plugin_enqueue(env, urls),
         PluginCommandRequest::ListDir { path } => {
-            log::info!("{} listing path: {}", env.name, path);
+            log::debug!("{} listing path: {}", env.name, path);
             let entries = std::fs::read_dir(path)?
                 .flatten()
                 .map(|entry| {
@@ -95,16 +95,21 @@ async fn handle_plugin_cmd_request(
                 Ok(row.get::<usize, String>(0).unwrap_or_default())
             })?;
 
-            let collected: Vec<String> = results
+            let urls: Vec<String> = results
                 .map(|x| x.unwrap_or_default())
                 .collect::<Vec<String>>()
                 .into_iter()
                 .filter(|x| !x.is_empty())
                 .collect();
 
-            wasi_write(&env.wasi_env, &collected)?;
+            log::debug!("PCR::SqliteQUery: found {} urls", urls.len());
+            handle_plugin_enqueue(env, &urls);
+        }
+        PluginCommandRequest::SyncFile { dst, src } => {
+            handle_sync_file(env, dst, src);
+            // Sleep a little bit to let the copy complete.
+            tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
         }
-        PluginCommandRequest::SyncFile { dst, src } => handle_sync_file(env, dst, src),
         // Walk through a path & enqueue matching files for indexing.
         PluginCommandRequest::WalkAndEnqueue { path, extensions } => {
             let dir_path = Path::new(&path);
@@ -153,7 +158,7 @@ pub(crate) fn plugin_log(env: &PluginEnv) {
 /// Adds a file into the plugin data directory. Use this to copy files from elsewhere
 /// in the filesystem so that it can be processed by the plugin.
 fn handle_sync_file(env: &PluginEnv, dst: &str, src: &str) {
-    log::info!("<{}> requesting access to folder: {}", env.name, src);
+    log::info!("<{}> requesting access to file: {}", env.name, src);
     let dst = Path::new(dst.trim_start_matches('/'));
     let src = Path::new(&src);