diff --git a/crates/entities/src/models/crawl_queue.rs b/crates/entities/src/models/crawl_queue.rs
index 3b828bdc9..f6a6bb8ef 100644
--- a/crates/entities/src/models/crawl_queue.rs
+++ b/crates/entities/src/models/crawl_queue.rs
@@ -135,7 +135,7 @@ pub async fn queue_stats(
Ok(res)
}
-pub async fn reset_processing(db: &DatabaseConnection) {
+pub async fn reset_processing(db: &DatabaseConnection) -> anyhow::Result<()> {
Entity::update_many()
.col_expr(
Column::Status,
@@ -145,8 +145,9 @@ pub async fn reset_processing(db: &DatabaseConnection) {
)
.filter(Column::Status.eq(CrawlStatus::Processing))
.exec(db)
- .await
- .unwrap();
+ .await?;
+
+ Ok(())
}
#[derive(Debug, FromQueryResult)]
@@ -431,15 +432,18 @@ pub async fn enqueue_all(
let mut result = None;
if !is_indexed.contains(&url) {
if let Ok(parsed) = Url::parse(&url) {
- if let Some(domain) = parsed.host_str() {
- result = Some(ActiveModel {
- domain: Set(domain.to_string()),
- crawl_type: Set(overrides.crawl_type.clone()),
- url: Set(url.to_string()),
- pipeline: Set(pipeline.clone()),
- ..Default::default()
- });
- }
+ let domain = match parsed.scheme() {
+ "file" => "localhost",
+ _ => parsed.host_str().expect("Invalid URL host"),
+ };
+
+ result = Some(ActiveModel {
+ domain: Set(domain.to_string()),
+ crawl_type: Set(overrides.crawl_type.clone()),
+ url: Set(url.to_string()),
+ pipeline: Set(pipeline.clone()),
+ ..Default::default()
+ });
}
}
result
diff --git a/crates/entities/src/models/fetch_history.rs b/crates/entities/src/models/fetch_history.rs
index 225773944..e2c0383d7 100644
--- a/crates/entities/src/models/fetch_history.rs
+++ b/crates/entities/src/models/fetch_history.rs
@@ -68,7 +68,7 @@ pub async fn find_by_url(
url: &Url,
) -> anyhow::Result
, sea_orm::DbErr> {
Entity::find()
- .filter(Column::Domain.eq(url.host_str().unwrap().to_string()))
+ .filter(Column::Domain.eq(url.host_str().unwrap_or_default().to_string()))
.filter(Column::Path.eq(url.path()))
.one(db)
.await
diff --git a/crates/entities/src/models/mod.rs b/crates/entities/src/models/mod.rs
index 97b4b311b..356587e86 100644
--- a/crates/entities/src/models/mod.rs
+++ b/crates/entities/src/models/mod.rs
@@ -22,7 +22,11 @@ pub async fn create_connection(
} else {
format!(
"sqlite://{}?mode=rwc",
- config.data_dir().join("db.sqlite").to_str().unwrap()
+ config
+ .data_dir()
+ .join("db.sqlite")
+ .to_str()
+ .expect("Unable to create db")
)
};
diff --git a/crates/migrations/Cargo.toml b/crates/migrations/Cargo.toml
index 48537e533..2c04df1b6 100644
--- a/crates/migrations/Cargo.toml
+++ b/crates/migrations/Cargo.toml
@@ -14,4 +14,5 @@ entities = { path = "../entities" }
shared = { path = "../shared" }
tantivy = "0.18"
rayon = "1.5"
-tar = "0.4"
\ No newline at end of file
+tar = "0.4"
+url = "2.3"
\ No newline at end of file
diff --git a/crates/migrations/src/lib.rs b/crates/migrations/src/lib.rs
index a5fa1d439..e7cc1dfff 100644
--- a/crates/migrations/src/lib.rs
+++ b/crates/migrations/src/lib.rs
@@ -14,6 +14,9 @@ mod m20221031_000001_add_error_column_to_crawl_queue;
mod m20221101_000001_add_open_url_col;
mod m20221107_000001_recreate_connection_table;
mod m20221109_add_tags_table;
+mod m20221115_000001_local_file_pathfix;
+mod m20221116_000001_add_connection_constraint;
+
mod utils;
pub struct Migrator;
@@ -33,6 +36,8 @@ impl MigratorTrait for Migrator {
Box::new(m20221101_000001_add_open_url_col::Migration),
Box::new(m20221107_000001_recreate_connection_table::Migration),
Box::new(m20221109_add_tags_table::Migration),
+ Box::new(m20221115_000001_local_file_pathfix::Migration),
+ Box::new(m20221116_000001_add_connection_constraint::Migration),
]
}
}
diff --git a/crates/migrations/src/m20221115_000001_local_file_pathfix.rs b/crates/migrations/src/m20221115_000001_local_file_pathfix.rs
new file mode 100644
index 000000000..e74368701
--- /dev/null
+++ b/crates/migrations/src/m20221115_000001_local_file_pathfix.rs
@@ -0,0 +1,183 @@
+use entities::schema::{DocFields, SearchDocument};
+use sea_orm_migration::prelude::*;
+
+use entities::models::{crawl_queue, indexed_document};
+use entities::sea_orm::{ActiveModelTrait, ColumnTrait, EntityTrait, QueryFilter, Set};
+use shared::config::Config;
+use tantivy::collector::TopDocs;
+use tantivy::directory::MmapDirectory;
+use tantivy::query::TermQuery;
+use tantivy::schema::IndexRecordOption;
+use tantivy::{Document, Index, IndexReader, IndexWriter, ReloadPolicy, Term};
+use url::Url;
+
+pub struct Migration;
+
+impl Migration {
+ fn fix_url(&self, url: &str) -> Option {
+ if let Ok(mut parsed) = Url::parse(url) {
+ // Switch host to localhost so that we can use `to_file_path`
+ let _ = parsed.set_host(Some("localhost"));
+
+ // If we're on Windows, fix the path bug we found where the `ignore`
+ // overescapes windows paths.
+ if cfg!(target_os = "windows") {
+ if let Ok(path_str) = parsed.to_file_path() {
+ let path_str = path_str.display().to_string();
+ parsed.set_path(&path_str.replace("\\\\", "\\"));
+ }
+ }
+
+ return Some(parsed.to_string());
+ }
+
+ None
+ }
+
+ fn open_index(&self) -> (IndexWriter, IndexReader) {
+ let config = Config::new();
+ let schema = DocFields::as_schema();
+
+ let dir = MmapDirectory::open(config.index_dir()).expect("Unable to create MmapDirectory");
+ let index = Index::open_or_create(dir, schema).expect("Unable to open / create directory");
+
+ let writer = index
+ .writer(50_000_000)
+ .expect("Unable to create index_writer");
+
+ // For a search server you will typically create on reader for the entire
+ // lifetime of your program.
+ let reader = index
+ .reader_builder()
+ .reload_policy(ReloadPolicy::OnCommit)
+ .try_into()
+ .expect("Unable to create reader");
+
+ (writer, reader)
+ }
+
+ fn update_index(
+ &self,
+ writer: &IndexWriter,
+ reader: &IndexReader,
+ doc_id: &str,
+ updated_url: &str,
+ ) {
+ let fields = DocFields::as_fields();
+
+ let searcher = reader.searcher();
+ let query = TermQuery::new(
+ Term::from_field_text(fields.id, doc_id),
+ IndexRecordOption::Basic,
+ );
+
+ let res = searcher
+ .search(&query, &TopDocs::with_limit(1))
+ .map_or(Vec::new(), |x| x)
+ .pop();
+
+ let doc = if let Some((_, doc_address)) = res {
+ searcher.doc(doc_address).ok()
+ } else {
+ None
+ };
+
+ // Doc exists! Lets remove it and update it with the new URL
+ if let Some(doc) = doc {
+ // Remove the old one
+ writer.delete_term(Term::from_field_text(fields.id, doc_id));
+ // Re-add the document w/ the updated domain & url
+ let mut new_doc = Document::default();
+ new_doc.add_text(fields.id, doc_id);
+ new_doc.add_text(fields.domain, "localhost");
+ new_doc.add_text(fields.url, updated_url);
+ // Everything else stays the same
+ new_doc.add_text(
+ fields.content,
+ doc.get_first(fields.content).unwrap().as_text().unwrap(),
+ );
+ new_doc.add_text(
+ fields.description,
+ doc.get_first(fields.description)
+ .unwrap()
+ .as_text()
+ .unwrap(),
+ );
+ new_doc.add_text(
+ fields.title,
+ doc.get_first(fields.title).unwrap().as_text().unwrap(),
+ );
+
+ let _ = writer.add_document(new_doc);
+ }
+ }
+}
+
+impl MigrationName for Migration {
+ fn name(&self) -> &str {
+ "m20221115_000001_local_file_pathfix"
+ }
+}
+
+#[async_trait::async_trait]
+impl MigrationTrait for Migration {
+ async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+ let (mut iwriter, ireader) = self.open_index();
+ let db = manager.get_connection();
+
+ println!("Updating crawl_queue");
+ let queued = crawl_queue::Entity::find()
+ .filter(crawl_queue::Column::Url.starts_with("file://"))
+ .all(db)
+ .await
+ .expect("Unable to query crawl_queue table.");
+
+ for doc in &queued {
+ if let Some(updated_url) = self.fix_url(&doc.url) {
+ let mut update: crawl_queue::ActiveModel = doc.to_owned().into();
+ update.domain = Set("localhost".to_string());
+ update.url = Set(updated_url);
+ let _ = update.save(db).await;
+ }
+ }
+
+ let docs = indexed_document::Entity::find()
+ .filter(indexed_document::Column::Url.starts_with("file://"))
+ .all(db)
+ .await
+ .expect("Unable to query indexed_document table.");
+
+ // No docs yet, nothing to migrate.
+ if docs.is_empty() {
+ return Ok(());
+ }
+
+ println!("Updating index");
+ for doc in &docs {
+ if let Some(updated_url) = self.fix_url(&doc.url) {
+ // Update the document in the index
+ self.update_index(&iwriter, &ireader, &doc.doc_id, &updated_url);
+
+ // Update document in the db
+ let mut update: indexed_document::ActiveModel = doc.to_owned().into();
+ update.domain = Set("localhost".to_string());
+ update.url = Set(updated_url.clone());
+ update.open_url = Set(Some(updated_url.clone()));
+ let _ = update.save(db).await;
+ }
+ }
+
+ if let Err(err) = iwriter.commit() {
+ return Err(DbErr::Custom(format!(
+ "Unable to save changes to index: {}",
+ err
+ )));
+ }
+
+ Ok(())
+ }
+
+ async fn down(&self, _: &SchemaManager) -> Result<(), DbErr> {
+ Ok(())
+ }
+}
diff --git a/crates/migrations/src/m20221116_000001_add_connection_constraint.rs b/crates/migrations/src/m20221116_000001_add_connection_constraint.rs
new file mode 100644
index 000000000..58929f06d
--- /dev/null
+++ b/crates/migrations/src/m20221116_000001_add_connection_constraint.rs
@@ -0,0 +1,32 @@
+use crate::sea_orm::Statement;
+use sea_orm_migration::prelude::*;
+use sea_orm_migration::sea_orm::ConnectionTrait;
+
+pub struct Migration;
+
+impl MigrationName for Migration {
+ fn name(&self) -> &str {
+ "m20221116_000001_add_connection_constraint"
+ }
+}
+
+#[async_trait::async_trait]
+impl MigrationTrait for Migration {
+ async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+ // Create index on (api_id, account). Should only every be one instance of a
+ // an account per service.
+ manager
+ .get_connection()
+ .execute(Statement::from_string(
+ manager.get_database_backend(),
+ "CREATE UNIQUE INDEX `idx-connections-api-id-account` ON `connections` (`api_id`, `account`);"
+ .to_string(),
+ ))
+ .await?;
+ Ok(())
+ }
+
+ async fn down(&self, _: &SchemaManager) -> Result<(), DbErr> {
+ Ok(())
+ }
+}
diff --git a/crates/shared/src/config.rs b/crates/shared/src/config.rs
index f320cf40b..76da88f03 100644
--- a/crates/shared/src/config.rs
+++ b/crates/shared/src/config.rs
@@ -218,8 +218,8 @@ impl Config {
match prefs_path.exists() {
true => {
- let mut settings: UserSettings =
- ron::from_str(&fs::read_to_string(prefs_path).unwrap())?;
+ let contents = &fs::read_to_string(prefs_path).unwrap_or_default();
+ let mut settings: UserSettings = ron::from_str(contents)?;
settings.constraint_limits();
Ok(settings)
}
@@ -228,7 +228,8 @@ impl Config {
// Write out default settings
fs::write(
prefs_path,
- ron::ser::to_string_pretty(&settings, Default::default()).unwrap(),
+ ron::ser::to_string_pretty(&settings, Default::default())
+ .expect("Unable to serialize settings."),
)
.expect("Unable to save user preferences file.");
@@ -278,7 +279,8 @@ impl Config {
}
pub fn default_data_dir() -> PathBuf {
- let proj_dirs = ProjectDirs::from("com", "athlabs", &Config::app_identifier()).unwrap();
+ let proj_dirs = ProjectDirs::from("com", "athlabs", &Config::app_identifier())
+ .expect("Unable to find a default data directory");
proj_dirs.data_dir().to_path_buf()
}
@@ -299,7 +301,8 @@ impl Config {
}
pub fn prefs_dir() -> PathBuf {
- let proj_dirs = ProjectDirs::from("com", "athlabs", &Config::app_identifier()).unwrap();
+ let proj_dirs = ProjectDirs::from("com", "athlabs", &Config::app_identifier())
+ .expect("Unable to find a suitable settings directory");
proj_dirs.preference_dir().to_path_buf()
}
diff --git a/crates/spyglass-plugin/src/utils.rs b/crates/spyglass-plugin/src/utils.rs
index 33cdcea87..9cd450cd9 100644
--- a/crates/spyglass-plugin/src/utils.rs
+++ b/crates/spyglass-plugin/src/utils.rs
@@ -3,20 +3,51 @@ use url::Url;
// Create a file URI
pub fn path_to_uri(path: PathBuf) -> String {
- let path_str = path.to_str().expect("Unable to convert path to string");
-
+ let path_str = path.display().to_string();
// Eventually this will be away to keep track of multiple devices and searching across
- // them. Might make sense to generate a UUID and assign to this computer(?) hostname
- // can be changed by the user.
- let host = if let Ok(hname) = std::env::var("HOST_NAME") {
- hname
- } else {
- "home.local".into()
- };
+ // them.
+ let host = "localhost";
let mut new_url = Url::parse("file://").expect("Base URI");
- let _ = new_url.set_host(Some(&host));
+ let _ = new_url.set_host(Some(host));
// Fixes issues handling windows drive letters
- new_url.set_path(&path_str.replace(':', "%3A"));
+ let path_str = path_str.replace(':', "%3A");
+ // Fixes an issue where DirEntry adds too many escapes.
+ let path_str = path_str.replace("\\\\", "\\");
+ new_url.set_path(&path_str);
new_url.to_string()
}
+
+#[cfg(test)]
+mod test {
+ use super::path_to_uri;
+ use std::path::Path;
+ use url::Url;
+
+ #[test]
+ fn test_path_to_uri() {
+ #[cfg(target_os = "windows")]
+ let test_folder = Path::new("C:\\tmp\\path_to_uri");
+
+ #[cfg(not(target_os = "windows"))]
+ let test_folder = Path::new("/tmp/path_to_uri");
+
+ std::fs::create_dir_all(test_folder).expect("Unable to create test dir");
+
+ let test_path = test_folder.join("test.txt");
+ let uri = path_to_uri(test_path.to_path_buf());
+
+ #[cfg(target_os = "windows")]
+ assert_eq!(uri, "file://localhost/C%3A/tmp/path_to_uri/test.txt");
+ #[cfg(not(target_os = "windows"))]
+ assert_eq!(uri, "file://localhost/tmp/path_to_uri/test.txt");
+
+ let url = Url::parse(&uri).unwrap();
+ let file_path = url.to_file_path().unwrap();
+ assert_eq!(file_path, test_path);
+
+ if test_folder.exists() {
+ std::fs::remove_dir_all(test_folder).expect("Unable to clean up test folder");
+ }
+ }
+}
diff --git a/crates/spyglass/Cargo.toml b/crates/spyglass/Cargo.toml
index e471a2793..76d322657 100644
--- a/crates/spyglass/Cargo.toml
+++ b/crates/spyglass/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "spyglass"
-version = "22.11.5"
+version = "22.11.6"
edition = "2021"
[dependencies]
@@ -34,6 +34,7 @@ reqwest = "0.11"
ron = "0.8"
rusqlite = { version = "*", features = ["bundled"] }
sentry = "0.28.0"
+sentry-tracing = "0.28.0"
serde = { version = "1.0", features = ["derive"] }
sha2 = "0.10"
shared = { path = "../shared" }
diff --git a/crates/spyglass/src/api/route.rs b/crates/spyglass/src/api/route.rs
index 80ab240e1..5643afb06 100644
--- a/crates/spyglass/src/api/route.rs
+++ b/crates/spyglass/src/api/route.rs
@@ -105,13 +105,20 @@ pub async fn authorize_connection(state: AppState, api_id: String) -> Result<(),
auth.scopes,
);
let res = new_conn.insert(&state.db).await;
- log::debug!("saved connection: {:?}", res);
- let _ = state
- .schedule_work(ManagerCommand::Collect(CollectTask::ConnectionSync {
- api_id,
- account: user.email,
- }))
- .await;
+ match res {
+ Ok(_) => {
+ log::debug!("saved connection {} for {}", user.email.clone(), api_id);
+ let _ = state
+ .schedule_work(ManagerCommand::Collect(
+ CollectTask::ConnectionSync {
+ api_id,
+ account: user.email,
+ },
+ ))
+ .await;
+ }
+ Err(err) => log::error!("Unable to save connection: {}", err.to_string()),
+ }
}
Err(err) => log::error!("unable to exchange token: {}", err),
}
@@ -380,8 +387,7 @@ pub async fn search(
.collect::>();
let docs =
- Searcher::search_with_lens(state.db.clone(), &applied, &index.reader, &search_req.query)
- .await;
+ Searcher::search_with_lens(state.db.clone(), &applied, index, &search_req.query).await;
let mut results: Vec = Vec::new();
for (score, doc_addr) in docs {
diff --git a/crates/spyglass/src/connection/mod.rs b/crates/spyglass/src/connection/mod.rs
index f5c3cc639..10bb53f15 100644
--- a/crates/spyglass/src/connection/mod.rs
+++ b/crates/spyglass/src/connection/mod.rs
@@ -32,10 +32,14 @@ pub async fn load_connection(
) -> Result> {
match api_id {
"calendar.google.com" => Ok(Box::new(
- gcal::GCalConnection::new(state, account).await.unwrap(),
+ gcal::GCalConnection::new(state, account)
+ .await
+ .expect("Unable to create gcal connection"),
)),
"drive.google.com" => Ok(Box::new(
- gdrive::DriveConnection::new(state, account).await.unwrap(),
+ gdrive::DriveConnection::new(state, account)
+ .await
+ .expect("Unable to create gdrive connection"),
)),
_ => Err(anyhow::anyhow!("Not suppported connection")),
}
diff --git a/crates/spyglass/src/crawler/mod.rs b/crates/spyglass/src/crawler/mod.rs
index d99dfa985..6b1656b0d 100644
--- a/crates/spyglass/src/crawler/mod.rs
+++ b/crates/spyglass/src/crawler/mod.rs
@@ -12,7 +12,6 @@ use url::{Host, Url};
use entities::models::{crawl_queue, fetch_history, tag};
use entities::sea_orm::prelude::*;
-use shared::url_to_file_path;
use crate::connection::load_connection;
use crate::crawler::bootstrap::create_archive_url;
@@ -340,24 +339,9 @@ impl Crawler {
url: &Url,
) -> Result {
// Attempt to convert from the URL to a file path
- #[allow(unused_assignments)]
- let mut url_path = url
- .to_file_path()
- .map(|p| p.display().to_string())
- .unwrap_or_else(|_| url.path().to_string());
+ let file_path = url.to_file_path().expect("Invalid file URL");
- #[cfg(not(target_os = "windows"))]
- {
- url_path = url_to_file_path(&url_path, false);
- }
-
- // Fixes issues handling Windows drive paths
- #[cfg(target_os = "windows")]
- {
- url_path = url_to_file_path(url.path(), true);
- }
-
- let path = Path::new(&url_path);
+ let path = Path::new(&file_path);
// Is this a file and does this exist?
if !path.exists() || !path.is_file() {
return Err(CrawlError::NotFound);
@@ -498,10 +482,11 @@ mod test {
use entities::models::{crawl_queue, resource_rule};
use entities::sea_orm::{ActiveModelTrait, Set};
use entities::test::setup_test_db;
+ use spyglass_plugin::utils::path_to_uri;
use crate::crawler::{determine_canonical, normalize_href, Crawler};
use crate::state::AppState;
-
+ use std::path::Path;
use url::Url;
#[tokio::test]
@@ -683,4 +668,37 @@ mod test {
let res = determine_canonical(&a, &b);
assert_eq!(res, "https://en.wikipedia.org/");
}
+
+ #[tokio::test]
+ async fn test_file_fetch() {
+ let crawler = Crawler::new();
+
+ let db = setup_test_db().await;
+ let state = AppState::builder().with_db(db).build();
+
+ #[cfg(target_os = "windows")]
+ let test_folder = Path::new("C:\\tmp\\path_to_uri");
+ #[cfg(not(target_os = "windows"))]
+ let test_folder = Path::new("/tmp/path_to_uri");
+
+ std::fs::create_dir_all(test_folder).expect("Unable to create test dir");
+
+ let test_path = test_folder.join("test.txt");
+ std::fs::write(test_path.clone(), "test_content").expect("Unable to write test file");
+
+ let uri = path_to_uri(test_path.to_path_buf());
+ let url = Url::parse(&uri).unwrap();
+
+ let query = crawl_queue::ActiveModel {
+ domain: Set("localhost".to_string()),
+ url: Set(url.to_string()),
+ crawl_type: Set(crawl_queue::CrawlType::Bootstrap),
+ ..Default::default()
+ };
+ let model = query.insert(&state.db).await.unwrap();
+
+ // Add resource rule to stop the crawl above
+ let res = crawler.fetch_by_job(&state, model.id, true).await;
+ assert!(res.is_ok());
+ }
}
diff --git a/crates/spyglass/src/main.rs b/crates/spyglass/src/main.rs
index fdabf6302..b555282f4 100644
--- a/crates/spyglass/src/main.rs
+++ b/crates/spyglass/src/main.rs
@@ -38,6 +38,7 @@ fn main() -> Result<(), Box> {
"https://5c1196909a4e4e5689406705be13aad3@o1334159.ingest.sentry.io/6600345",
sentry::ClientOptions {
release: sentry::release_name!(),
+ traces_sample_rate: 0.1,
..Default::default()
},
)))
@@ -65,7 +66,8 @@ fn main() -> Result<(), Box> {
.add_directive("docx=WARN".parse().expect("Invalid EnvFilter")),
)
.with(fmt::Layer::new().with_writer(io::stdout))
- .with(fmt::Layer::new().with_ansi(false).with_writer(non_blocking));
+ .with(fmt::Layer::new().with_ansi(false).with_writer(non_blocking))
+ .with(sentry_tracing::layer());
tracing::subscriber::set_global_default(subscriber).expect("Unable to set a global subscriber");
LogTracer::init()?;
@@ -112,7 +114,7 @@ fn main() -> Result<(), Box> {
async fn start_backend(state: &mut AppState, config: &Config) {
// Initialize crawl_queue, requeue all in-flight tasks.
- crawl_queue::reset_processing(&state.db).await;
+ let _ = crawl_queue::reset_processing(&state.db).await;
if let Err(e) = lens::reset(&state.db).await {
log::error!("Unable to reset lenses: {}", e);
}
diff --git a/crates/spyglass/src/plugin/exports.rs b/crates/spyglass/src/plugin/exports.rs
index 23d20aa31..24cd7c829 100644
--- a/crates/spyglass/src/plugin/exports.rs
+++ b/crates/spyglass/src/plugin/exports.rs
@@ -211,7 +211,9 @@ async fn handle_walk_and_enqueue(
path: PathBuf,
supported_exts: &HashSet,
) -> WalkStats {
- let walker = WalkBuilder::new(path).standard_filters(true).build();
+ let walker = WalkBuilder::new(path.clone())
+ .standard_filters(true)
+ .build();
let enqueue_settings = EnqueueSettings {
force_allow: true,
..Default::default()
@@ -231,7 +233,7 @@ async fn handle_walk_and_enqueue(
if let Some(ext) = ext {
if supported_exts.contains(ext) {
- to_enqueue.push(path_to_uri(entry.path().to_path_buf()));
+ to_enqueue.push(path_to_uri(entry.into_path()));
stats.files += 1;
} else {
stats.skipped += 1;
diff --git a/crates/spyglass/src/search/mod.rs b/crates/spyglass/src/search/mod.rs
index 5efa9765c..5ce76476c 100644
--- a/crates/spyglass/src/search/mod.rs
+++ b/crates/spyglass/src/search/mod.rs
@@ -184,15 +184,17 @@ impl Searcher {
pub async fn search_with_lens(
_db: DatabaseConnection,
applied_lenses: &Vec,
- reader: &IndexReader,
+ searcher: &Searcher,
query_string: &str,
) -> Vec {
let start_timer = Instant::now();
+ let index = &searcher.index;
+ let reader = &searcher.reader;
let fields = DocFields::as_fields();
let searcher = reader.searcher();
-
- let query = build_query(fields.clone(), query_string);
+ let tokenizers = index.tokenizers().clone();
+ let query = build_query(index.schema(), tokenizers, fields.clone(), query_string);
let mut allowed = Vec::new();
let mut skipped = Vec::new();
@@ -379,7 +381,7 @@ mod test {
_build_test_index(&mut searcher);
let query = "salinas";
- let results = Searcher::search_with_lens(db, &applied_lens, &searcher.reader, query).await;
+ let results = Searcher::search_with_lens(db, &applied_lens, &searcher, query).await;
assert_eq!(results.len(), 1);
}
@@ -404,7 +406,7 @@ mod test {
_build_test_index(&mut searcher);
let query = "salinas";
- let results = Searcher::search_with_lens(db, &applied_lens, &searcher.reader, query).await;
+ let results = Searcher::search_with_lens(db, &applied_lens, &searcher, query).await;
assert_eq!(results.len(), 1);
}
@@ -430,7 +432,7 @@ mod test {
_build_test_index(&mut searcher);
let query = "salinas";
- let results = Searcher::search_with_lens(db, &applied_lens, &searcher.reader, query).await;
+ let results = Searcher::search_with_lens(db, &applied_lens, &searcher, query).await;
assert_eq!(results.len(), 0);
}
}
diff --git a/crates/spyglass/src/search/query.rs b/crates/spyglass/src/search/query.rs
index 1f82f99c0..48fe7d277 100644
--- a/crates/spyglass/src/search/query.rs
+++ b/crates/spyglass/src/search/query.rs
@@ -1,15 +1,16 @@
-use tantivy::query::{BooleanQuery, BoostQuery, Occur, Query, TermQuery};
+use tantivy::query::{BooleanQuery, BoostQuery, Occur, PhraseQuery, Query, TermQuery};
use tantivy::schema::*;
+use tantivy::tokenizer::TokenizerManager;
use tantivy::Score;
use super::DocFields;
type QueryVec = Vec<(Occur, Box)>;
-fn _boosted_term(field: Field, term: &str, boost: Score) -> Box {
+fn _boosted_term(term: Term, boost: Score) -> Box {
Box::new(BoostQuery::new(
Box::new(TermQuery::new(
- Term::from_field_text(field, term),
+ term,
// Needs WithFreqs otherwise scoring is wonky.
IndexRecordOption::WithFreqs,
)),
@@ -17,34 +18,73 @@ fn _boosted_term(field: Field, term: &str, boost: Score) -> Box {
))
}
-pub fn build_query(fields: DocFields, query_string: &str) -> BooleanQuery {
- // Tokenize query string
- let query_string = query_string.to_lowercase();
- let terms: Vec<&str> = query_string
- .split(' ')
- .into_iter()
- .map(|token| token.trim())
- .collect();
+fn _boosted_phrase(terms: Vec, boost: Score) -> Box {
+ Box::new(BoostQuery::new(Box::new(PhraseQuery::new(terms)), boost))
+}
+
+pub fn build_query(
+ schema: Schema,
+ tokenizers: TokenizerManager,
+ fields: DocFields,
+ query_string: &str,
+) -> BooleanQuery {
+ let content_terms = terms_for_field(&schema, &tokenizers, query_string, fields.content);
+ let title_terms: Vec = terms_for_field(&schema, &tokenizers, query_string, fields.title);
let mut term_query: QueryVec = Vec::new();
// Boost exact matches to the full query string
- if terms.len() > 1 {
- term_query.push((
- Occur::Should,
- _boosted_term(fields.title, &query_string, 5.0),
- ));
- term_query.push((
- Occur::Should,
- _boosted_term(fields.content, &query_string, 5.0),
- ));
+ if content_terms.len() > 1 {
+ // boosting phrases relative to the number of segments in a
+ // continuous phrase
+ let boost = 2.0 * content_terms.len() as f32;
+ term_query.push((Occur::Should, _boosted_phrase(content_terms.clone(), boost)));
+ }
+
+ // Boost exact matches to the full query string
+ if title_terms.len() > 1 {
+ // boosting phrases relative to the number of segments in a
+ // continuous phrase, base score higher for title
+ // than content
+ let boost = 2.5 * title_terms.len() as f32;
+ term_query.push((Occur::Should, _boosted_phrase(title_terms.clone(), boost)));
+ }
+
+ for term in content_terms {
+ term_query.push((Occur::Should, _boosted_term(term, 1.0)));
}
- for term in terms {
- // Emphasize matches in the title more than words in the content
- term_query.push((Occur::Should, _boosted_term(fields.content, term, 1.0)));
- term_query.push((Occur::Should, _boosted_term(fields.title, term, 5.0)));
+ for term in title_terms {
+ term_query.push((Occur::Should, _boosted_term(term, 2.0)));
}
BooleanQuery::new(vec![(Occur::Must, Box::new(BooleanQuery::new(term_query)))])
}
+
+/**
+ * Responsible for parsing the input query for a particular field. The tokenizer for the field
+ * is used to ensure consistent tokens between indexing and queries.
+ */
+fn terms_for_field(
+ schema: &Schema,
+ tokenizers: &TokenizerManager,
+ query: &str,
+ field: Field,
+) -> Vec {
+ let mut terms = Vec::new();
+
+ let field_entry = schema.get_field_entry(field);
+ let field_type = field_entry.field_type();
+ if let FieldType::Str(ref str_options) = field_type {
+ let option = str_options.get_indexing_options().unwrap();
+ let text_analyzer = tokenizers.get(option.tokenizer()).unwrap();
+
+ let mut token_stream = text_analyzer.token_stream(query);
+ token_stream.process(&mut |token| {
+ let term = Term::from_field_text(field, &token.text);
+ terms.push(term);
+ });
+ }
+
+ terms
+}
diff --git a/crates/spyglass/src/task/worker.rs b/crates/spyglass/src/task/worker.rs
index e43b52ea4..92b13939a 100644
--- a/crates/spyglass/src/task/worker.rs
+++ b/crates/spyglass/src/task/worker.rs
@@ -98,7 +98,10 @@ pub async fn handle_fetch(state: AppState, task: CrawlTask) -> FetchResult {
// Add / update search index w/ crawl result.
if let Some(content) = crawl_result.content {
let url = Url::parse(&crawl_result.url).expect("Invalid crawl URL");
- let url_host = url.host_str().expect("Invalid URL host");
+ let url_host = match url.scheme() {
+ "file" => "localhost",
+ _ => url.host_str().expect("Invalid URL host"),
+ };
let existing = indexed_document::Entity::find()
.filter(indexed_document::Column::Url.eq(url.as_str()))
diff --git a/crates/tauri/Cargo.toml b/crates/tauri/Cargo.toml
index 351bb0e44..0c8c0e25d 100644
--- a/crates/tauri/Cargo.toml
+++ b/crates/tauri/Cargo.toml
@@ -24,7 +24,8 @@ num-format = "0.4"
open = "3"
reqwest = { version = "0.11", features = ["json"] }
ron = "0.8"
-sentry = "0.27.0"
+sentry = "0.28.0"
+sentry-tracing = "0.28.0"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
shared = { path = "../shared" }
@@ -42,6 +43,7 @@ url = "2.2"
[target.'cfg(target_os = "macos")'.dependencies]
cocoa = "0.24"
+objc = "0.2.7"
[features]
default = [ "custom-protocol" ]
diff --git a/crates/tauri/src/cmd.rs b/crates/tauri/src/cmd.rs
index 0809b1642..c8ffecc24 100644
--- a/crates/tauri/src/cmd.rs
+++ b/crates/tauri/src/cmd.rs
@@ -81,12 +81,17 @@ pub async fn open_result(_: tauri::Window, url: &str) -> Result<(), String> {
{
use shared::url_to_file_path;
let path = url_to_file_path(url.path(), true);
- open::that(format!("file://{}", path)).unwrap();
+ if let Err(err) = open::that(format!("file://{}", path)) {
+ log::error!("Unable to open file://{} due to: {}", path, err);
+ }
+
return Ok(());
}
}
- open::that(url.to_string()).unwrap();
+ if let Err(err) = open::that(url.to_string()) {
+ log::error!("Unable to open {} due to: {}", url.to_string(), err);
+ }
}
Ok(())
}
diff --git a/crates/tauri/src/main.rs b/crates/tauri/src/main.rs
index dc6612346..0fd5fb003 100644
--- a/crates/tauri/src/main.rs
+++ b/crates/tauri/src/main.rs
@@ -31,15 +31,21 @@ mod cmd;
mod constants;
mod menu;
use menu::MenuID;
+mod platform;
mod plugins;
mod rpc;
mod window;
use window::{
show_connection_manager_window, show_crawl_stats_window, show_lens_manager_window,
- show_plugin_manager, show_search_bar, show_user_settings, show_wizard_window,
+ show_plugin_manager, show_search_bar, show_update_window, show_user_settings,
+ show_wizard_window,
};
-use crate::window::show_update_window;
+const LOG_LEVEL: tracing::Level = tracing::Level::INFO;
+#[cfg(not(debug_assertions))]
+const SPYGLASS_LEVEL: &str = "spyglass_app=INFO";
+#[cfg(debug_assertions)]
+const SPYGLASS_LEVEL: &str = "spyglass_app=DEBUG";
#[derive(Clone)]
pub struct AppShutdown;
@@ -56,6 +62,7 @@ fn main() -> Result<(), Box> {
"https://13d7d51a8293459abd0aba88f99f4c18@o1334159.ingest.sentry.io/6600471",
sentry::ClientOptions {
release: Some(Cow::from(ctx.package_info().version.to_string())),
+ traces_sample_rate: 0.1,
..Default::default()
},
)))
@@ -83,13 +90,18 @@ fn main() -> Result<(), Box> {
let (non_blocking, _guard) = tracing_appender::non_blocking(file_appender);
let subscriber = tracing_subscriber::registry()
- .with(EnvFilter::from_default_env().add_directive(tracing::Level::INFO.into()))
+ .with(
+ EnvFilter::from_default_env()
+ .add_directive(LOG_LEVEL.into())
+ .add_directive(SPYGLASS_LEVEL.parse().expect("Invalid EnvFilter")),
+ )
.with(
fmt::Layer::new()
.with_thread_names(true)
.with_writer(io::stdout),
)
- .with(fmt::Layer::new().with_ansi(false).with_writer(non_blocking));
+ .with(fmt::Layer::new().with_ansi(false).with_writer(non_blocking))
+ .with(sentry_tracing::layer());
tracing::subscriber::set_global_default(subscriber).expect("Unable to set a global subscriber");
LogTracer::init()?;
@@ -141,13 +153,8 @@ fn main() -> Result<(), Box> {
log::error!("Unable to copy default plugins: {}", e);
}
- // macOS: hide from dock (also hides menu bar)
- #[cfg(target_os = "macos")]
- app.set_activation_policy(tauri::ActivationPolicy::Accessory);
-
let window = app.get_window(constants::SEARCH_WIN_NAME).expect("Main window not found");
window::center_search_bar(&window);
-
// macOS: Handle multiple spaces correctly
#[cfg(target_os = "macos")]
{
@@ -176,14 +183,7 @@ fn main() -> Result<(), Box> {
if let Err(e) = shortcuts
.register(&config.user_settings.shortcut, move || {
let window = window_clone.clone();
- let _ = window.is_visible()
- .map(|is_visible| {
- if is_visible {
- window::hide_search_bar(&window);
- } else {
- window::show_search_bar(&window);
- }
- });
+ window::show_search_bar(&window);
}) {
window::alert(&window, "Error registering global shortcut", &format!("{}", e));
}
@@ -192,17 +192,17 @@ fn main() -> Result<(), Box> {
Err(e) => window::alert(&window_clone, "Error registering global shortcut", &format!("{}", e))
}
- // Hide searchbar on start.
- let _ = window.hide();
Ok(())
})
- .on_window_event(|event| {
- let window = event.window();
- if window.label() == "main" {
- if let tauri::WindowEvent::Focused(is_focused) = event.event() {
- if !is_focused {
- let handle = event.window();
- window::hide_search_bar(handle);
+ .on_window_event(|_event| {
+ #[cfg(target_os = "macos")]
+ {
+ let window = _event.window();
+ if window.label() == constants::SEARCH_WIN_NAME {
+ if let tauri::WindowEvent::Focused(is_focused) = _event.event() {
+ if !is_focused {
+ window::hide_search_bar(window);
+ }
}
}
}
@@ -233,17 +233,8 @@ fn main() -> Result<(), Box> {
MenuID::OPEN_LOGS_FOLDER => open_folder(config.logs_dir()),
MenuID::OPEN_SETTINGS_MANAGER => { show_user_settings(app) },
MenuID::OPEN_WIZARD => { show_wizard_window(app); }
- MenuID::SHOW_CRAWL_STATUS => {
- show_crawl_stats_window(app);
- }
- MenuID::SHOW_SEARCHBAR => {
- let _ = window.is_visible()
- .map(|is_visible| {
- if !is_visible {
- window::hide_search_bar(&window);
- }
- });
- }
+ MenuID::SHOW_CRAWL_STATUS => { show_crawl_stats_window(app); }
+ MenuID::SHOW_SEARCHBAR => { window::show_search_bar(&window); }
MenuID::QUIT => app.exit(0),
MenuID::DEV_SHOW_CONSOLE => window.open_devtools(),
MenuID::JOIN_DISCORD => {
@@ -261,6 +252,14 @@ fn main() -> Result<(), Box> {
.expect("error while running tauri application");
app.run(|app_handle, e| match e {
+ RunEvent::MainEventsCleared => {
+ #[cfg(target_os = "macos")]
+ {
+ if let Some(window) = app_handle.get_window(constants::SEARCH_WIN_NAME) {
+ crate::platform::mac::poll_app_events(&window);
+ }
+ }
+ }
RunEvent::ExitRequested { .. } => {
// Do some cleanup for long running tasks
let shutdown_tx = app_handle.state::>();
diff --git a/crates/tauri/src/menu.rs b/crates/tauri/src/menu.rs
index fd3196cae..cca84401d 100644
--- a/crates/tauri/src/menu.rs
+++ b/crates/tauri/src/menu.rs
@@ -113,6 +113,7 @@ pub fn get_app_menu(ctx: &Context) -> Menu {
.add_native_item(MenuItem::Paste)
.add_native_item(MenuItem::SelectAll)
.add_native_item(MenuItem::Separator)
+ .add_native_item(MenuItem::Hide)
.add_native_item(MenuItem::Quit),
))
}
diff --git a/crates/tauri/src/platform/linux.rs b/crates/tauri/src/platform/linux.rs
new file mode 100644
index 000000000..8a1470cc4
--- /dev/null
+++ b/crates/tauri/src/platform/linux.rs
@@ -0,0 +1,17 @@
+use crate::window;
+use shared::event::ClientEvent;
+use tauri::Window;
+
+pub fn show_search_bar(window: &Window) {
+ let _ = window.unminimize();
+ window::center_search_bar(window);
+ let _ = window.set_focus();
+ let _ = window.set_always_on_top(true);
+
+ let _ = window.emit(ClientEvent::FocusWindow.as_ref(), true);
+}
+
+pub fn hide_search_bar(window: &Window) {
+ let _ = window.minimize();
+ let _ = window.emit(ClientEvent::ClearSearch.as_ref(), true);
+}
diff --git a/crates/tauri/src/platform/mac.rs b/crates/tauri/src/platform/mac.rs
new file mode 100644
index 000000000..aae024db5
--- /dev/null
+++ b/crates/tauri/src/platform/mac.rs
@@ -0,0 +1,51 @@
+use cocoa::appkit::{NSApp, NSApplication, NSEvent, NSEventMask, NSEventSubtype};
+use cocoa::base::nil;
+use cocoa::foundation::{NSAutoreleasePool, NSDate, NSString};
+
+use tauri::Window;
+
+use crate::window;
+use shared::event::ClientEvent;
+
+/// Poll for dock events
+pub fn poll_app_events(window: &Window) {
+ unsafe {
+ let _pool = NSAutoreleasePool::new(nil);
+ let ns_event = NSApp().nextEventMatchingMask_untilDate_inMode_dequeue_(
+ NSEventMask::NSAppKitDefinedMask.bits(),
+ NSDate::distantPast(cocoa::base::nil),
+ // Use custom event loop name so we don't trampled on others.
+ NSString::alloc(nil).init_str("spyglassEventLoop"),
+ cocoa::base::YES,
+ );
+
+ if ns_event == nil || ns_event.eventType() as u64 == 21 {
+ return;
+ }
+
+ let subtype = ns_event.subtype();
+ match subtype {
+ NSEventSubtype::NSApplicationActivatedEventType => {
+ window::show_search_bar(window);
+ }
+ NSEventSubtype::NSApplicationDeactivatedEventType => {
+ window::hide_search_bar(window);
+ }
+ _ => {}
+ }
+ }
+}
+
+pub fn show_search_bar(window: &Window) {
+ let _ = window.show();
+ window::center_search_bar(window);
+ let _ = window.set_always_on_top(true);
+ let _ = window.set_focus();
+
+ let _ = window.emit(ClientEvent::FocusWindow.as_ref(), true);
+}
+
+pub fn hide_search_bar(window: &Window) {
+ let _ = window.hide();
+ let _ = window.emit(ClientEvent::ClearSearch.as_ref(), true);
+}
diff --git a/crates/tauri/src/platform/mod.rs b/crates/tauri/src/platform/mod.rs
new file mode 100644
index 000000000..a0b6368ab
--- /dev/null
+++ b/crates/tauri/src/platform/mod.rs
@@ -0,0 +1,7 @@
+/// Platform specific implementation of things
+#[cfg(target_os = "linux")]
+pub mod linux;
+#[cfg(target_os = "macos")]
+pub mod mac;
+#[cfg(target_os = "windows")]
+pub mod windows;
diff --git a/crates/tauri/src/platform/windows.rs b/crates/tauri/src/platform/windows.rs
new file mode 100644
index 000000000..2c9d1b433
--- /dev/null
+++ b/crates/tauri/src/platform/windows.rs
@@ -0,0 +1,16 @@
+use crate::window;
+use shared::event::ClientEvent;
+use tauri::Window;
+
+pub fn show_search_bar(window: &Window) {
+ let _ = window.unminimize();
+ window::center_search_bar(window);
+ let _ = window.set_focus();
+
+ let _ = window.emit(ClientEvent::FocusWindow.as_ref(), true);
+}
+
+pub fn hide_search_bar(window: &Window) {
+ let _ = window.minimize();
+ let _ = window.emit(ClientEvent::ClearSearch.as_ref(), true);
+}
diff --git a/crates/tauri/src/plugins/lens_updater.rs b/crates/tauri/src/plugins/lens_updater.rs
index 0af859b39..a10b85576 100644
--- a/crates/tauri/src/plugins/lens_updater.rs
+++ b/crates/tauri/src/plugins/lens_updater.rs
@@ -129,8 +129,12 @@ pub async fn install_lens_to_path(download_url: &str, lens_folder: PathBuf) -> a
// Grab the file name from the end of the URL
let url = Url::parse(download_url)?;
- let mut segments = url.path_segments().map(|c| c.collect::>()).unwrap();
- let file_name = segments.pop().unwrap();
+ let file_name = url
+ .path_segments()
+ .map(|c| c.collect::>())
+ .and_then(|mut segs| segs.pop())
+ .expect("Unable to determine filename from lens path");
+
// Write file out to lens folder
fs::write(lens_folder.join(file_name), file_contents)?;
diff --git a/crates/tauri/src/window.rs b/crates/tauri/src/window.rs
index 7b9efedd8..7d08eb918 100644
--- a/crates/tauri/src/window.rs
+++ b/crates/tauri/src/window.rs
@@ -1,4 +1,4 @@
-use crate::constants;
+use crate::{constants, platform};
use shared::event::ClientEvent;
use tauri::api::dialog::{MessageDialogBuilder, MessageDialogButtons, MessageDialogKind};
use tauri::{AppHandle, LogicalSize, Manager, Monitor, Size, Window, WindowBuilder, WindowUrl};
@@ -38,16 +38,25 @@ pub fn center_search_bar(window: &Window) {
}
pub fn show_search_bar(window: &Window) {
- let _ = window.emit(ClientEvent::FocusWindow.as_ref(), true);
- let _ = window.show();
- let _ = window.set_focus();
- let _ = window.set_always_on_top(true);
- center_search_bar(window);
+ #[cfg(target_os = "linux")]
+ platform::linux::show_search_bar(window);
+
+ #[cfg(target_os = "macos")]
+ platform::mac::show_search_bar(window);
+
+ #[cfg(target_os = "windows")]
+ platform::windows::show_search_bar(window);
}
pub fn hide_search_bar(window: &Window) {
- let _ = window.hide();
- let _ = window.emit(ClientEvent::ClearSearch.as_ref(), true);
+ #[cfg(target_os = "linux")]
+ platform::linux::hide_search_bar(window);
+
+ #[cfg(target_os = "macos")]
+ platform::mac::hide_search_bar(window);
+
+ #[cfg(target_os = "windows")]
+ platform::windows::hide_search_bar(window);
}
pub async fn resize_window(window: &Window, height: f64) {
diff --git a/crates/tauri/tauri.conf.json b/crates/tauri/tauri.conf.json
index e8c8066e0..3343b82f4 100644
--- a/crates/tauri/tauri.conf.json
+++ b/crates/tauri/tauri.conf.json
@@ -1,7 +1,7 @@
{
"package": {
"productName": "Spyglass",
- "version": "22.11.5"
+ "version": "22.11.6"
},
"build": {
"distDir": "../client/dist",
@@ -74,7 +74,7 @@
"width": 640,
"height": 96,
"transparent": true,
- "skipTaskbar": true
+ "skipTaskbar": false
}],
"security": {
"csp": "default-src blob: data: filesystem: ws: wss: http: https: tauri: 'unsafe-eval' 'unsafe-inline' 'self'; script-src 'unsafe-eval'"