+
{result.title.clone()}
-
+
{result.description.clone()}
@@ -150,10 +153,10 @@ pub fn search_result_component(props: &SearchResultProps) -> Html {
ResultListType::LensSearch => {
html! {
-
+
{result.title.clone()}
-
+
{result.description.clone()}
diff --git a/crates/client/src/icons.rs b/crates/client/src/icons.rs
new file mode 100644
index 000000000..e69de29bb
diff --git a/crates/client/src/main.rs b/crates/client/src/main.rs
index 82af93c2b..2cb2444f5 100644
--- a/crates/client/src/main.rs
+++ b/crates/client/src/main.rs
@@ -7,13 +7,22 @@ mod constants;
mod events;
mod pages;
-use crate::pages::{SearchPage, StatsPage};
+use crate::pages::{LensManagerPage, SearchPage, StatsPage};
#[wasm_bindgen(module = "/public/glue.js")]
extern "C" {
#[wasm_bindgen(js_name = "deleteDoc", catch)]
pub async fn delete_doc(id: String) -> Result<(), JsValue>;
+ #[wasm_bindgen(catch)]
+ pub async fn install_lens(download_url: String) -> Result<(), JsValue>;
+
+ #[wasm_bindgen(js_name = "listInstalledLenses", catch)]
+ pub async fn list_installed_lenses() -> Result;
+
+ #[wasm_bindgen(js_name = "listInstallableLenses", catch)]
+ pub async fn list_installable_lenses() -> Result;
+
#[wasm_bindgen(js_name = "searchDocs", catch)]
pub async fn search_docs(lenses: JsValue, query: String) -> Result;
@@ -29,9 +38,15 @@ extern "C" {
#[wasm_bindgen(js_name = "onRefreshResults")]
pub async fn on_refresh_results(callback: &Closure);
+ #[wasm_bindgen]
+ pub async fn on_refresh_lens_manager(callback: &Closure);
+
#[wasm_bindgen(js_name = "openResult", catch)]
pub async fn open(url: String) -> Result<(), JsValue>;
+ #[wasm_bindgen(js_name = "openLensFolder", catch)]
+ pub async fn open_lens_folder() -> Result<(), JsValue>;
+
#[wasm_bindgen(js_name = "escape", catch)]
pub async fn escape() -> Result<(), JsValue>;
@@ -46,6 +61,8 @@ extern "C" {
enum Route {
#[at("/")]
Search,
+ #[at("/settings/lens")]
+ LensManager,
#[at("/stats")]
Status,
}
@@ -66,6 +83,7 @@ pub fn app() -> Html {
fn switch(routes: &Route) -> Html {
match routes {
+ Route::LensManager => html! { >,
+ req_state: UseStateHandle,
+) {
+ spawn_local(async move {
+ match list_installed_lenses().await {
+ Ok(results) => {
+ lenses_handle.set(results.into_serde().unwrap());
+ req_state.set(RequestState::Finished);
+ }
+ Err(e) => {
+ log::info!("Error: {:?}", e);
+ req_state.set(RequestState::Error);
+ }
+ }
+ });
+}
+
+fn fetch_installable_lenses(
+ data_handle: UseStateHandle>,
+ req_state: UseStateHandle,
+) {
+ spawn_local(async move {
+ match list_installable_lenses().await {
+ Ok(results) => {
+ let lenses: Vec = results.into_serde().unwrap();
+ let parsed: Vec = lenses
+ .iter()
+ .map(|lens| LensResult {
+ author: lens.author.to_owned(),
+ title: lens.name.to_owned(),
+ description: lens.description.to_owned(),
+ html_url: Some(lens.html_url.to_owned()),
+ download_url: Some(lens.download_url.to_owned()),
+ })
+ .collect();
+
+ data_handle.set(parsed);
+ req_state.set(RequestState::Finished);
+ }
+ Err(e) => {
+ log::info!("Error: {:?}", e);
+ req_state.set(RequestState::Error);
+ }
+ }
+ });
+}
+
+fn handle_install_lens(download_url: String) {
+ spawn_local(async move {
+ if let Err(e) = install_lens(download_url.clone()).await {
+ log::error!("error installing lens: {} {:?}", download_url.clone(), e);
+ }
+ });
+}
+
+#[derive(Properties, PartialEq)]
+pub struct InstallBtnProps {
+ pub download_url: String,
+}
+
+#[function_component(InstallButton)]
+pub fn install_btn(props: &InstallBtnProps) -> Html {
+ let is_installing = use_state_eq(|| false);
+ let download_url = props.download_url.clone();
+
+ let onclick = {
+ let is_installing = is_installing.clone();
+ move |_| {
+ is_installing.set(true);
+ // Download to lens directory
+ handle_install_lens(download_url.clone());
+ }
+ };
+
+ if *is_installing {
+ html! {
+
+ = vec![
+ "border-t".into(),
+ "border-neutral-600".into(),
+ "p-4".into(),
+ "pr-0".into(),
+ "text-white".into(),
+ "bg-netural-800".into(),
+ ];
+ let result = &props.result;
+
+ let installed_el = if props.is_installed {
+ html! {
+
+
+ {result.title.clone()}
+
+
+ {"Crafted By:"}
+
+ {format!("@{}", result.author)}
+
+
+
+ {result.description.clone()}
+
+
+ {installed_el}
+ {view_link}
+
+
+ }
+}
+
+#[function_component(LensManagerPage)]
+pub fn lens_manager_page() -> Html {
+ let user_installed: UseStateHandle> = use_state_eq(Vec::new);
+ let installable: UseStateHandle> = use_state_eq(Vec::new);
+
+ let ui_req_state = use_state_eq(|| RequestState::NotStarted);
+ if *ui_req_state == RequestState::NotStarted {
+ ui_req_state.set(RequestState::InProgress);
+ fetch_installed_lenses(user_installed.clone(), ui_req_state.clone());
+ }
+
+ let i_req_state = use_state_eq(|| RequestState::NotStarted);
+ if *i_req_state == RequestState::NotStarted {
+ i_req_state.set(RequestState::InProgress);
+ fetch_installable_lenses(installable.clone(), i_req_state.clone());
+ }
+
+ let on_open_folder = {
+ move |_| {
+ spawn_local(async {
+ let _ = crate::open_lens_folder().await;
+ });
+ }
+ };
+
+ let on_refresh = {
+ let ui_req_state = ui_req_state.clone();
+ let i_req_state = i_req_state.clone();
+ move |_| {
+ ui_req_state.set(RequestState::NotStarted);
+ i_req_state.set(RequestState::NotStarted);
+ }
+ };
+
+ let already_installed: HashSet =
+ user_installed.iter().map(|x| x.title.clone()).collect();
+ installable.set(
+ installable
+ .iter()
+ .filter(|x| !already_installed.contains(&x.title))
+ .map(|x| x.to_owned())
+ .collect::>(),
+ );
+
+ // Handle refreshing the list
+ {
+ let ui_req_state = ui_req_state.clone();
+ let i_req_state = i_req_state.clone();
+ spawn_local(async move {
+ let cb = Closure::wrap(Box::new(move || {
+ ui_req_state.set(RequestState::NotStarted);
+ i_req_state.set(RequestState::NotStarted);
+ }) as Box);
+
+ on_refresh_lens_manager(&cb).await;
+ cb.forget();
+ });
+ }
+
+ let contents = if ui_req_state.is_done() && i_req_state.is_done() {
+ html! {
+ <>
+ {
+ user_installed.iter().map(|data| {
+ html! {{"Lens Manager"}
+
+
+
+
+
+ {contents}
+
+
+ }
+}
diff --git a/crates/client/src/pages/mod.rs b/crates/client/src/pages/mod.rs
index 44cbc773d..ef47f47b6 100644
--- a/crates/client/src/pages/mod.rs
+++ b/crates/client/src/pages/mod.rs
@@ -1,3 +1,6 @@
+mod lens_manager;
+pub use lens_manager::*;
+
mod search;
pub use search::*;
diff --git a/crates/client/src/pages/search.rs b/crates/client/src/pages/search.rs
index b9c3bae1c..3b325926a 100644
--- a/crates/client/src/pages/search.rs
+++ b/crates/client/src/pages/search.rs
@@ -172,18 +172,18 @@ pub fn search_page() -> Html {
html! {
-
+
>) {
+fn fetch_crawl_stats(
+ stats_handle: UseStateHandle>,
+ request_finished: UseStateHandle,
+) {
spawn_local(async move {
match crawl_stats().await {
Ok(results) => {
@@ -13,8 +17,12 @@ fn fetch_crawl_stats(stats_handle: UseStateHandle>) {
let mut sorted = results.by_domain;
sorted.sort_by(|(_, a), (_, b)| b.num_completed.cmp(&a.num_completed));
stats_handle.set(sorted);
+ request_finished.set(true);
+ }
+ Err(e) => {
+ log::info!("Error: {:?}", e);
+ request_finished.set(true);
}
- Err(e) => log::info!("Error: {:?}", e),
}
});
}
@@ -39,7 +47,7 @@ pub fn legend_icon(props: &LegendIconProps) -> Html {
];
html! {
-
+
{props.label.clone()}
@@ -82,7 +90,7 @@ fn stats_bar(props: &StatsBarProps) -> Html {
html! {
}
}
@@ -90,15 +98,19 @@ fn stats_bar(props: &StatsBarProps) -> Html {
#[function_component(StatsPage)]
pub fn stats_page() -> Html {
let stats: UseStateHandle> = use_state_eq(Vec::new);
- if stats.is_empty() {
- fetch_crawl_stats(stats.clone());
+ let request_finished = use_state(|| false);
+
+ if stats.is_empty() && !(*request_finished) {
+ fetch_crawl_stats(stats.clone(), request_finished.clone());
}
let onclick = {
+ let request_finished = request_finished.clone();
let stats = stats.clone();
move |_| {
+ request_finished.set(false);
stats.set(Vec::new());
- fetch_crawl_stats(stats.clone());
+ fetch_crawl_stats(stats.clone(), request_finished.clone());
}
};
@@ -107,11 +119,11 @@ pub fn stats_page() -> Html {
.map(|(domain, stats)| {
let total = stats.total() as f64;
html! {
-
-
+
+
{domain}
-
+
+
{"Crawl Status"}
@@ -158,7 +174,7 @@ pub fn stats_page() -> Html {
-
+
{rendered}
diff --git a/crates/entities/src/models/crawl_queue.rs b/crates/entities/src/models/crawl_queue.rs
index 3fc69c639..9ace3ffd7 100644
--- a/crates/entities/src/models/crawl_queue.rs
+++ b/crates/entities/src/models/crawl_queue.rs
@@ -152,7 +152,7 @@ struct CrawlQueueCount {
count: i64,
}
-#[derive(FromQueryResult)]
+#[derive(Debug, FromQueryResult)]
pub struct QueueCountByStatus {
pub count: i64,
pub domain: String,
diff --git a/crates/entities/src/models/indexed_document.rs b/crates/entities/src/models/indexed_document.rs
index e7500f7f3..3bd10626a 100644
--- a/crates/entities/src/models/indexed_document.rs
+++ b/crates/entities/src/models/indexed_document.rs
@@ -46,7 +46,7 @@ impl ActiveModelBehavior for ActiveModel {
}
}
-#[derive(FromQueryResult)]
+#[derive(Debug, FromQueryResult)]
pub struct CountByDomain {
pub count: i64,
pub domain: String,
diff --git a/crates/shared/src/response.rs b/crates/shared/src/response.rs
index 5ab26460d..e54be89a4 100644
--- a/crates/shared/src/response.rs
+++ b/crates/shared/src/response.rs
@@ -25,6 +25,26 @@ pub struct CrawlStats {
pub by_domain: Vec<(String, QueueStatus)>,
}
+#[derive(Clone, Deserialize, Serialize)]
+pub struct InstallableLens {
+ pub author: String,
+ pub description: String,
+ pub name: String,
+ pub sha: String,
+ pub download_url: String,
+ pub html_url: String,
+}
+
+#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)]
+pub struct LensResult {
+ pub author: String,
+ pub title: String,
+ pub description: String,
+ // Only relevant for installable lenses
+ pub html_url: Option,
+ pub download_url: Option,
+}
+
#[derive(Debug, Deserialize, Serialize)]
pub struct SearchMeta {
pub query: String,
@@ -48,13 +68,7 @@ pub struct SearchResults {
pub meta: SearchMeta,
}
-#[derive(Clone, Debug, Deserialize, Serialize)]
-pub struct LensResult {
- pub title: String,
- pub description: String,
-}
-
-#[derive(Debug, Deserialize, Serialize)]
+#[derive(Debug, Default, Deserialize, Serialize)]
pub struct SearchLensesResp {
pub results: Vec,
}
diff --git a/crates/shared/src/rpc.rs b/crates/shared/src/rpc.rs
index 098f20a4e..e4d142e39 100644
--- a/crates/shared/src/rpc.rs
+++ b/crates/shared/src/rpc.rs
@@ -2,7 +2,7 @@ use jsonrpc_core::{BoxFuture, Result};
use jsonrpc_derive::rpc;
use crate::request::{SearchLensesParam, SearchParam};
-use crate::response::{AppStatus, CrawlStats, SearchLensesResp, SearchResults};
+use crate::response::{AppStatus, CrawlStats, LensResult, SearchLensesResp, SearchResults};
pub fn gen_ipc_path() -> String {
if cfg!(windows) {
@@ -28,12 +28,15 @@ pub trait Rpc {
#[rpc(name = "delete_doc")]
fn delete_doc(&self, id: String) -> BoxFuture>;
- #[rpc(name = "toggle_pause")]
- fn toggle_pause(&self) -> BoxFuture>;
+ #[rpc(name = "list_installed_lenses")]
+ fn list_installed_lenses(&self) -> BoxFuture>>;
#[rpc(name = "search_docs")]
fn search_docs(&self, query: SearchParam) -> BoxFuture>;
#[rpc(name = "search_lenses")]
fn search_lenses(&self, query: SearchLensesParam) -> BoxFuture>;
+
+ #[rpc(name = "toggle_pause")]
+ fn toggle_pause(&self) -> BoxFuture>;
}
diff --git a/crates/spyglass/src/api/mod.rs b/crates/spyglass/src/api/mod.rs
index 953d154c0..cac18a252 100644
--- a/crates/spyglass/src/api/mod.rs
+++ b/crates/spyglass/src/api/mod.rs
@@ -6,7 +6,7 @@ use jsonrpc_ipc_server::{Server, ServerBuilder};
use libspyglass::state::AppState;
use shared::request::{SearchLensesParam, SearchParam};
-use shared::response::{AppStatus, CrawlStats, SearchLensesResp, SearchResults};
+use shared::response::{AppStatus, CrawlStats, LensResult, SearchLensesResp, SearchResults};
use shared::rpc::{gen_ipc_path, Rpc};
mod response;
@@ -33,8 +33,8 @@ impl Rpc for SpyglassRPC {
Box::pin(route::delete_doc(self.state.clone(), id))
}
- fn toggle_pause(&self) -> BoxFuture> {
- Box::pin(route::toggle_pause(self.state.clone()))
+ fn list_installed_lenses(&self) -> BoxFuture>> {
+ Box::pin(route::list_installed_lenses(self.state.clone()))
}
fn search_docs(&self, query: SearchParam) -> BoxFuture> {
@@ -44,6 +44,10 @@ impl Rpc for SpyglassRPC {
fn search_lenses(&self, query: SearchLensesParam) -> BoxFuture> {
Box::pin(route::search_lenses(self.state.clone(), query))
}
+
+ fn toggle_pause(&self) -> BoxFuture> {
+ Box::pin(route::toggle_pause(self.state.clone()))
+ }
}
pub fn start_api_ipc(state: &AppState) -> anyhow::Result {
diff --git a/crates/spyglass/src/api/route.rs b/crates/spyglass/src/api/route.rs
index e4e27692d..06385d5ff 100644
--- a/crates/spyglass/src/api/route.rs
+++ b/crates/spyglass/src/api/route.rs
@@ -19,74 +19,6 @@ use libspyglass::state::AppState;
use super::response;
-#[instrument(skip(state))]
-pub async fn search(state: AppState, search_req: request::SearchParam) -> Result {
- let fields = Searcher::doc_fields();
-
- let index = state.index;
- let searcher = index.reader.searcher();
-
- // Create a copy of the lenses for this search
- let mut lenses = HashMap::new();
- for entry in state.lenses.iter() {
- lenses.insert(entry.key().clone(), entry.value().clone());
- }
-
- let docs = Searcher::search_with_lens(
- &lenses,
- &index.index,
- &index.reader,
- &search_req.lenses,
- &search_req.query,
- );
-
- let mut results: Vec = Vec::new();
- for (score, doc_addr) in docs {
- let retrieved = searcher.doc(doc_addr).unwrap();
-
- let doc_id = retrieved.get_first(fields.id).unwrap();
- let domain = retrieved.get_first(fields.domain).unwrap();
- let title = retrieved.get_first(fields.title).unwrap();
- let description = retrieved.get_first(fields.description).unwrap();
- let url = retrieved.get_first(fields.url).unwrap();
-
- let result = SearchResult {
- doc_id: doc_id.as_text().unwrap().to_string(),
- domain: domain.as_text().unwrap().to_string(),
- title: title.as_text().unwrap().to_string(),
- description: description.as_text().unwrap().to_string(),
- url: url.as_text().unwrap().to_string(),
- score,
- };
-
- results.push(result);
- }
-
- let meta = SearchMeta {
- query: search_req.query,
- num_docs: searcher.num_docs(),
- wall_time_ms: 1000,
- };
-
- Ok(SearchResults { results, meta })
-}
-
-/// Show the list of URLs in the queue and their status
-#[instrument(skip(state))]
-pub async fn list_queue(state: AppState) -> Result {
- let db = &state.db;
- let queue = crawl_queue::Entity::find().all(db).await;
-
- match queue {
- Ok(queue) => Ok(response::ListQueue { queue }),
- Err(err) => Err(Error {
- code: ErrorCode::InternalError,
- message: err.to_string(),
- data: None,
- }),
- }
-}
-
/// Add url to queue
#[instrument(skip(state))]
pub async fn add_queue(state: AppState, queue_item: request::QueueItemParam) -> Result {
@@ -170,12 +102,61 @@ pub async fn crawl_stats(state: AppState) -> jsonrpc_core::Result {
let by_domain = by_domain
.into_iter()
- .filter(|(_, stats)| stats.total() >= 100)
+ .filter(|(_, stats)| stats.total() >= 10)
.collect();
Ok(CrawlStats { by_domain })
}
+/// Remove a doc from the index
+#[instrument(skip(state))]
+pub async fn delete_doc(state: AppState, id: String) -> Result<()> {
+ if let Ok(mut writer) = state.index.writer.lock() {
+ if let Err(e) = Searcher::delete(&mut writer, &id) {
+ log::error!("Unable to delete doc {} due to {}", id, e);
+ } else {
+ let _ = writer.commit();
+ }
+ }
+
+ Ok(())
+}
+
+/// List of installed lenses
+#[instrument(skip(state))]
+pub async fn list_installed_lenses(state: AppState) -> Result> {
+ let mut lenses: Vec = state
+ .lenses
+ .iter()
+ .map(|lens| LensResult {
+ author: lens.author.clone(),
+ title: lens.name.clone(),
+ description: lens.description.clone().unwrap_or_else(|| "".into()),
+ ..Default::default()
+ })
+ .collect();
+
+ lenses.sort_by(|x, y| x.title.cmp(&y.title));
+
+ Ok(lenses)
+}
+
+/// Show the list of URLs in the queue and their status
+#[instrument(skip(state))]
+pub async fn list_queue(state: AppState) -> Result {
+ let db = &state.db;
+ let queue = crawl_queue::Entity::find().all(db).await;
+
+ match queue {
+ Ok(queue) => Ok(response::ListQueue { queue }),
+ Err(err) => Err(Error {
+ code: ErrorCode::InternalError,
+ message: err.to_string(),
+ data: None,
+ }),
+ }
+}
+
#[instrument(skip(state))]
pub async fn toggle_pause(state: AppState) -> jsonrpc_core::Result {
// Scope so that the app_state mutex is correctly released.
@@ -191,6 +172,60 @@ pub async fn toggle_pause(state: AppState) -> jsonrpc_core::Result {
_get_current_status(state.clone()).await
}
+/// Search the user's indexed documents
+#[instrument(skip(state))]
+pub async fn search(state: AppState, search_req: request::SearchParam) -> Result {
+ let fields = Searcher::doc_fields();
+
+ let index = state.index;
+ let searcher = index.reader.searcher();
+
+ // Create a copy of the lenses for this search
+ let mut lenses = HashMap::new();
+ for entry in state.lenses.iter() {
+ lenses.insert(entry.key().clone(), entry.value().clone());
+ }
+
+ let docs = Searcher::search_with_lens(
+ &lenses,
+ &index.index,
+ &index.reader,
+ &search_req.lenses,
+ &search_req.query,
+ );
+
+ let mut results: Vec = Vec::new();
+ for (score, doc_addr) in docs {
+ let retrieved = searcher.doc(doc_addr).unwrap();
+
+ let doc_id = retrieved.get_first(fields.id).unwrap();
+ let domain = retrieved.get_first(fields.domain).unwrap();
+ let title = retrieved.get_first(fields.title).unwrap();
+ let description = retrieved.get_first(fields.description).unwrap();
+ let url = retrieved.get_first(fields.url).unwrap();
+
+ let result = SearchResult {
+ doc_id: doc_id.as_text().unwrap().to_string(),
+ domain: domain.as_text().unwrap().to_string(),
+ title: title.as_text().unwrap().to_string(),
+ description: description.as_text().unwrap().to_string(),
+ url: url.as_text().unwrap().to_string(),
+ score,
+ };
+
+ results.push(result);
+ }
+
+ let meta = SearchMeta {
+ query: search_req.query,
+ num_docs: searcher.num_docs(),
+ wall_time_ms: 1000,
+ };
+
+ Ok(SearchResults { results, meta })
+}
+
+/// Search the user's installed lenses
#[instrument(skip(state))]
pub async fn search_lenses(
state: AppState,
@@ -212,23 +247,12 @@ pub async fn search_lenses(
let query_results = query_results.unwrap();
for lens in query_results {
results.push(LensResult {
+ author: lens.author,
title: lens.name,
description: lens.description.unwrap_or_else(|| "".to_string()),
+ ..Default::default()
});
}
Ok(SearchLensesResp { results })
}
-
-#[instrument(skip(state))]
-pub async fn delete_doc(state: AppState, id: String) -> Result<()> {
- if let Ok(mut writer) = state.index.writer.lock() {
- if let Err(e) = Searcher::delete(&mut writer, &id) {
- log::error!("Unable to delete doc {} due to {}", id, e);
- } else {
- let _ = writer.commit();
- }
- }
-
- Ok(())
-}
diff --git a/crates/spyglass/src/search/lens.rs b/crates/spyglass/src/search/lens.rs
index cd2427595..17f3178bb 100644
--- a/crates/spyglass/src/search/lens.rs
+++ b/crates/spyglass/src/search/lens.rs
@@ -9,30 +9,6 @@ use crate::crawler::bootstrap;
use crate::search::Searcher;
use crate::state::AppState;
-async fn create_default_lens(config: &Config) {
- // Create a default lens as an example.
- let lens = Lens {
- author: "Spyglass".to_string(),
- version: "1".to_string(),
- name: "rust".to_string(),
- description: Some(
- "All things Rustlang. Search through Rust blogs, the rust book, and
- more."
- .to_string(),
- ),
- domains: vec!["blog.rust-lang.org".into()],
- urls: vec!["https://doc.rust-lang.org/book/".into()],
- is_enabled: true,
- rules: Vec::new(),
- };
-
- fs::write(
- config.lenses_dir().join("rust.ron"),
- ron::ser::to_string_pretty(&lens, Default::default()).unwrap(),
- )
- .expect("Unable to save default lens file.");
-}
-
/// Check if we've already bootstrapped a prefix / otherwise add it to the queue.
async fn check_and_bootstrap(
db: &DatabaseConnection,
@@ -63,7 +39,6 @@ pub async fn read_lenses(state: &AppState, config: &Config) -> anyhow::Result<()
state.lenses.clear();
let lense_dir = config.lenses_dir();
- let mut num_lenses = 0;
for entry in (fs::read_dir(lense_dir)?).flatten() {
let path = entry.path();
@@ -72,7 +47,6 @@ pub async fn read_lenses(state: &AppState, config: &Config) -> anyhow::Result<()
match ron::from_str::(&file_contents) {
Err(err) => log::error!("Unable to load lens {:?}: {}", entry.path(), err),
Ok(lens) => {
- num_lenses += 1;
if lens.is_enabled {
state.lenses.insert(lens.name.clone(), lens);
}
@@ -82,10 +56,6 @@ pub async fn read_lenses(state: &AppState, config: &Config) -> anyhow::Result<()
}
}
- if num_lenses == 0 {
- create_default_lens(config).await;
- }
-
Ok(())
}
diff --git a/crates/tauri/Cargo.toml b/crates/tauri/Cargo.toml
index f649e8614..598baeb26 100644
--- a/crates/tauri/Cargo.toml
+++ b/crates/tauri/Cargo.toml
@@ -22,6 +22,7 @@ log = "0.4"
num-format = "0.4"
open = "2"
reqwest = { version = "0.11", features = ["json"] }
+ron = "0.7"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
shared = { path = "../shared" }
@@ -32,6 +33,7 @@ tracing = "0.1"
tracing-appender = "0.2"
tracing-log = "0.1.3"
tracing-subscriber = { version = "0.3", features = ["env-filter", "std"] }
+url = "2.2"
[features]
default = [ "custom-protocol" ]
diff --git a/crates/tauri/src/cmd.rs b/crates/tauri/src/cmd.rs
index a286aa20a..ae231c8a2 100644
--- a/crates/tauri/src/cmd.rs
+++ b/crates/tauri/src/cmd.rs
@@ -1,8 +1,15 @@
+use std::fs;
+
use jsonrpc_core::Value;
use tauri::State;
+use url::Url;
-use crate::{rpc, window};
-use shared::{request, response};
+use crate::{constants, open_folder, rpc, window};
+use shared::{
+ config::Config,
+ request,
+ response::{self, InstallableLens},
+};
#[tauri::command]
pub async fn escape(window: tauri::Window) -> Result<(), String> {
@@ -10,6 +17,12 @@ pub async fn escape(window: tauri::Window) -> Result<(), String> {
Ok(())
}
+#[tauri::command]
+pub async fn open_lens_folder(_: tauri::Window, config: State<'_, Config>) -> Result<(), String> {
+ open_folder(config.lenses_dir());
+ Ok(())
+}
+
#[tauri::command]
pub async fn open_result(_: tauri::Window, url: &str) -> Result<(), String> {
open::that(url).unwrap();
@@ -21,6 +34,60 @@ pub async fn resize_window(window: tauri::Window, height: f64) {
window::resize_window(&window, height).await;
}
+#[tauri::command]
+pub async fn crawl_stats<'r>(
+ _: tauri::Window,
+ rpc: State<'_, rpc::RpcMutex>,
+) -> Result {
+ let mut rpc = rpc.lock().await;
+ match rpc
+ .client
+ .call_method::("crawl_stats", "", Value::Null)
+ .await
+ {
+ Ok(resp) => Ok(resp),
+ Err(err) => {
+ log::error!("Error sending RPC: {}", err);
+ rpc.reconnect().await;
+ Ok(response::CrawlStats {
+ by_domain: Vec::new(),
+ })
+ }
+ }
+}
+
+#[tauri::command]
+pub async fn list_installed_lenses(
+ _: tauri::Window,
+ rpc: State<'_, rpc::RpcMutex>,
+) -> Result, String> {
+ let mut rpc = rpc.lock().await;
+ Ok(rpc
+ .call::>("list_installed_lenses", Value::Null)
+ .await)
+}
+
+#[tauri::command]
+pub async fn list_installable_lenses(
+ _: tauri::Window,
+) -> Result, String> {
+ let client = reqwest::Client::builder()
+ .user_agent(constants::APP_USER_AGENT)
+ .build()
+ .expect("Unable to create reqwest client");
+
+ if let Ok(res) = client.get(constants::LENS_DIRECTORY_INDEX_URL).send().await {
+ if let Ok(file_contents) = res.text().await {
+ return match ron::from_str::>(&file_contents) {
+ Ok(json) => Ok(json),
+ Err(e) => Err(format!("Unable to parse index: {}", e)),
+ };
+ }
+ }
+
+ Ok(Vec::new())
+}
+
#[tauri::command]
pub async fn search_docs<'r>(
_: tauri::Window,
@@ -57,44 +124,11 @@ pub async fn search_lenses<'r>(
query: query.to_string(),
};
- let rpc = rpc.lock().await;
- match rpc
- .client
- .call_method::<(request::SearchLensesParam,), response::SearchLensesResp>(
- "search_lenses",
- "",
- (data,),
- )
- .await
- {
- Ok(resp) => Ok(resp.results.to_vec()),
- Err(err) => {
- log::error!("{}", err);
- Ok(Vec::new())
- }
- }
-}
-
-#[tauri::command]
-pub async fn crawl_stats<'r>(
- _: tauri::Window,
- rpc: State<'_, rpc::RpcMutex>,
-) -> Result {
let mut rpc = rpc.lock().await;
- match rpc
- .client
- .call_method::("crawl_stats", "", Value::Null)
- .await
- {
- Ok(resp) => Ok(resp),
- Err(err) => {
- log::error!("Error sending RPC: {}", err);
- rpc.reconnect().await;
- Ok(response::CrawlStats {
- by_domain: Vec::new(),
- })
- }
- }
+ let resp = rpc
+ .call::<(request::SearchLensesParam,), response::SearchLensesResp>("search_lenses", (data,))
+ .await;
+ Ok(resp.results)
}
#[tauri::command]
@@ -120,3 +154,45 @@ pub async fn delete_doc<'r>(
}
}
}
+
+/// Install a lens (assumes correct format) from a URL
+#[tauri::command]
+pub async fn install_lens<'r>(
+ window: tauri::Window,
+ config: State<'_, Config>,
+ download_url: &str,
+) -> Result<(), String> {
+ log::trace!("installing lens from <{}>", download_url);
+
+ let client = reqwest::Client::builder()
+ .user_agent(constants::APP_USER_AGENT)
+ .build()
+ .expect("Unable to create reqwest client");
+
+ if let Ok(resp) = client.get(download_url).send().await {
+ if let Ok(file_contents) = resp.text().await {
+ // Grab the file name from the end of the URL
+ let url = Url::parse(download_url).unwrap();
+ let mut segments = url.path_segments().map(|c| c.collect::>()).unwrap();
+ let file_name = segments.pop().unwrap();
+ // Create path from file name + lens directory
+ let lens_path = config.lenses_dir().join(file_name);
+ log::info!("installing lens to {:?}", lens_path);
+
+ if let Err(e) = fs::write(lens_path.clone(), file_contents) {
+ log::error!(
+ "Unable to install lens {} to {:?} due to error: {}",
+ download_url,
+ lens_path,
+ e
+ );
+ } else {
+ // Sleep for a second to let the app reload the lenses and then let the client know we're done.
+ tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;
+ let _ = window.emit("refresh_lens_manager", true);
+ }
+ }
+ }
+
+ Ok(())
+}
diff --git a/crates/tauri/src/constants.rs b/crates/tauri/src/constants.rs
index 8c38e77c7..d132111bc 100644
--- a/crates/tauri/src/constants.rs
+++ b/crates/tauri/src/constants.rs
@@ -3,4 +3,9 @@ pub const INPUT_Y: f64 = 128.0;
pub const APP_USER_AGENT: &str = "spyglass (github.com/a5huynh/spyglass)";
pub const DISCORD_JOIN_URL: &str = "https://discord.gg/663wPVBSTB";
+pub const LENS_DIRECTORY_INDEX_URL: &str =
+ "https://raw.githubusercontent.com/spyglass-search/lens-box/main/index.ron";
pub const VERSION_CHECK_URL: &str = "https://api.github.com/repos/a5huynh/spyglass/releases";
+
+pub const STATS_WIN_NAME: &str = "crawl_stats";
+pub const LENS_MANAGER_WIN_NAME: &str = "lens_manager";
diff --git a/crates/tauri/src/main.rs b/crates/tauri/src/main.rs
index 654be022b..f86189b20 100644
--- a/crates/tauri/src/main.rs
+++ b/crates/tauri/src/main.rs
@@ -27,7 +27,7 @@ mod constants;
mod menu;
mod rpc;
mod window;
-use window::show_crawl_stats_window;
+use window::{show_crawl_stats_window, show_lens_manager_window};
fn main() -> Result<(), Box> {
let config = Config::new();
@@ -52,13 +52,17 @@ fn main() -> Result<(), Box> {
tauri::Builder::default()
.invoke_handler(tauri::generate_handler![
+ cmd::crawl_stats,
+ cmd::delete_doc,
cmd::escape,
+ cmd::install_lens,
+ cmd::list_installable_lenses,
+ cmd::list_installed_lenses,
+ cmd::open_lens_folder,
cmd::open_result,
+ cmd::resize_window,
cmd::search_docs,
cmd::search_lenses,
- cmd::resize_window,
- cmd::crawl_stats,
- cmd::delete_doc,
])
.menu(menu::get_app_menu())
.system_tray(SystemTray::new().with_menu(menu::get_tray_menu(&config)))
@@ -165,7 +169,9 @@ fn main() -> Result<(), Box> {
let _ = open::that(&version.html_url);
}
}
- menu::OPEN_LENSES_FOLDER => open_folder(config.lenses_dir()),
+ menu::OPEN_LENS_MANAGER => {
+ show_lens_manager_window(app);
+ }
menu::OPEN_LOGS_FOLDER => open_folder(config.logs_dir()),
menu::OPEN_SETTINGS_FOLDER => open_folder(Config::prefs_dir()),
menu::SHOW_CRAWL_STATUS => {
diff --git a/crates/tauri/src/menu.rs b/crates/tauri/src/menu.rs
index 74223e571..5d682386a 100644
--- a/crates/tauri/src/menu.rs
+++ b/crates/tauri/src/menu.rs
@@ -7,7 +7,7 @@ pub const QUIT_MENU_ITEM: &str = "quit";
pub const NUM_DOCS_MENU_ITEM: &str = "num_docs";
pub const CRAWL_STATUS_MENU_ITEM: &str = "crawl_status";
-pub const OPEN_LENSES_FOLDER: &str = "open_lenses_folder";
+pub const OPEN_LENS_MANAGER: &str = "open_lens_manager";
pub const OPEN_SETTINGS_FOLDER: &str = "open_settings_folder";
pub const OPEN_LOGS_FOLDER: &str = "open_logs_folder";
pub const SHOW_SEARCHBAR: &str = "show_searchbar";
@@ -25,12 +25,10 @@ pub fn get_tray_menu(config: &Config) -> SystemTrayMenu {
let pause = CustomMenuItem::new(CRAWL_STATUS_MENU_ITEM.to_string(), "");
let quit = CustomMenuItem::new(QUIT_MENU_ITEM.to_string(), "Quit");
- let open_lenses_folder =
- CustomMenuItem::new(OPEN_LENSES_FOLDER.to_string(), "Show lenses folder");
let open_settings_folder =
- CustomMenuItem::new(OPEN_SETTINGS_FOLDER.to_string(), "Show settings folder");
+ CustomMenuItem::new(OPEN_SETTINGS_FOLDER.to_string(), "Open settings folder");
- let open_logs_folder = CustomMenuItem::new(OPEN_LOGS_FOLDER.to_string(), "Show logs folder");
+ let open_logs_folder = CustomMenuItem::new(OPEN_LOGS_FOLDER.to_string(), "Open logs folder");
let app_version = format!("v20{}", ctx.package_info().version);
let mut tray = SystemTrayMenu::new();
@@ -47,8 +45,11 @@ pub fn get_tray_menu(config: &Config) -> SystemTrayMenu {
SHOW_CRAWL_STATUS.to_string(),
"Show crawl status",
))
+ .add_item(CustomMenuItem::new(
+ OPEN_LENS_MANAGER.to_string(),
+ "Manage/install lenses",
+ ))
.add_native_item(SystemTrayMenuItem::Separator)
- .add_item(open_lenses_folder)
.add_item(open_settings_folder)
.add_item(open_logs_folder);
@@ -56,7 +57,7 @@ pub fn get_tray_menu(config: &Config) -> SystemTrayMenu {
if cfg!(debug_assertions) {
tray = tray
.add_native_item(SystemTrayMenuItem::Separator)
- .add_item(CustomMenuItem::new(DEV_SHOW_CONSOLE, "Show console"));
+ .add_item(CustomMenuItem::new(DEV_SHOW_CONSOLE, "Open dev console"));
}
tray.add_native_item(SystemTrayMenuItem::Separator)
diff --git a/crates/tauri/src/rpc.rs b/crates/tauri/src/rpc.rs
index ee09c19b4..7b7dfc8dd 100644
--- a/crates/tauri/src/rpc.rs
+++ b/crates/tauri/src/rpc.rs
@@ -1,6 +1,8 @@
use std::sync::Arc;
use jsonrpc_core_client::{transports::ipc, TypedClient};
+use serde::de::DeserializeOwned;
+use serde::Serialize;
use shared::rpc::gen_ipc_path;
use tauri::api::process::{Command, CommandEvent};
use tokio::sync::Mutex;
@@ -63,6 +65,21 @@ impl RpcClient {
}
}
+ pub async fn call(
+ &mut self,
+ method: &str,
+ args: T,
+ ) -> R {
+ match self.client.call_method::(method, "", args).await {
+ Ok(resp) => resp,
+ Err(err) => {
+ log::error!("Error sending RPC: {}", err);
+ self.reconnect().await;
+ R::default()
+ }
+ }
+ }
+
pub async fn reconnect(&mut self) {
log::info!("Attempting to restart backend");
// Attempt to reconnect
diff --git a/crates/tauri/src/window.rs b/crates/tauri/src/window.rs
index b59dc4d30..ea110e2b4 100644
--- a/crates/tauri/src/window.rs
+++ b/crates/tauri/src/window.rs
@@ -40,14 +40,35 @@ pub fn show_window(window: &Window) {
}
pub fn show_crawl_stats_window(app: &AppHandle) -> Window {
- if let Some(window) = app.get_window("stats") {
+ if let Some(window) = app.get_window(constants::STATS_WIN_NAME) {
let _ = window.show();
let _ = window.set_focus();
return window;
}
- WindowBuilder::new(app, "stats", WindowUrl::App("/stats".into()))
- .title("Status")
- .build()
- .unwrap()
+ WindowBuilder::new(
+ app,
+ constants::STATS_WIN_NAME,
+ WindowUrl::App("/stats".into()),
+ )
+ .title("Status")
+ .build()
+ .unwrap()
+}
+
+pub fn show_lens_manager_window(app: &AppHandle) -> Window {
+ if let Some(window) = app.get_window(constants::LENS_MANAGER_WIN_NAME) {
+ let _ = window.show();
+ let _ = window.set_focus();
+ return window;
+ }
+
+ WindowBuilder::new(
+ app,
+ constants::LENS_MANAGER_WIN_NAME,
+ WindowUrl::App("/settings/lens".into()),
+ )
+ .title("Lens Manager")
+ .build()
+ .unwrap()
}
diff --git a/crates/tauri/tauri.conf.json b/crates/tauri/tauri.conf.json
index b9cc8aad5..fd52bfb26 100644
--- a/crates/tauri/tauri.conf.json
+++ b/crates/tauri/tauri.conf.json
@@ -1,7 +1,7 @@
{
"package": {
"productName": "Spyglass",
- "version": "22.5.28"
+ "version": "22.6.1"
},
"build": {
"distDir": "../client/dist",
{result.description.clone()}
+
{result.title.clone()}
-
+
{result.description.clone()}
diff --git a/crates/client/src/icons.rs b/crates/client/src/icons.rs
new file mode 100644
index 000000000..e69de29bb
diff --git a/crates/client/src/main.rs b/crates/client/src/main.rs
index 82af93c2b..2cb2444f5 100644
--- a/crates/client/src/main.rs
+++ b/crates/client/src/main.rs
@@ -7,13 +7,22 @@ mod constants;
mod events;
mod pages;
-use crate::pages::{SearchPage, StatsPage};
+use crate::pages::{LensManagerPage, SearchPage, StatsPage};
#[wasm_bindgen(module = "/public/glue.js")]
extern "C" {
#[wasm_bindgen(js_name = "deleteDoc", catch)]
pub async fn delete_doc(id: String) -> Result<(), JsValue>;
+ #[wasm_bindgen(catch)]
+ pub async fn install_lens(download_url: String) -> Result<(), JsValue>;
+
+ #[wasm_bindgen(js_name = "listInstalledLenses", catch)]
+ pub async fn list_installed_lenses() -> Result;
+
+ #[wasm_bindgen(js_name = "listInstallableLenses", catch)]
+ pub async fn list_installable_lenses() -> Result;
+
#[wasm_bindgen(js_name = "searchDocs", catch)]
pub async fn search_docs(lenses: JsValue, query: String) -> Result;
@@ -29,9 +38,15 @@ extern "C" {
#[wasm_bindgen(js_name = "onRefreshResults")]
pub async fn on_refresh_results(callback: &Closure);
+ #[wasm_bindgen]
+ pub async fn on_refresh_lens_manager(callback: &Closure);
+
#[wasm_bindgen(js_name = "openResult", catch)]
pub async fn open(url: String) -> Result<(), JsValue>;
+ #[wasm_bindgen(js_name = "openLensFolder", catch)]
+ pub async fn open_lens_folder() -> Result<(), JsValue>;
+
#[wasm_bindgen(js_name = "escape", catch)]
pub async fn escape() -> Result<(), JsValue>;
@@ -46,6 +61,8 @@ extern "C" {
enum Route {
#[at("/")]
Search,
+ #[at("/settings/lens")]
+ LensManager,
#[at("/stats")]
Status,
}
@@ -66,6 +83,7 @@ pub fn app() -> Html {
fn switch(routes: &Route) -> Html {
match routes {
+ Route::LensManager => html! { >,
+ req_state: UseStateHandle,
+) {
+ spawn_local(async move {
+ match list_installed_lenses().await {
+ Ok(results) => {
+ lenses_handle.set(results.into_serde().unwrap());
+ req_state.set(RequestState::Finished);
+ }
+ Err(e) => {
+ log::info!("Error: {:?}", e);
+ req_state.set(RequestState::Error);
+ }
+ }
+ });
+}
+
+fn fetch_installable_lenses(
+ data_handle: UseStateHandle>,
+ req_state: UseStateHandle,
+) {
+ spawn_local(async move {
+ match list_installable_lenses().await {
+ Ok(results) => {
+ let lenses: Vec = results.into_serde().unwrap();
+ let parsed: Vec = lenses
+ .iter()
+ .map(|lens| LensResult {
+ author: lens.author.to_owned(),
+ title: lens.name.to_owned(),
+ description: lens.description.to_owned(),
+ html_url: Some(lens.html_url.to_owned()),
+ download_url: Some(lens.download_url.to_owned()),
+ })
+ .collect();
+
+ data_handle.set(parsed);
+ req_state.set(RequestState::Finished);
+ }
+ Err(e) => {
+ log::info!("Error: {:?}", e);
+ req_state.set(RequestState::Error);
+ }
+ }
+ });
+}
+
+fn handle_install_lens(download_url: String) {
+ spawn_local(async move {
+ if let Err(e) = install_lens(download_url.clone()).await {
+ log::error!("error installing lens: {} {:?}", download_url.clone(), e);
+ }
+ });
+}
+
+#[derive(Properties, PartialEq)]
+pub struct InstallBtnProps {
+ pub download_url: String,
+}
+
+#[function_component(InstallButton)]
+pub fn install_btn(props: &InstallBtnProps) -> Html {
+ let is_installing = use_state_eq(|| false);
+ let download_url = props.download_url.clone();
+
+ let onclick = {
+ let is_installing = is_installing.clone();
+ move |_| {
+ is_installing.set(true);
+ // Download to lens directory
+ handle_install_lens(download_url.clone());
+ }
+ };
+
+ if *is_installing {
+ html! {
+
+ = vec![
+ "border-t".into(),
+ "border-neutral-600".into(),
+ "p-4".into(),
+ "pr-0".into(),
+ "text-white".into(),
+ "bg-netural-800".into(),
+ ];
+ let result = &props.result;
+
+ let installed_el = if props.is_installed {
+ html! {
+
+
+ {result.title.clone()}
+
+
+ {"Crafted By:"}
+
+ {format!("@{}", result.author)}
+
+
+
+ {result.description.clone()}
+
+
+ {installed_el}
+ {view_link}
+
+
+ }
+}
+
+#[function_component(LensManagerPage)]
+pub fn lens_manager_page() -> Html {
+ let user_installed: UseStateHandle> = use_state_eq(Vec::new);
+ let installable: UseStateHandle> = use_state_eq(Vec::new);
+
+ let ui_req_state = use_state_eq(|| RequestState::NotStarted);
+ if *ui_req_state == RequestState::NotStarted {
+ ui_req_state.set(RequestState::InProgress);
+ fetch_installed_lenses(user_installed.clone(), ui_req_state.clone());
+ }
+
+ let i_req_state = use_state_eq(|| RequestState::NotStarted);
+ if *i_req_state == RequestState::NotStarted {
+ i_req_state.set(RequestState::InProgress);
+ fetch_installable_lenses(installable.clone(), i_req_state.clone());
+ }
+
+ let on_open_folder = {
+ move |_| {
+ spawn_local(async {
+ let _ = crate::open_lens_folder().await;
+ });
+ }
+ };
+
+ let on_refresh = {
+ let ui_req_state = ui_req_state.clone();
+ let i_req_state = i_req_state.clone();
+ move |_| {
+ ui_req_state.set(RequestState::NotStarted);
+ i_req_state.set(RequestState::NotStarted);
+ }
+ };
+
+ let already_installed: HashSet =
+ user_installed.iter().map(|x| x.title.clone()).collect();
+ installable.set(
+ installable
+ .iter()
+ .filter(|x| !already_installed.contains(&x.title))
+ .map(|x| x.to_owned())
+ .collect::>(),
+ );
+
+ // Handle refreshing the list
+ {
+ let ui_req_state = ui_req_state.clone();
+ let i_req_state = i_req_state.clone();
+ spawn_local(async move {
+ let cb = Closure::wrap(Box::new(move || {
+ ui_req_state.set(RequestState::NotStarted);
+ i_req_state.set(RequestState::NotStarted);
+ }) as Box);
+
+ on_refresh_lens_manager(&cb).await;
+ cb.forget();
+ });
+ }
+
+ let contents = if ui_req_state.is_done() && i_req_state.is_done() {
+ html! {
+ <>
+ {
+ user_installed.iter().map(|data| {
+ html! {{"Lens Manager"}
+
+
+
+
+
+ {contents}
+
+
+ }
+}
diff --git a/crates/client/src/pages/mod.rs b/crates/client/src/pages/mod.rs
index 44cbc773d..ef47f47b6 100644
--- a/crates/client/src/pages/mod.rs
+++ b/crates/client/src/pages/mod.rs
@@ -1,3 +1,6 @@
+mod lens_manager;
+pub use lens_manager::*;
+
mod search;
pub use search::*;
diff --git a/crates/client/src/pages/search.rs b/crates/client/src/pages/search.rs
index b9c3bae1c..3b325926a 100644
--- a/crates/client/src/pages/search.rs
+++ b/crates/client/src/pages/search.rs
@@ -172,18 +172,18 @@ pub fn search_page() -> Html {
html! {
-
+
>) {
+fn fetch_crawl_stats(
+ stats_handle: UseStateHandle>,
+ request_finished: UseStateHandle,
+) {
spawn_local(async move {
match crawl_stats().await {
Ok(results) => {
@@ -13,8 +17,12 @@ fn fetch_crawl_stats(stats_handle: UseStateHandle>) {
let mut sorted = results.by_domain;
sorted.sort_by(|(_, a), (_, b)| b.num_completed.cmp(&a.num_completed));
stats_handle.set(sorted);
+ request_finished.set(true);
+ }
+ Err(e) => {
+ log::info!("Error: {:?}", e);
+ request_finished.set(true);
}
- Err(e) => log::info!("Error: {:?}", e),
}
});
}
@@ -39,7 +47,7 @@ pub fn legend_icon(props: &LegendIconProps) -> Html {
];
html! {
-
+
{props.label.clone()}
@@ -82,7 +90,7 @@ fn stats_bar(props: &StatsBarProps) -> Html {
html! {
}
}
@@ -90,15 +98,19 @@ fn stats_bar(props: &StatsBarProps) -> Html {
#[function_component(StatsPage)]
pub fn stats_page() -> Html {
let stats: UseStateHandle> = use_state_eq(Vec::new);
- if stats.is_empty() {
- fetch_crawl_stats(stats.clone());
+ let request_finished = use_state(|| false);
+
+ if stats.is_empty() && !(*request_finished) {
+ fetch_crawl_stats(stats.clone(), request_finished.clone());
}
let onclick = {
+ let request_finished = request_finished.clone();
let stats = stats.clone();
move |_| {
+ request_finished.set(false);
stats.set(Vec::new());
- fetch_crawl_stats(stats.clone());
+ fetch_crawl_stats(stats.clone(), request_finished.clone());
}
};
@@ -107,11 +119,11 @@ pub fn stats_page() -> Html {
.map(|(domain, stats)| {
let total = stats.total() as f64;
html! {
-
-
+
+
{domain}
-
+
+
{"Crawl Status"}
@@ -158,7 +174,7 @@ pub fn stats_page() -> Html {
-
+
{rendered}
diff --git a/crates/entities/src/models/crawl_queue.rs b/crates/entities/src/models/crawl_queue.rs
index 3fc69c639..9ace3ffd7 100644
--- a/crates/entities/src/models/crawl_queue.rs
+++ b/crates/entities/src/models/crawl_queue.rs
@@ -152,7 +152,7 @@ struct CrawlQueueCount {
count: i64,
}
-#[derive(FromQueryResult)]
+#[derive(Debug, FromQueryResult)]
pub struct QueueCountByStatus {
pub count: i64,
pub domain: String,
diff --git a/crates/entities/src/models/indexed_document.rs b/crates/entities/src/models/indexed_document.rs
index e7500f7f3..3bd10626a 100644
--- a/crates/entities/src/models/indexed_document.rs
+++ b/crates/entities/src/models/indexed_document.rs
@@ -46,7 +46,7 @@ impl ActiveModelBehavior for ActiveModel {
}
}
-#[derive(FromQueryResult)]
+#[derive(Debug, FromQueryResult)]
pub struct CountByDomain {
pub count: i64,
pub domain: String,
diff --git a/crates/shared/src/response.rs b/crates/shared/src/response.rs
index 5ab26460d..e54be89a4 100644
--- a/crates/shared/src/response.rs
+++ b/crates/shared/src/response.rs
@@ -25,6 +25,26 @@ pub struct CrawlStats {
pub by_domain: Vec<(String, QueueStatus)>,
}
+#[derive(Clone, Deserialize, Serialize)]
+pub struct InstallableLens {
+ pub author: String,
+ pub description: String,
+ pub name: String,
+ pub sha: String,
+ pub download_url: String,
+ pub html_url: String,
+}
+
+#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)]
+pub struct LensResult {
+ pub author: String,
+ pub title: String,
+ pub description: String,
+ // Only relevant for installable lenses
+ pub html_url: Option,
+ pub download_url: Option,
+}
+
#[derive(Debug, Deserialize, Serialize)]
pub struct SearchMeta {
pub query: String,
@@ -48,13 +68,7 @@ pub struct SearchResults {
pub meta: SearchMeta,
}
-#[derive(Clone, Debug, Deserialize, Serialize)]
-pub struct LensResult {
- pub title: String,
- pub description: String,
-}
-
-#[derive(Debug, Deserialize, Serialize)]
+#[derive(Debug, Default, Deserialize, Serialize)]
pub struct SearchLensesResp {
pub results: Vec,
}
diff --git a/crates/shared/src/rpc.rs b/crates/shared/src/rpc.rs
index 098f20a4e..e4d142e39 100644
--- a/crates/shared/src/rpc.rs
+++ b/crates/shared/src/rpc.rs
@@ -2,7 +2,7 @@ use jsonrpc_core::{BoxFuture, Result};
use jsonrpc_derive::rpc;
use crate::request::{SearchLensesParam, SearchParam};
-use crate::response::{AppStatus, CrawlStats, SearchLensesResp, SearchResults};
+use crate::response::{AppStatus, CrawlStats, LensResult, SearchLensesResp, SearchResults};
pub fn gen_ipc_path() -> String {
if cfg!(windows) {
@@ -28,12 +28,15 @@ pub trait Rpc {
#[rpc(name = "delete_doc")]
fn delete_doc(&self, id: String) -> BoxFuture>;
- #[rpc(name = "toggle_pause")]
- fn toggle_pause(&self) -> BoxFuture>;
+ #[rpc(name = "list_installed_lenses")]
+ fn list_installed_lenses(&self) -> BoxFuture>>;
#[rpc(name = "search_docs")]
fn search_docs(&self, query: SearchParam) -> BoxFuture>;
#[rpc(name = "search_lenses")]
fn search_lenses(&self, query: SearchLensesParam) -> BoxFuture>;
+
+ #[rpc(name = "toggle_pause")]
+ fn toggle_pause(&self) -> BoxFuture>;
}
diff --git a/crates/spyglass/src/api/mod.rs b/crates/spyglass/src/api/mod.rs
index 953d154c0..cac18a252 100644
--- a/crates/spyglass/src/api/mod.rs
+++ b/crates/spyglass/src/api/mod.rs
@@ -6,7 +6,7 @@ use jsonrpc_ipc_server::{Server, ServerBuilder};
use libspyglass::state::AppState;
use shared::request::{SearchLensesParam, SearchParam};
-use shared::response::{AppStatus, CrawlStats, SearchLensesResp, SearchResults};
+use shared::response::{AppStatus, CrawlStats, LensResult, SearchLensesResp, SearchResults};
use shared::rpc::{gen_ipc_path, Rpc};
mod response;
@@ -33,8 +33,8 @@ impl Rpc for SpyglassRPC {
Box::pin(route::delete_doc(self.state.clone(), id))
}
- fn toggle_pause(&self) -> BoxFuture> {
- Box::pin(route::toggle_pause(self.state.clone()))
+ fn list_installed_lenses(&self) -> BoxFuture>> {
+ Box::pin(route::list_installed_lenses(self.state.clone()))
}
fn search_docs(&self, query: SearchParam) -> BoxFuture> {
@@ -44,6 +44,10 @@ impl Rpc for SpyglassRPC {
fn search_lenses(&self, query: SearchLensesParam) -> BoxFuture> {
Box::pin(route::search_lenses(self.state.clone(), query))
}
+
+ fn toggle_pause(&self) -> BoxFuture> {
+ Box::pin(route::toggle_pause(self.state.clone()))
+ }
}
pub fn start_api_ipc(state: &AppState) -> anyhow::Result {
diff --git a/crates/spyglass/src/api/route.rs b/crates/spyglass/src/api/route.rs
index e4e27692d..06385d5ff 100644
--- a/crates/spyglass/src/api/route.rs
+++ b/crates/spyglass/src/api/route.rs
@@ -19,74 +19,6 @@ use libspyglass::state::AppState;
use super::response;
-#[instrument(skip(state))]
-pub async fn search(state: AppState, search_req: request::SearchParam) -> Result {
- let fields = Searcher::doc_fields();
-
- let index = state.index;
- let searcher = index.reader.searcher();
-
- // Create a copy of the lenses for this search
- let mut lenses = HashMap::new();
- for entry in state.lenses.iter() {
- lenses.insert(entry.key().clone(), entry.value().clone());
- }
-
- let docs = Searcher::search_with_lens(
- &lenses,
- &index.index,
- &index.reader,
- &search_req.lenses,
- &search_req.query,
- );
-
- let mut results: Vec = Vec::new();
- for (score, doc_addr) in docs {
- let retrieved = searcher.doc(doc_addr).unwrap();
-
- let doc_id = retrieved.get_first(fields.id).unwrap();
- let domain = retrieved.get_first(fields.domain).unwrap();
- let title = retrieved.get_first(fields.title).unwrap();
- let description = retrieved.get_first(fields.description).unwrap();
- let url = retrieved.get_first(fields.url).unwrap();
-
- let result = SearchResult {
- doc_id: doc_id.as_text().unwrap().to_string(),
- domain: domain.as_text().unwrap().to_string(),
- title: title.as_text().unwrap().to_string(),
- description: description.as_text().unwrap().to_string(),
- url: url.as_text().unwrap().to_string(),
- score,
- };
-
- results.push(result);
- }
-
- let meta = SearchMeta {
- query: search_req.query,
- num_docs: searcher.num_docs(),
- wall_time_ms: 1000,
- };
-
- Ok(SearchResults { results, meta })
-}
-
-/// Show the list of URLs in the queue and their status
-#[instrument(skip(state))]
-pub async fn list_queue(state: AppState) -> Result {
- let db = &state.db;
- let queue = crawl_queue::Entity::find().all(db).await;
-
- match queue {
- Ok(queue) => Ok(response::ListQueue { queue }),
- Err(err) => Err(Error {
- code: ErrorCode::InternalError,
- message: err.to_string(),
- data: None,
- }),
- }
-}
-
/// Add url to queue
#[instrument(skip(state))]
pub async fn add_queue(state: AppState, queue_item: request::QueueItemParam) -> Result {
@@ -170,12 +102,61 @@ pub async fn crawl_stats(state: AppState) -> jsonrpc_core::Result {
let by_domain = by_domain
.into_iter()
- .filter(|(_, stats)| stats.total() >= 100)
+ .filter(|(_, stats)| stats.total() >= 10)
.collect();
Ok(CrawlStats { by_domain })
}
+/// Remove a doc from the index
+#[instrument(skip(state))]
+pub async fn delete_doc(state: AppState, id: String) -> Result<()> {
+ if let Ok(mut writer) = state.index.writer.lock() {
+ if let Err(e) = Searcher::delete(&mut writer, &id) {
+ log::error!("Unable to delete doc {} due to {}", id, e);
+ } else {
+ let _ = writer.commit();
+ }
+ }
+
+ Ok(())
+}
+
+/// List of installed lenses
+#[instrument(skip(state))]
+pub async fn list_installed_lenses(state: AppState) -> Result> {
+ let mut lenses: Vec = state
+ .lenses
+ .iter()
+ .map(|lens| LensResult {
+ author: lens.author.clone(),
+ title: lens.name.clone(),
+ description: lens.description.clone().unwrap_or_else(|| "".into()),
+ ..Default::default()
+ })
+ .collect();
+
+ lenses.sort_by(|x, y| x.title.cmp(&y.title));
+
+ Ok(lenses)
+}
+
+/// Show the list of URLs in the queue and their status
+#[instrument(skip(state))]
+pub async fn list_queue(state: AppState) -> Result {
+ let db = &state.db;
+ let queue = crawl_queue::Entity::find().all(db).await;
+
+ match queue {
+ Ok(queue) => Ok(response::ListQueue { queue }),
+ Err(err) => Err(Error {
+ code: ErrorCode::InternalError,
+ message: err.to_string(),
+ data: None,
+ }),
+ }
+}
+
#[instrument(skip(state))]
pub async fn toggle_pause(state: AppState) -> jsonrpc_core::Result {
// Scope so that the app_state mutex is correctly released.
@@ -191,6 +172,60 @@ pub async fn toggle_pause(state: AppState) -> jsonrpc_core::Result {
_get_current_status(state.clone()).await
}
+/// Search the user's indexed documents
+#[instrument(skip(state))]
+pub async fn search(state: AppState, search_req: request::SearchParam) -> Result {
+ let fields = Searcher::doc_fields();
+
+ let index = state.index;
+ let searcher = index.reader.searcher();
+
+ // Create a copy of the lenses for this search
+ let mut lenses = HashMap::new();
+ for entry in state.lenses.iter() {
+ lenses.insert(entry.key().clone(), entry.value().clone());
+ }
+
+ let docs = Searcher::search_with_lens(
+ &lenses,
+ &index.index,
+ &index.reader,
+ &search_req.lenses,
+ &search_req.query,
+ );
+
+ let mut results: Vec = Vec::new();
+ for (score, doc_addr) in docs {
+ let retrieved = searcher.doc(doc_addr).unwrap();
+
+ let doc_id = retrieved.get_first(fields.id).unwrap();
+ let domain = retrieved.get_first(fields.domain).unwrap();
+ let title = retrieved.get_first(fields.title).unwrap();
+ let description = retrieved.get_first(fields.description).unwrap();
+ let url = retrieved.get_first(fields.url).unwrap();
+
+ let result = SearchResult {
+ doc_id: doc_id.as_text().unwrap().to_string(),
+ domain: domain.as_text().unwrap().to_string(),
+ title: title.as_text().unwrap().to_string(),
+ description: description.as_text().unwrap().to_string(),
+ url: url.as_text().unwrap().to_string(),
+ score,
+ };
+
+ results.push(result);
+ }
+
+ let meta = SearchMeta {
+ query: search_req.query,
+ num_docs: searcher.num_docs(),
+ wall_time_ms: 1000,
+ };
+
+ Ok(SearchResults { results, meta })
+}
+
+/// Search the user's installed lenses
#[instrument(skip(state))]
pub async fn search_lenses(
state: AppState,
@@ -212,23 +247,12 @@ pub async fn search_lenses(
let query_results = query_results.unwrap();
for lens in query_results {
results.push(LensResult {
+ author: lens.author,
title: lens.name,
description: lens.description.unwrap_or_else(|| "".to_string()),
+ ..Default::default()
});
}
Ok(SearchLensesResp { results })
}
-
-#[instrument(skip(state))]
-pub async fn delete_doc(state: AppState, id: String) -> Result<()> {
- if let Ok(mut writer) = state.index.writer.lock() {
- if let Err(e) = Searcher::delete(&mut writer, &id) {
- log::error!("Unable to delete doc {} due to {}", id, e);
- } else {
- let _ = writer.commit();
- }
- }
-
- Ok(())
-}
diff --git a/crates/spyglass/src/search/lens.rs b/crates/spyglass/src/search/lens.rs
index cd2427595..17f3178bb 100644
--- a/crates/spyglass/src/search/lens.rs
+++ b/crates/spyglass/src/search/lens.rs
@@ -9,30 +9,6 @@ use crate::crawler::bootstrap;
use crate::search::Searcher;
use crate::state::AppState;
-async fn create_default_lens(config: &Config) {
- // Create a default lens as an example.
- let lens = Lens {
- author: "Spyglass".to_string(),
- version: "1".to_string(),
- name: "rust".to_string(),
- description: Some(
- "All things Rustlang. Search through Rust blogs, the rust book, and
- more."
- .to_string(),
- ),
- domains: vec!["blog.rust-lang.org".into()],
- urls: vec!["https://doc.rust-lang.org/book/".into()],
- is_enabled: true,
- rules: Vec::new(),
- };
-
- fs::write(
- config.lenses_dir().join("rust.ron"),
- ron::ser::to_string_pretty(&lens, Default::default()).unwrap(),
- )
- .expect("Unable to save default lens file.");
-}
-
/// Check if we've already bootstrapped a prefix / otherwise add it to the queue.
async fn check_and_bootstrap(
db: &DatabaseConnection,
@@ -63,7 +39,6 @@ pub async fn read_lenses(state: &AppState, config: &Config) -> anyhow::Result<()
state.lenses.clear();
let lense_dir = config.lenses_dir();
- let mut num_lenses = 0;
for entry in (fs::read_dir(lense_dir)?).flatten() {
let path = entry.path();
@@ -72,7 +47,6 @@ pub async fn read_lenses(state: &AppState, config: &Config) -> anyhow::Result<()
match ron::from_str::(&file_contents) {
Err(err) => log::error!("Unable to load lens {:?}: {}", entry.path(), err),
Ok(lens) => {
- num_lenses += 1;
if lens.is_enabled {
state.lenses.insert(lens.name.clone(), lens);
}
@@ -82,10 +56,6 @@ pub async fn read_lenses(state: &AppState, config: &Config) -> anyhow::Result<()
}
}
- if num_lenses == 0 {
- create_default_lens(config).await;
- }
-
Ok(())
}
diff --git a/crates/tauri/Cargo.toml b/crates/tauri/Cargo.toml
index f649e8614..598baeb26 100644
--- a/crates/tauri/Cargo.toml
+++ b/crates/tauri/Cargo.toml
@@ -22,6 +22,7 @@ log = "0.4"
num-format = "0.4"
open = "2"
reqwest = { version = "0.11", features = ["json"] }
+ron = "0.7"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
shared = { path = "../shared" }
@@ -32,6 +33,7 @@ tracing = "0.1"
tracing-appender = "0.2"
tracing-log = "0.1.3"
tracing-subscriber = { version = "0.3", features = ["env-filter", "std"] }
+url = "2.2"
[features]
default = [ "custom-protocol" ]
diff --git a/crates/tauri/src/cmd.rs b/crates/tauri/src/cmd.rs
index a286aa20a..ae231c8a2 100644
--- a/crates/tauri/src/cmd.rs
+++ b/crates/tauri/src/cmd.rs
@@ -1,8 +1,15 @@
+use std::fs;
+
use jsonrpc_core::Value;
use tauri::State;
+use url::Url;
-use crate::{rpc, window};
-use shared::{request, response};
+use crate::{constants, open_folder, rpc, window};
+use shared::{
+ config::Config,
+ request,
+ response::{self, InstallableLens},
+};
#[tauri::command]
pub async fn escape(window: tauri::Window) -> Result<(), String> {
@@ -10,6 +17,12 @@ pub async fn escape(window: tauri::Window) -> Result<(), String> {
Ok(())
}
+#[tauri::command]
+pub async fn open_lens_folder(_: tauri::Window, config: State<'_, Config>) -> Result<(), String> {
+ open_folder(config.lenses_dir());
+ Ok(())
+}
+
#[tauri::command]
pub async fn open_result(_: tauri::Window, url: &str) -> Result<(), String> {
open::that(url).unwrap();
@@ -21,6 +34,60 @@ pub async fn resize_window(window: tauri::Window, height: f64) {
window::resize_window(&window, height).await;
}
+#[tauri::command]
+pub async fn crawl_stats<'r>(
+ _: tauri::Window,
+ rpc: State<'_, rpc::RpcMutex>,
+) -> Result {
+ let mut rpc = rpc.lock().await;
+ match rpc
+ .client
+ .call_method::("crawl_stats", "", Value::Null)
+ .await
+ {
+ Ok(resp) => Ok(resp),
+ Err(err) => {
+ log::error!("Error sending RPC: {}", err);
+ rpc.reconnect().await;
+ Ok(response::CrawlStats {
+ by_domain: Vec::new(),
+ })
+ }
+ }
+}
+
+#[tauri::command]
+pub async fn list_installed_lenses(
+ _: tauri::Window,
+ rpc: State<'_, rpc::RpcMutex>,
+) -> Result, String> {
+ let mut rpc = rpc.lock().await;
+ Ok(rpc
+ .call::>("list_installed_lenses", Value::Null)
+ .await)
+}
+
+#[tauri::command]
+pub async fn list_installable_lenses(
+ _: tauri::Window,
+) -> Result, String> {
+ let client = reqwest::Client::builder()
+ .user_agent(constants::APP_USER_AGENT)
+ .build()
+ .expect("Unable to create reqwest client");
+
+ if let Ok(res) = client.get(constants::LENS_DIRECTORY_INDEX_URL).send().await {
+ if let Ok(file_contents) = res.text().await {
+ return match ron::from_str::>(&file_contents) {
+ Ok(json) => Ok(json),
+ Err(e) => Err(format!("Unable to parse index: {}", e)),
+ };
+ }
+ }
+
+ Ok(Vec::new())
+}
+
#[tauri::command]
pub async fn search_docs<'r>(
_: tauri::Window,
@@ -57,44 +124,11 @@ pub async fn search_lenses<'r>(
query: query.to_string(),
};
- let rpc = rpc.lock().await;
- match rpc
- .client
- .call_method::<(request::SearchLensesParam,), response::SearchLensesResp>(
- "search_lenses",
- "",
- (data,),
- )
- .await
- {
- Ok(resp) => Ok(resp.results.to_vec()),
- Err(err) => {
- log::error!("{}", err);
- Ok(Vec::new())
- }
- }
-}
-
-#[tauri::command]
-pub async fn crawl_stats<'r>(
- _: tauri::Window,
- rpc: State<'_, rpc::RpcMutex>,
-) -> Result {
let mut rpc = rpc.lock().await;
- match rpc
- .client
- .call_method::("crawl_stats", "", Value::Null)
- .await
- {
- Ok(resp) => Ok(resp),
- Err(err) => {
- log::error!("Error sending RPC: {}", err);
- rpc.reconnect().await;
- Ok(response::CrawlStats {
- by_domain: Vec::new(),
- })
- }
- }
+ let resp = rpc
+ .call::<(request::SearchLensesParam,), response::SearchLensesResp>("search_lenses", (data,))
+ .await;
+ Ok(resp.results)
}
#[tauri::command]
@@ -120,3 +154,45 @@ pub async fn delete_doc<'r>(
}
}
}
+
+/// Install a lens (assumes correct format) from a URL
+#[tauri::command]
+pub async fn install_lens<'r>(
+ window: tauri::Window,
+ config: State<'_, Config>,
+ download_url: &str,
+) -> Result<(), String> {
+ log::trace!("installing lens from <{}>", download_url);
+
+ let client = reqwest::Client::builder()
+ .user_agent(constants::APP_USER_AGENT)
+ .build()
+ .expect("Unable to create reqwest client");
+
+ if let Ok(resp) = client.get(download_url).send().await {
+ if let Ok(file_contents) = resp.text().await {
+ // Grab the file name from the end of the URL
+ let url = Url::parse(download_url).unwrap();
+ let mut segments = url.path_segments().map(|c| c.collect::>()).unwrap();
+ let file_name = segments.pop().unwrap();
+ // Create path from file name + lens directory
+ let lens_path = config.lenses_dir().join(file_name);
+ log::info!("installing lens to {:?}", lens_path);
+
+ if let Err(e) = fs::write(lens_path.clone(), file_contents) {
+ log::error!(
+ "Unable to install lens {} to {:?} due to error: {}",
+ download_url,
+ lens_path,
+ e
+ );
+ } else {
+ // Sleep for a second to let the app reload the lenses and then let the client know we're done.
+ tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;
+ let _ = window.emit("refresh_lens_manager", true);
+ }
+ }
+ }
+
+ Ok(())
+}
diff --git a/crates/tauri/src/constants.rs b/crates/tauri/src/constants.rs
index 8c38e77c7..d132111bc 100644
--- a/crates/tauri/src/constants.rs
+++ b/crates/tauri/src/constants.rs
@@ -3,4 +3,9 @@ pub const INPUT_Y: f64 = 128.0;
pub const APP_USER_AGENT: &str = "spyglass (github.com/a5huynh/spyglass)";
pub const DISCORD_JOIN_URL: &str = "https://discord.gg/663wPVBSTB";
+pub const LENS_DIRECTORY_INDEX_URL: &str =
+ "https://raw.githubusercontent.com/spyglass-search/lens-box/main/index.ron";
pub const VERSION_CHECK_URL: &str = "https://api.github.com/repos/a5huynh/spyglass/releases";
+
+pub const STATS_WIN_NAME: &str = "crawl_stats";
+pub const LENS_MANAGER_WIN_NAME: &str = "lens_manager";
diff --git a/crates/tauri/src/main.rs b/crates/tauri/src/main.rs
index 654be022b..f86189b20 100644
--- a/crates/tauri/src/main.rs
+++ b/crates/tauri/src/main.rs
@@ -27,7 +27,7 @@ mod constants;
mod menu;
mod rpc;
mod window;
-use window::show_crawl_stats_window;
+use window::{show_crawl_stats_window, show_lens_manager_window};
fn main() -> Result<(), Box> {
let config = Config::new();
@@ -52,13 +52,17 @@ fn main() -> Result<(), Box> {
tauri::Builder::default()
.invoke_handler(tauri::generate_handler![
+ cmd::crawl_stats,
+ cmd::delete_doc,
cmd::escape,
+ cmd::install_lens,
+ cmd::list_installable_lenses,
+ cmd::list_installed_lenses,
+ cmd::open_lens_folder,
cmd::open_result,
+ cmd::resize_window,
cmd::search_docs,
cmd::search_lenses,
- cmd::resize_window,
- cmd::crawl_stats,
- cmd::delete_doc,
])
.menu(menu::get_app_menu())
.system_tray(SystemTray::new().with_menu(menu::get_tray_menu(&config)))
@@ -165,7 +169,9 @@ fn main() -> Result<(), Box> {
let _ = open::that(&version.html_url);
}
}
- menu::OPEN_LENSES_FOLDER => open_folder(config.lenses_dir()),
+ menu::OPEN_LENS_MANAGER => {
+ show_lens_manager_window(app);
+ }
menu::OPEN_LOGS_FOLDER => open_folder(config.logs_dir()),
menu::OPEN_SETTINGS_FOLDER => open_folder(Config::prefs_dir()),
menu::SHOW_CRAWL_STATUS => {
diff --git a/crates/tauri/src/menu.rs b/crates/tauri/src/menu.rs
index 74223e571..5d682386a 100644
--- a/crates/tauri/src/menu.rs
+++ b/crates/tauri/src/menu.rs
@@ -7,7 +7,7 @@ pub const QUIT_MENU_ITEM: &str = "quit";
pub const NUM_DOCS_MENU_ITEM: &str = "num_docs";
pub const CRAWL_STATUS_MENU_ITEM: &str = "crawl_status";
-pub const OPEN_LENSES_FOLDER: &str = "open_lenses_folder";
+pub const OPEN_LENS_MANAGER: &str = "open_lens_manager";
pub const OPEN_SETTINGS_FOLDER: &str = "open_settings_folder";
pub const OPEN_LOGS_FOLDER: &str = "open_logs_folder";
pub const SHOW_SEARCHBAR: &str = "show_searchbar";
@@ -25,12 +25,10 @@ pub fn get_tray_menu(config: &Config) -> SystemTrayMenu {
let pause = CustomMenuItem::new(CRAWL_STATUS_MENU_ITEM.to_string(), "");
let quit = CustomMenuItem::new(QUIT_MENU_ITEM.to_string(), "Quit");
- let open_lenses_folder =
- CustomMenuItem::new(OPEN_LENSES_FOLDER.to_string(), "Show lenses folder");
let open_settings_folder =
- CustomMenuItem::new(OPEN_SETTINGS_FOLDER.to_string(), "Show settings folder");
+ CustomMenuItem::new(OPEN_SETTINGS_FOLDER.to_string(), "Open settings folder");
- let open_logs_folder = CustomMenuItem::new(OPEN_LOGS_FOLDER.to_string(), "Show logs folder");
+ let open_logs_folder = CustomMenuItem::new(OPEN_LOGS_FOLDER.to_string(), "Open logs folder");
let app_version = format!("v20{}", ctx.package_info().version);
let mut tray = SystemTrayMenu::new();
@@ -47,8 +45,11 @@ pub fn get_tray_menu(config: &Config) -> SystemTrayMenu {
SHOW_CRAWL_STATUS.to_string(),
"Show crawl status",
))
+ .add_item(CustomMenuItem::new(
+ OPEN_LENS_MANAGER.to_string(),
+ "Manage/install lenses",
+ ))
.add_native_item(SystemTrayMenuItem::Separator)
- .add_item(open_lenses_folder)
.add_item(open_settings_folder)
.add_item(open_logs_folder);
@@ -56,7 +57,7 @@ pub fn get_tray_menu(config: &Config) -> SystemTrayMenu {
if cfg!(debug_assertions) {
tray = tray
.add_native_item(SystemTrayMenuItem::Separator)
- .add_item(CustomMenuItem::new(DEV_SHOW_CONSOLE, "Show console"));
+ .add_item(CustomMenuItem::new(DEV_SHOW_CONSOLE, "Open dev console"));
}
tray.add_native_item(SystemTrayMenuItem::Separator)
diff --git a/crates/tauri/src/rpc.rs b/crates/tauri/src/rpc.rs
index ee09c19b4..7b7dfc8dd 100644
--- a/crates/tauri/src/rpc.rs
+++ b/crates/tauri/src/rpc.rs
@@ -1,6 +1,8 @@
use std::sync::Arc;
use jsonrpc_core_client::{transports::ipc, TypedClient};
+use serde::de::DeserializeOwned;
+use serde::Serialize;
use shared::rpc::gen_ipc_path;
use tauri::api::process::{Command, CommandEvent};
use tokio::sync::Mutex;
@@ -63,6 +65,21 @@ impl RpcClient {
}
}
+ pub async fn call(
+ &mut self,
+ method: &str,
+ args: T,
+ ) -> R {
+ match self.client.call_method::(method, "", args).await {
+ Ok(resp) => resp,
+ Err(err) => {
+ log::error!("Error sending RPC: {}", err);
+ self.reconnect().await;
+ R::default()
+ }
+ }
+ }
+
pub async fn reconnect(&mut self) {
log::info!("Attempting to restart backend");
// Attempt to reconnect
diff --git a/crates/tauri/src/window.rs b/crates/tauri/src/window.rs
index b59dc4d30..ea110e2b4 100644
--- a/crates/tauri/src/window.rs
+++ b/crates/tauri/src/window.rs
@@ -40,14 +40,35 @@ pub fn show_window(window: &Window) {
}
pub fn show_crawl_stats_window(app: &AppHandle) -> Window {
- if let Some(window) = app.get_window("stats") {
+ if let Some(window) = app.get_window(constants::STATS_WIN_NAME) {
let _ = window.show();
let _ = window.set_focus();
return window;
}
- WindowBuilder::new(app, "stats", WindowUrl::App("/stats".into()))
- .title("Status")
- .build()
- .unwrap()
+ WindowBuilder::new(
+ app,
+ constants::STATS_WIN_NAME,
+ WindowUrl::App("/stats".into()),
+ )
+ .title("Status")
+ .build()
+ .unwrap()
+}
+
+pub fn show_lens_manager_window(app: &AppHandle) -> Window {
+ if let Some(window) = app.get_window(constants::LENS_MANAGER_WIN_NAME) {
+ let _ = window.show();
+ let _ = window.set_focus();
+ return window;
+ }
+
+ WindowBuilder::new(
+ app,
+ constants::LENS_MANAGER_WIN_NAME,
+ WindowUrl::App("/settings/lens".into()),
+ )
+ .title("Lens Manager")
+ .build()
+ .unwrap()
}
diff --git a/crates/tauri/tauri.conf.json b/crates/tauri/tauri.conf.json
index b9cc8aad5..fd52bfb26 100644
--- a/crates/tauri/tauri.conf.json
+++ b/crates/tauri/tauri.conf.json
@@ -1,7 +1,7 @@
{
"package": {
"productName": "Spyglass",
- "version": "22.5.28"
+ "version": "22.6.1"
},
"build": {
"distDir": "../client/dist",
{result.description.clone()}
+
+ }
+ } else {
+ html! {
+
+ }
+ }
+}
+
+#[function_component(Lens)]
+pub fn lens_component(props: &LensProps) -> Html {
+ let component_styles: Vec{"Installing"}
+
+
+ }
+ } else {
+ html! { {"Installed"}
+ {"View Source"}
+
+ }
+ } else {
+ html! {}
+ };
+
+ html! {
+
+
+ }
+}
+
+#[function_component(LensManagerPage)]
+pub fn lens_manager_page() -> Html {
+ let user_installed: UseStateHandle+ {result.title.clone()} +
++ {"Crafted By:"} + + {format!("@{}", result.author)} + +
+
+ {result.description.clone()}
+
+
+ {installed_el}
+ {view_link}
+
+
+
+ }
+ };
+
+ html! {
+
+
+
+
+ }
+}
diff --git a/crates/client/src/pages/mod.rs b/crates/client/src/pages/mod.rs
index 44cbc773d..ef47f47b6 100644
--- a/crates/client/src/pages/mod.rs
+++ b/crates/client/src/pages/mod.rs
@@ -1,3 +1,6 @@
+mod lens_manager;
+pub use lens_manager::*;
+
mod search;
pub use search::*;
diff --git a/crates/client/src/pages/search.rs b/crates/client/src/pages/search.rs
index b9c3bae1c..3b325926a 100644
--- a/crates/client/src/pages/search.rs
+++ b/crates/client/src/pages/search.rs
@@ -172,18 +172,18 @@ pub fn search_page() -> Html {
html! {
+
+
+
+ {"Lens Manager"}
+ + +
+ {contents}
+
+
-
+
>) {
+fn fetch_crawl_stats(
+ stats_handle: UseStateHandle>,
+ request_finished: UseStateHandle,
+) {
spawn_local(async move {
match crawl_stats().await {
Ok(results) => {
@@ -13,8 +17,12 @@ fn fetch_crawl_stats(stats_handle: UseStateHandle>) {
let mut sorted = results.by_domain;
sorted.sort_by(|(_, a), (_, b)| b.num_completed.cmp(&a.num_completed));
stats_handle.set(sorted);
+ request_finished.set(true);
+ }
+ Err(e) => {
+ log::info!("Error: {:?}", e);
+ request_finished.set(true);
}
- Err(e) => log::info!("Error: {:?}", e),
}
});
}
@@ -39,7 +47,7 @@ pub fn legend_icon(props: &LegendIconProps) -> Html {
];
html! {
-
{ results }
diff --git a/crates/client/src/pages/stats.rs b/crates/client/src/pages/stats.rs
index c3eb15579..9b2b13583 100644
--- a/crates/client/src/pages/stats.rs
+++ b/crates/client/src/pages/stats.rs
@@ -2,10 +2,14 @@ use num_format::{Buffer, Locale};
use wasm_bindgen_futures::spawn_local;
use yew::prelude::*;
+use crate::components::icons;
use crate::crawl_stats;
use shared::response::{CrawlStats, QueueStatus};
-fn fetch_crawl_stats(stats_handle: UseStateHandle
+ > = use_state_eq(Vec::new);
- if stats.is_empty() {
- fetch_crawl_stats(stats.clone());
+ let request_finished = use_state(|| false);
+
+ if stats.is_empty() && !(*request_finished) {
+ fetch_crawl_stats(stats.clone(), request_finished.clone());
}
let onclick = {
+ let request_finished = request_finished.clone();
let stats = stats.clone();
move |_| {
+ request_finished.set(false);
stats.set(Vec::new());
- fetch_crawl_stats(stats.clone());
+ fetch_crawl_stats(stats.clone(), request_finished.clone());
}
};
@@ -107,11 +119,11 @@ pub fn stats_page() -> Html {
.map(|(domain, stats)| {
let total = stats.total() as f64;
html! {
-
{props.label.clone()}
@@ -82,7 +90,7 @@ fn stats_bar(props: &StatsBarProps) -> Html {
html! {
}
}
@@ -90,15 +98,19 @@ fn stats_bar(props: &StatsBarProps) -> Html {
#[function_component(StatsPage)]
pub fn stats_page() -> Html {
let stats: UseStateHandle
-
+
+
{domain}
-
+
-
+
+
+
}
}
html! {
-
+
+
- ,
+ pub download_url: Option,
+}
+
#[derive(Debug, Deserialize, Serialize)]
pub struct SearchMeta {
pub query: String,
@@ -48,13 +68,7 @@ pub struct SearchResults {
pub meta: SearchMeta,
}
-#[derive(Clone, Debug, Deserialize, Serialize)]
-pub struct LensResult {
- pub title: String,
- pub description: String,
-}
-
-#[derive(Debug, Deserialize, Serialize)]
+#[derive(Debug, Default, Deserialize, Serialize)]
pub struct SearchLensesResp {
pub results: Vec,
}
diff --git a/crates/shared/src/rpc.rs b/crates/shared/src/rpc.rs
index 098f20a4e..e4d142e39 100644
--- a/crates/shared/src/rpc.rs
+++ b/crates/shared/src/rpc.rs
@@ -2,7 +2,7 @@ use jsonrpc_core::{BoxFuture, Result};
use jsonrpc_derive::rpc;
use crate::request::{SearchLensesParam, SearchParam};
-use crate::response::{AppStatus, CrawlStats, SearchLensesResp, SearchResults};
+use crate::response::{AppStatus, CrawlStats, LensResult, SearchLensesResp, SearchResults};
pub fn gen_ipc_path() -> String {
if cfg!(windows) {
@@ -28,12 +28,15 @@ pub trait Rpc {
#[rpc(name = "delete_doc")]
fn delete_doc(&self, id: String) -> BoxFuture>;
- #[rpc(name = "toggle_pause")]
- fn toggle_pause(&self) -> BoxFuture>;
+ #[rpc(name = "list_installed_lenses")]
+ fn list_installed_lenses(&self) -> BoxFuture>>;
#[rpc(name = "search_docs")]
fn search_docs(&self, query: SearchParam) -> BoxFuture>;
#[rpc(name = "search_lenses")]
fn search_lenses(&self, query: SearchLensesParam) -> BoxFuture>;
+
+ #[rpc(name = "toggle_pause")]
+ fn toggle_pause(&self) -> BoxFuture>;
}
diff --git a/crates/spyglass/src/api/mod.rs b/crates/spyglass/src/api/mod.rs
index 953d154c0..cac18a252 100644
--- a/crates/spyglass/src/api/mod.rs
+++ b/crates/spyglass/src/api/mod.rs
@@ -6,7 +6,7 @@ use jsonrpc_ipc_server::{Server, ServerBuilder};
use libspyglass::state::AppState;
use shared::request::{SearchLensesParam, SearchParam};
-use shared::response::{AppStatus, CrawlStats, SearchLensesResp, SearchResults};
+use shared::response::{AppStatus, CrawlStats, LensResult, SearchLensesResp, SearchResults};
use shared::rpc::{gen_ipc_path, Rpc};
mod response;
@@ -33,8 +33,8 @@ impl Rpc for SpyglassRPC {
Box::pin(route::delete_doc(self.state.clone(), id))
}
- fn toggle_pause(&self) -> BoxFuture> {
- Box::pin(route::toggle_pause(self.state.clone()))
+ fn list_installed_lenses(&self) -> BoxFuture>> {
+ Box::pin(route::list_installed_lenses(self.state.clone()))
}
fn search_docs(&self, query: SearchParam) -> BoxFuture> {
@@ -44,6 +44,10 @@ impl Rpc for SpyglassRPC {
fn search_lenses(&self, query: SearchLensesParam) -> BoxFuture> {
Box::pin(route::search_lenses(self.state.clone(), query))
}
+
+ fn toggle_pause(&self) -> BoxFuture> {
+ Box::pin(route::toggle_pause(self.state.clone()))
+ }
}
pub fn start_api_ipc(state: &AppState) -> anyhow::Result {
diff --git a/crates/spyglass/src/api/route.rs b/crates/spyglass/src/api/route.rs
index e4e27692d..06385d5ff 100644
--- a/crates/spyglass/src/api/route.rs
+++ b/crates/spyglass/src/api/route.rs
@@ -19,74 +19,6 @@ use libspyglass::state::AppState;
use super::response;
-#[instrument(skip(state))]
-pub async fn search(state: AppState, search_req: request::SearchParam) -> Result {
- let fields = Searcher::doc_fields();
-
- let index = state.index;
- let searcher = index.reader.searcher();
-
- // Create a copy of the lenses for this search
- let mut lenses = HashMap::new();
- for entry in state.lenses.iter() {
- lenses.insert(entry.key().clone(), entry.value().clone());
- }
-
- let docs = Searcher::search_with_lens(
- &lenses,
- &index.index,
- &index.reader,
- &search_req.lenses,
- &search_req.query,
- );
-
- let mut results: Vec = Vec::new();
- for (score, doc_addr) in docs {
- let retrieved = searcher.doc(doc_addr).unwrap();
-
- let doc_id = retrieved.get_first(fields.id).unwrap();
- let domain = retrieved.get_first(fields.domain).unwrap();
- let title = retrieved.get_first(fields.title).unwrap();
- let description = retrieved.get_first(fields.description).unwrap();
- let url = retrieved.get_first(fields.url).unwrap();
-
- let result = SearchResult {
- doc_id: doc_id.as_text().unwrap().to_string(),
- domain: domain.as_text().unwrap().to_string(),
- title: title.as_text().unwrap().to_string(),
- description: description.as_text().unwrap().to_string(),
- url: url.as_text().unwrap().to_string(),
- score,
- };
-
- results.push(result);
- }
-
- let meta = SearchMeta {
- query: search_req.query,
- num_docs: searcher.num_docs(),
- wall_time_ms: 1000,
- };
-
- Ok(SearchResults { results, meta })
-}
-
-/// Show the list of URLs in the queue and their status
-#[instrument(skip(state))]
-pub async fn list_queue(state: AppState) -> Result {
- let db = &state.db;
- let queue = crawl_queue::Entity::find().all(db).await;
-
- match queue {
- Ok(queue) => Ok(response::ListQueue { queue }),
- Err(err) => Err(Error {
- code: ErrorCode::InternalError,
- message: err.to_string(),
- data: None,
- }),
- }
-}
-
/// Add url to queue
#[instrument(skip(state))]
pub async fn add_queue(state: AppState, queue_item: request::QueueItemParam) -> Result {
@@ -170,12 +102,61 @@ pub async fn crawl_stats(state: AppState) -> jsonrpc_core::Result {
let by_domain = by_domain
.into_iter()
- .filter(|(_, stats)| stats.total() >= 100)
+ .filter(|(_, stats)| stats.total() >= 10)
.collect();
Ok(CrawlStats { by_domain })
}
+/// Remove a doc from the index
+#[instrument(skip(state))]
+pub async fn delete_doc(state: AppState, id: String) -> Result<()> {
+ if let Ok(mut writer) = state.index.writer.lock() {
+ if let Err(e) = Searcher::delete(&mut writer, &id) {
+ log::error!("Unable to delete doc {} due to {}", id, e);
+ } else {
+ let _ = writer.commit();
+ }
+ }
+
+ Ok(())
+}
+
+/// List of installed lenses
+#[instrument(skip(state))]
+pub async fn list_installed_lenses(state: AppState) -> Result> {
+ let mut lenses: Vec = state
+ .lenses
+ .iter()
+ .map(|lens| LensResult {
+ author: lens.author.clone(),
+ title: lens.name.clone(),
+ description: lens.description.clone().unwrap_or_else(|| "".into()),
+ ..Default::default()
+ })
+ .collect();
+
+ lenses.sort_by(|x, y| x.title.cmp(&y.title));
+
+ Ok(lenses)
+}
+
+/// Show the list of URLs in the queue and their status
+#[instrument(skip(state))]
+pub async fn list_queue(state: AppState) -> Result {
+ let db = &state.db;
+ let queue = crawl_queue::Entity::find().all(db).await;
+
+ match queue {
+ Ok(queue) => Ok(response::ListQueue { queue }),
+ Err(err) => Err(Error {
+ code: ErrorCode::InternalError,
+ message: err.to_string(),
+ data: None,
+ }),
+ }
+}
+
#[instrument(skip(state))]
pub async fn toggle_pause(state: AppState) -> jsonrpc_core::Result {
// Scope so that the app_state mutex is correctly released.
@@ -191,6 +172,60 @@ pub async fn toggle_pause(state: AppState) -> jsonrpc_core::Result {
_get_current_status(state.clone()).await
}
+/// Search the user's indexed documents
+#[instrument(skip(state))]
+pub async fn search(state: AppState, search_req: request::SearchParam) -> Result {
+ let fields = Searcher::doc_fields();
+
+ let index = state.index;
+ let searcher = index.reader.searcher();
+
+ // Create a copy of the lenses for this search
+ let mut lenses = HashMap::new();
+ for entry in state.lenses.iter() {
+ lenses.insert(entry.key().clone(), entry.value().clone());
+ }
+
+ let docs = Searcher::search_with_lens(
+ &lenses,
+ &index.index,
+ &index.reader,
+ &search_req.lenses,
+ &search_req.query,
+ );
+
+ let mut results: Vec = Vec::new();
+ for (score, doc_addr) in docs {
+ let retrieved = searcher.doc(doc_addr).unwrap();
+
+ let doc_id = retrieved.get_first(fields.id).unwrap();
+ let domain = retrieved.get_first(fields.domain).unwrap();
+ let title = retrieved.get_first(fields.title).unwrap();
+ let description = retrieved.get_first(fields.description).unwrap();
+ let url = retrieved.get_first(fields.url).unwrap();
+
+ let result = SearchResult {
+ doc_id: doc_id.as_text().unwrap().to_string(),
+ domain: domain.as_text().unwrap().to_string(),
+ title: title.as_text().unwrap().to_string(),
+ description: description.as_text().unwrap().to_string(),
+ url: url.as_text().unwrap().to_string(),
+ score,
+ };
+
+ results.push(result);
+ }
+
+ let meta = SearchMeta {
+ query: search_req.query,
+ num_docs: searcher.num_docs(),
+ wall_time_ms: 1000,
+ };
+
+ Ok(SearchResults { results, meta })
+}
+
+/// Search the user's installed lenses
#[instrument(skip(state))]
pub async fn search_lenses(
state: AppState,
@@ -212,23 +247,12 @@ pub async fn search_lenses(
let query_results = query_results.unwrap();
for lens in query_results {
results.push(LensResult {
+ author: lens.author,
title: lens.name,
description: lens.description.unwrap_or_else(|| "".to_string()),
+ ..Default::default()
});
}
Ok(SearchLensesResp { results })
}
-
-#[instrument(skip(state))]
-pub async fn delete_doc(state: AppState, id: String) -> Result<()> {
- if let Ok(mut writer) = state.index.writer.lock() {
- if let Err(e) = Searcher::delete(&mut writer, &id) {
- log::error!("Unable to delete doc {} due to {}", id, e);
- } else {
- let _ = writer.commit();
- }
- }
-
- Ok(())
-}
diff --git a/crates/spyglass/src/search/lens.rs b/crates/spyglass/src/search/lens.rs
index cd2427595..17f3178bb 100644
--- a/crates/spyglass/src/search/lens.rs
+++ b/crates/spyglass/src/search/lens.rs
@@ -9,30 +9,6 @@ use crate::crawler::bootstrap;
use crate::search::Searcher;
use crate::state::AppState;
-async fn create_default_lens(config: &Config) {
- // Create a default lens as an example.
- let lens = Lens {
- author: "Spyglass".to_string(),
- version: "1".to_string(),
- name: "rust".to_string(),
- description: Some(
- "All things Rustlang. Search through Rust blogs, the rust book, and
- more."
- .to_string(),
- ),
- domains: vec!["blog.rust-lang.org".into()],
- urls: vec!["https://doc.rust-lang.org/book/".into()],
- is_enabled: true,
- rules: Vec::new(),
- };
-
- fs::write(
- config.lenses_dir().join("rust.ron"),
- ron::ser::to_string_pretty(&lens, Default::default()).unwrap(),
- )
- .expect("Unable to save default lens file.");
-}
-
/// Check if we've already bootstrapped a prefix / otherwise add it to the queue.
async fn check_and_bootstrap(
db: &DatabaseConnection,
@@ -63,7 +39,6 @@ pub async fn read_lenses(state: &AppState, config: &Config) -> anyhow::Result<()
state.lenses.clear();
let lense_dir = config.lenses_dir();
- let mut num_lenses = 0;
for entry in (fs::read_dir(lense_dir)?).flatten() {
let path = entry.path();
@@ -72,7 +47,6 @@ pub async fn read_lenses(state: &AppState, config: &Config) -> anyhow::Result<()
match ron::from_str::(&file_contents) {
Err(err) => log::error!("Unable to load lens {:?}: {}", entry.path(), err),
Ok(lens) => {
- num_lenses += 1;
if lens.is_enabled {
state.lenses.insert(lens.name.clone(), lens);
}
@@ -82,10 +56,6 @@ pub async fn read_lenses(state: &AppState, config: &Config) -> anyhow::Result<()
}
}
- if num_lenses == 0 {
- create_default_lens(config).await;
- }
-
Ok(())
}
diff --git a/crates/tauri/Cargo.toml b/crates/tauri/Cargo.toml
index f649e8614..598baeb26 100644
--- a/crates/tauri/Cargo.toml
+++ b/crates/tauri/Cargo.toml
@@ -22,6 +22,7 @@ log = "0.4"
num-format = "0.4"
open = "2"
reqwest = { version = "0.11", features = ["json"] }
+ron = "0.7"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
shared = { path = "../shared" }
@@ -32,6 +33,7 @@ tracing = "0.1"
tracing-appender = "0.2"
tracing-log = "0.1.3"
tracing-subscriber = { version = "0.3", features = ["env-filter", "std"] }
+url = "2.2"
[features]
default = [ "custom-protocol" ]
diff --git a/crates/tauri/src/cmd.rs b/crates/tauri/src/cmd.rs
index a286aa20a..ae231c8a2 100644
--- a/crates/tauri/src/cmd.rs
+++ b/crates/tauri/src/cmd.rs
@@ -1,8 +1,15 @@
+use std::fs;
+
use jsonrpc_core::Value;
use tauri::State;
+use url::Url;
-use crate::{rpc, window};
-use shared::{request, response};
+use crate::{constants, open_folder, rpc, window};
+use shared::{
+ config::Config,
+ request,
+ response::{self, InstallableLens},
+};
#[tauri::command]
pub async fn escape(window: tauri::Window) -> Result<(), String> {
@@ -10,6 +17,12 @@ pub async fn escape(window: tauri::Window) -> Result<(), String> {
Ok(())
}
+#[tauri::command]
+pub async fn open_lens_folder(_: tauri::Window, config: State<'_, Config>) -> Result<(), String> {
+ open_folder(config.lenses_dir());
+ Ok(())
+}
+
#[tauri::command]
pub async fn open_result(_: tauri::Window, url: &str) -> Result<(), String> {
open::that(url).unwrap();
@@ -21,6 +34,60 @@ pub async fn resize_window(window: tauri::Window, height: f64) {
window::resize_window(&window, height).await;
}
+#[tauri::command]
+pub async fn crawl_stats<'r>(
+ _: tauri::Window,
+ rpc: State<'_, rpc::RpcMutex>,
+) -> Result {
+ let mut rpc = rpc.lock().await;
+ match rpc
+ .client
+ .call_method::("crawl_stats", "", Value::Null)
+ .await
+ {
+ Ok(resp) => Ok(resp),
+ Err(err) => {
+ log::error!("Error sending RPC: {}", err);
+ rpc.reconnect().await;
+ Ok(response::CrawlStats {
+ by_domain: Vec::new(),
+ })
+ }
+ }
+}
+
+#[tauri::command]
+pub async fn list_installed_lenses(
+ _: tauri::Window,
+ rpc: State<'_, rpc::RpcMutex>,
+) -> Result, String> {
+ let mut rpc = rpc.lock().await;
+ Ok(rpc
+ .call::>("list_installed_lenses", Value::Null)
+ .await)
+}
+
+#[tauri::command]
+pub async fn list_installable_lenses(
+ _: tauri::Window,
+) -> Result, String> {
+ let client = reqwest::Client::builder()
+ .user_agent(constants::APP_USER_AGENT)
+ .build()
+ .expect("Unable to create reqwest client");
+
+ if let Ok(res) = client.get(constants::LENS_DIRECTORY_INDEX_URL).send().await {
+ if let Ok(file_contents) = res.text().await {
+ return match ron::from_str::>(&file_contents) {
+ Ok(json) => Ok(json),
+ Err(e) => Err(format!("Unable to parse index: {}", e)),
+ };
+ }
+ }
+
+ Ok(Vec::new())
+}
+
#[tauri::command]
pub async fn search_docs<'r>(
_: tauri::Window,
@@ -57,44 +124,11 @@ pub async fn search_lenses<'r>(
query: query.to_string(),
};
- let rpc = rpc.lock().await;
- match rpc
- .client
- .call_method::<(request::SearchLensesParam,), response::SearchLensesResp>(
- "search_lenses",
- "",
- (data,),
- )
- .await
- {
- Ok(resp) => Ok(resp.results.to_vec()),
- Err(err) => {
- log::error!("{}", err);
- Ok(Vec::new())
- }
- }
-}
-
-#[tauri::command]
-pub async fn crawl_stats<'r>(
- _: tauri::Window,
- rpc: State<'_, rpc::RpcMutex>,
-) -> Result {
let mut rpc = rpc.lock().await;
- match rpc
- .client
- .call_method::("crawl_stats", "", Value::Null)
- .await
- {
- Ok(resp) => Ok(resp),
- Err(err) => {
- log::error!("Error sending RPC: {}", err);
- rpc.reconnect().await;
- Ok(response::CrawlStats {
- by_domain: Vec::new(),
- })
- }
- }
+ let resp = rpc
+ .call::<(request::SearchLensesParam,), response::SearchLensesResp>("search_lenses", (data,))
+ .await;
+ Ok(resp.results)
}
#[tauri::command]
@@ -120,3 +154,45 @@ pub async fn delete_doc<'r>(
}
}
}
+
+/// Install a lens (assumes correct format) from a URL
+#[tauri::command]
+pub async fn install_lens<'r>(
+ window: tauri::Window,
+ config: State<'_, Config>,
+ download_url: &str,
+) -> Result<(), String> {
+ log::trace!("installing lens from <{}>", download_url);
+
+ let client = reqwest::Client::builder()
+ .user_agent(constants::APP_USER_AGENT)
+ .build()
+ .expect("Unable to create reqwest client");
+
+ if let Ok(resp) = client.get(download_url).send().await {
+ if let Ok(file_contents) = resp.text().await {
+ // Grab the file name from the end of the URL
+ let url = Url::parse(download_url).unwrap();
+ let mut segments = url.path_segments().map(|c| c.collect::>()).unwrap();
+ let file_name = segments.pop().unwrap();
+ // Create path from file name + lens directory
+ let lens_path = config.lenses_dir().join(file_name);
+ log::info!("installing lens to {:?}", lens_path);
+
+ if let Err(e) = fs::write(lens_path.clone(), file_contents) {
+ log::error!(
+ "Unable to install lens {} to {:?} due to error: {}",
+ download_url,
+ lens_path,
+ e
+ );
+ } else {
+ // Sleep for a second to let the app reload the lenses and then let the client know we're done.
+ tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;
+ let _ = window.emit("refresh_lens_manager", true);
+ }
+ }
+ }
+
+ Ok(())
+}
diff --git a/crates/tauri/src/constants.rs b/crates/tauri/src/constants.rs
index 8c38e77c7..d132111bc 100644
--- a/crates/tauri/src/constants.rs
+++ b/crates/tauri/src/constants.rs
@@ -3,4 +3,9 @@ pub const INPUT_Y: f64 = 128.0;
pub const APP_USER_AGENT: &str = "spyglass (github.com/a5huynh/spyglass)";
pub const DISCORD_JOIN_URL: &str = "https://discord.gg/663wPVBSTB";
+pub const LENS_DIRECTORY_INDEX_URL: &str =
+ "https://raw.githubusercontent.com/spyglass-search/lens-box/main/index.ron";
pub const VERSION_CHECK_URL: &str = "https://api.github.com/repos/a5huynh/spyglass/releases";
+
+pub const STATS_WIN_NAME: &str = "crawl_stats";
+pub const LENS_MANAGER_WIN_NAME: &str = "lens_manager";
diff --git a/crates/tauri/src/main.rs b/crates/tauri/src/main.rs
index 654be022b..f86189b20 100644
--- a/crates/tauri/src/main.rs
+++ b/crates/tauri/src/main.rs
@@ -27,7 +27,7 @@ mod constants;
mod menu;
mod rpc;
mod window;
-use window::show_crawl_stats_window;
+use window::{show_crawl_stats_window, show_lens_manager_window};
fn main() -> Result<(), Box> {
let config = Config::new();
@@ -52,13 +52,17 @@ fn main() -> Result<(), Box> {
tauri::Builder::default()
.invoke_handler(tauri::generate_handler![
+ cmd::crawl_stats,
+ cmd::delete_doc,
cmd::escape,
+ cmd::install_lens,
+ cmd::list_installable_lenses,
+ cmd::list_installed_lenses,
+ cmd::open_lens_folder,
cmd::open_result,
+ cmd::resize_window,
cmd::search_docs,
cmd::search_lenses,
- cmd::resize_window,
- cmd::crawl_stats,
- cmd::delete_doc,
])
.menu(menu::get_app_menu())
.system_tray(SystemTray::new().with_menu(menu::get_tray_menu(&config)))
@@ -165,7 +169,9 @@ fn main() -> Result<(), Box> {
let _ = open::that(&version.html_url);
}
}
- menu::OPEN_LENSES_FOLDER => open_folder(config.lenses_dir()),
+ menu::OPEN_LENS_MANAGER => {
+ show_lens_manager_window(app);
+ }
menu::OPEN_LOGS_FOLDER => open_folder(config.logs_dir()),
menu::OPEN_SETTINGS_FOLDER => open_folder(Config::prefs_dir()),
menu::SHOW_CRAWL_STATUS => {
diff --git a/crates/tauri/src/menu.rs b/crates/tauri/src/menu.rs
index 74223e571..5d682386a 100644
--- a/crates/tauri/src/menu.rs
+++ b/crates/tauri/src/menu.rs
@@ -7,7 +7,7 @@ pub const QUIT_MENU_ITEM: &str = "quit";
pub const NUM_DOCS_MENU_ITEM: &str = "num_docs";
pub const CRAWL_STATUS_MENU_ITEM: &str = "crawl_status";
-pub const OPEN_LENSES_FOLDER: &str = "open_lenses_folder";
+pub const OPEN_LENS_MANAGER: &str = "open_lens_manager";
pub const OPEN_SETTINGS_FOLDER: &str = "open_settings_folder";
pub const OPEN_LOGS_FOLDER: &str = "open_logs_folder";
pub const SHOW_SEARCHBAR: &str = "show_searchbar";
@@ -25,12 +25,10 @@ pub fn get_tray_menu(config: &Config) -> SystemTrayMenu {
let pause = CustomMenuItem::new(CRAWL_STATUS_MENU_ITEM.to_string(), "");
let quit = CustomMenuItem::new(QUIT_MENU_ITEM.to_string(), "Quit");
- let open_lenses_folder =
- CustomMenuItem::new(OPEN_LENSES_FOLDER.to_string(), "Show lenses folder");
let open_settings_folder =
- CustomMenuItem::new(OPEN_SETTINGS_FOLDER.to_string(), "Show settings folder");
+ CustomMenuItem::new(OPEN_SETTINGS_FOLDER.to_string(), "Open settings folder");
- let open_logs_folder = CustomMenuItem::new(OPEN_LOGS_FOLDER.to_string(), "Show logs folder");
+ let open_logs_folder = CustomMenuItem::new(OPEN_LOGS_FOLDER.to_string(), "Open logs folder");
let app_version = format!("v20{}", ctx.package_info().version);
let mut tray = SystemTrayMenu::new();
@@ -47,8 +45,11 @@ pub fn get_tray_menu(config: &Config) -> SystemTrayMenu {
SHOW_CRAWL_STATUS.to_string(),
"Show crawl status",
))
+ .add_item(CustomMenuItem::new(
+ OPEN_LENS_MANAGER.to_string(),
+ "Manage/install lenses",
+ ))
.add_native_item(SystemTrayMenuItem::Separator)
- .add_item(open_lenses_folder)
.add_item(open_settings_folder)
.add_item(open_logs_folder);
@@ -56,7 +57,7 @@ pub fn get_tray_menu(config: &Config) -> SystemTrayMenu {
if cfg!(debug_assertions) {
tray = tray
.add_native_item(SystemTrayMenuItem::Separator)
- .add_item(CustomMenuItem::new(DEV_SHOW_CONSOLE, "Show console"));
+ .add_item(CustomMenuItem::new(DEV_SHOW_CONSOLE, "Open dev console"));
}
tray.add_native_item(SystemTrayMenuItem::Separator)
diff --git a/crates/tauri/src/rpc.rs b/crates/tauri/src/rpc.rs
index ee09c19b4..7b7dfc8dd 100644
--- a/crates/tauri/src/rpc.rs
+++ b/crates/tauri/src/rpc.rs
@@ -1,6 +1,8 @@
use std::sync::Arc;
use jsonrpc_core_client::{transports::ipc, TypedClient};
+use serde::de::DeserializeOwned;
+use serde::Serialize;
use shared::rpc::gen_ipc_path;
use tauri::api::process::{Command, CommandEvent};
use tokio::sync::Mutex;
@@ -63,6 +65,21 @@ impl RpcClient {
}
}
+ pub async fn call(
+ &mut self,
+ method: &str,
+ args: T,
+ ) -> R {
+ match self.client.call_method::(method, "", args).await {
+ Ok(resp) => resp,
+ Err(err) => {
+ log::error!("Error sending RPC: {}", err);
+ self.reconnect().await;
+ R::default()
+ }
+ }
+ }
+
pub async fn reconnect(&mut self) {
log::info!("Attempting to restart backend");
// Attempt to reconnect
diff --git a/crates/tauri/src/window.rs b/crates/tauri/src/window.rs
index b59dc4d30..ea110e2b4 100644
--- a/crates/tauri/src/window.rs
+++ b/crates/tauri/src/window.rs
@@ -40,14 +40,35 @@ pub fn show_window(window: &Window) {
}
pub fn show_crawl_stats_window(app: &AppHandle) -> Window {
- if let Some(window) = app.get_window("stats") {
+ if let Some(window) = app.get_window(constants::STATS_WIN_NAME) {
let _ = window.show();
let _ = window.set_focus();
return window;
}
- WindowBuilder::new(app, "stats", WindowUrl::App("/stats".into()))
- .title("Status")
- .build()
- .unwrap()
+ WindowBuilder::new(
+ app,
+ constants::STATS_WIN_NAME,
+ WindowUrl::App("/stats".into()),
+ )
+ .title("Status")
+ .build()
+ .unwrap()
+}
+
+pub fn show_lens_manager_window(app: &AppHandle) -> Window {
+ if let Some(window) = app.get_window(constants::LENS_MANAGER_WIN_NAME) {
+ let _ = window.show();
+ let _ = window.set_focus();
+ return window;
+ }
+
+ WindowBuilder::new(
+ app,
+ constants::LENS_MANAGER_WIN_NAME,
+ WindowUrl::App("/settings/lens".into()),
+ )
+ .title("Lens Manager")
+ .build()
+ .unwrap()
}
diff --git a/crates/tauri/tauri.conf.json b/crates/tauri/tauri.conf.json
index b9cc8aad5..fd52bfb26 100644
--- a/crates/tauri/tauri.conf.json
+++ b/crates/tauri/tauri.conf.json
@@ -1,7 +1,7 @@
{
"package": {
"productName": "Spyglass",
- "version": "22.5.28"
+ "version": "22.6.1"
},
"build": {
"distDir": "../client/dist",
-
+
+
{"Crawl Status"}
@@ -158,7 +174,7 @@ pub fn stats_page() -> Html {
+
diff --git a/crates/entities/src/models/crawl_queue.rs b/crates/entities/src/models/crawl_queue.rs
index 3fc69c639..9ace3ffd7 100644
--- a/crates/entities/src/models/crawl_queue.rs
+++ b/crates/entities/src/models/crawl_queue.rs
@@ -152,7 +152,7 @@ struct CrawlQueueCount {
count: i64,
}
-#[derive(FromQueryResult)]
+#[derive(Debug, FromQueryResult)]
pub struct QueueCountByStatus {
pub count: i64,
pub domain: String,
diff --git a/crates/entities/src/models/indexed_document.rs b/crates/entities/src/models/indexed_document.rs
index e7500f7f3..3bd10626a 100644
--- a/crates/entities/src/models/indexed_document.rs
+++ b/crates/entities/src/models/indexed_document.rs
@@ -46,7 +46,7 @@ impl ActiveModelBehavior for ActiveModel {
}
}
-#[derive(FromQueryResult)]
+#[derive(Debug, FromQueryResult)]
pub struct CountByDomain {
pub count: i64,
pub domain: String,
diff --git a/crates/shared/src/response.rs b/crates/shared/src/response.rs
index 5ab26460d..e54be89a4 100644
--- a/crates/shared/src/response.rs
+++ b/crates/shared/src/response.rs
@@ -25,6 +25,26 @@ pub struct CrawlStats {
pub by_domain: Vec<(String, QueueStatus)>,
}
+#[derive(Clone, Deserialize, Serialize)]
+pub struct InstallableLens {
+ pub author: String,
+ pub description: String,
+ pub name: String,
+ pub sha: String,
+ pub download_url: String,
+ pub html_url: String,
+}
+
+#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)]
+pub struct LensResult {
+ pub author: String,
+ pub title: String,
+ pub description: String,
+ // Only relevant for installable lenses
+ pub html_url: Option
{rendered}