release: v2023.11.1

Cargo.toml

@@ -1,29 +1,30 @@
-[workspace]
-members = [
-    "crates/client",
-    "crates/entities",
-    "crates/migrations",
-    "crates/shared",
-    "crates/spyglass",
-    "crates/tauri",
-    "crates/ui-components",
-
-    "apps/web",
-
-    # Public published crates
-    "crates/spyglass-plugin",
-    "crates/spyglass-lens",
-    "crates/spyglass-rpc",
-    "crates/spyglass-searcher",
-    "crates/spyglass-processor"
-]
-
-[profile.release]
-codegen-units = 1
-lto = true
-incremental = false
-opt-level = "s"
-
-[profile.sentry]
-inherits = "dev"
-debug = true
+[workspace]
+resolver = "2"
+members = [
+    "crates/client",
+    "crates/entities",
+    "crates/migrations",
+    "crates/shared",
+    "crates/spyglass",
+    "crates/tauri",
+    "crates/ui-components",
+
+    "apps/web",
+
+    # Public published crates
+    "crates/spyglass-plugin",
+    "crates/spyglass-lens",
+    "crates/spyglass-rpc",
+    "crates/spyglass-searcher",
+    "crates/spyglass-processor"
+]
+
+[profile.release]
+codegen-units = 1
+lto = true
+incremental = false
+opt-level = "s"
+
+[profile.sentry]
+inherits = "dev"
+debug = true

Makefile

@@ -85,8 +85,14 @@ setup-dev:
 	cd ./crates/client && npm install
 # Download whisper model used in development
 	mkdir -p assets/models;
-	curl -L --output whisper.base.en.bin https://huggingface.co/datasets/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin;
+	curl -L --output whisper.base.en.bin https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin;
 	mv whisper.base.en.bin assets/models;
+# Check if .env exists and if not create it
+	test -f .env || cp .env.template .env
+# Check if /dist folder exists for Tauri and if not create it
+	mkdir -p ./crates/tauri/dist
+# Build backend to copy binaries for Tauri
+	make build-backend
 
 # Specifically for debian based distros
 setup-dev-linux:
@@ -102,8 +108,11 @@ setup-dev-linux:
 		libsdl2-dev \
 		clang
 
+run-backend-dev:
+	cargo run -p spyglass
+
 run-client-dev:
-	cargo tauri dev
+	cargo tauri dev --config ./crates/tauri/tauri.dev.conf.json
 
 run-client-headless:
 	cd ./crates/client && HEADLESS_CLIENT=true trunk serve

Makefile.toml

@@ -0,0 +1,165 @@
+[config]
+default_to_workspace = false
+reduce_output = false
+skip_git_env_info = true
+
+[env]
+SPYGLASS_CLIENT_DIR = "${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/crates/client"
+TAURI_DEV_CONFIG = "${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/crates/tauri/tauri.dev.conf.json"
+EXECUTABLE_EXTENSION = { source = "${CARGO_MAKE_RUST_TARGET_OS}", mapping = { "windows" = ".exe" } }
+TARGET_FOLDER = { default_value = "debug", mapping = { "release" = "release" }, source = "${PROFILE}" }
+
+CHANNEL = "${CARGO_MAKE_RUST_CHANNEL}"
+CARGO_MAKE_CRATE_INSTALLATION_LOCKED = true
+CARGO_MAKE_EXTEND_WORKSPACE_MAKEFILE = true
+IS_RELEASE = { default_value = "false", mapping = { "release" = "true" }, source = "${PROFILE}" }
+PLUGINS = "()"
+PLUGINS_DEV_FOLDER = "~/Library/Application Support/com.athlabs.spyglass-dev"
+PROFILE = "${CARGO_MAKE_CARGO_PROFILE}"
+WORKSPACE_TARGET_DIR = "${CARGO_MAKE_CRATE_TARGET_DIRECTORY}"
+
+[tasks.run]
+args = ["cargo make run-backend", "cargo make run-client"]
+command = "mprocs"
+install_crate = { crate_name = "mprocs" }
+
+[tasks.run-backend]
+args = ["run", "-p", "spyglass", "--profile", "${CARGO_MAKE_CARGO_PROFILE}"]
+command = "cargo"
+
+[tasks.run-client]
+args = ["tauri", "dev", "--config", "${TAURI_DEV_CONFIG}"]
+command = "cargo"
+
+[tasks.setup]
+dependencies = [
+    "setup-linux",
+    "install-client-npm",
+    "setup-tauri",
+    "download-whisper",
+    "prepare-env-file",
+    "build-backend",
+    "copy-backend-binaries",
+    "copy-pdftotext-binaries",
+]
+
+[tasks.build]
+command = "cargo"
+args = ["build", "-p", "spyglass-client", "-p", "spyglass-app", "--profile", "${CARGO_MAKE_CARGO_PROFILE}"]
+dependencies = [
+    "build-backend",
+    "copy-backend-binaries",
+    "copy-pdftotext-binaries"
+]
+
+###################
+# Private Helpers #
+###################
+
+[tasks.build-backend]
+args = ["build", "-p", "spyglass", "--profile", "${CARGO_MAKE_CARGO_PROFILE}"]
+command = "cargo"
+
+[tasks.copy-backend-binaries]
+dependencies = ["create-binary-directory", "set-platform-specific-variables"]
+private = true
+script = '''
+cp "${SPYGLASS_BACKEND_BIN}" "${SPYGLASS_BACKEND_TAURI_BIN}"
+cp "${SPYGLASS_BACKEND_DEBUG_BIN}" "${SPYGLASS_BACKEND_DEBUG_TAURI_BIN}"
+'''
+script_runner = "@shell"
+
+[tasks.set-platform-specific-variables]
+env = { SPYGLASS_BACKEND_BIN = "${CARGO_MAKE_CRATE_TARGET_DIRECTORY}/${TARGET_FOLDER}}/spyglass", SPYGLASS_BACKEND_DEBUG_BIN = "${CARGO_MAKE_CRATE_TARGET_DIRECTORY}/${TARGET_FOLDER}/spyglass-debug", SPYGLASS_TAURI_DIR = "${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/crates/tauri/binaries", SPYGLASS_BACKEND_TAURI_BIN = "${SPYGLASS_TAURI_DIR}/spyglass-server-${CARGO_MAKE_RUST_TARGET_TRIPLE}", SPYGLASS_BACKEND_DEBUG_TAURI_BIN = "${SPYGLASS_TAURI_DIR}/spyglass-debug-${CARGO_MAKE_RUST_TARGET_TRIPLE}" }
+private = true
+
+[tasks.set-platform-specific-variables.windows]
+# Only needed for these kinds of path-building, composite environment variables.
+env = { SPYGLASS_BACKEND_BIN = '''${CARGO_MAKE_CRATE_TARGET_DIRECTORY}\${TARGET_FOLDER}\spyglass.exe''', SPYGLASS_BACKEND_DEBUG_BIN = '''${CARGO_MAKE_CRATE_TARGET_DIRECTORY}\${TARGET_FOLDER}\spyglass-debug.exe''', SPYGLASS_TAURI_DIR = '''${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}\crates\tauri\binaries''', SPYGLASS_BACKEND_TAURI_BIN = '''${SPYGLASS_TAURI_DIR}\spyglass-server-${CARGO_MAKE_RUST_TARGET_TRIPLE}.exe''', SPYGLASS_BACKEND_DEBUG_TAURI_BIN = '''${SPYGLASS_TAURI_DIR}\spyglass-debug-${CARGO_MAKE_RUST_TARGET_TRIPLE}.exe''' }
+private = true
+
+[tasks.copy-pdftotext-binaries]
+private = true
+script_runner = "@shell"
+[tasks.copy-pdftotext-binaries.linux]
+script = '''cp utils/linux/pdftotext crates/tauri/binaries/pdftotext-${CARGO_MAKE_RUST_TARGET_TRIPLE}'''
+[tasks.copy-pdftotext-binaries.mac]
+script = '''cp utils/mac/pdftotext crates/tauri/binaries/pdftotext-${CARGO_MAKE_RUST_TARGET_TRIPLE}'''
+[tasks.copy-pdftotext-binaries.windows]
+script = '''cp utils/win/pdftotext.exe crates/tauri/binaries/pdftotext-${CARGO_MAKE_RUST_TARGET_TRIPLE}.exe'''
+
+[tasks.create-binary-directory]
+private = true
+script = '''mkdir -p crates/tauri/binaries'''
+script_runner = "@shell"
+[tasks.create-binary-directory.windows]
+script = '''IF NOT EXIST .\crates\tauri\binaries mkdir .\crates\tauri\binaries'''
+
+[tasks.download-whisper]
+private = true
+script = '''
+mkdir -p assets/models
+curl -L --output whisper.base.en.bin https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin
+mv whisper.base.en.bin assets/models
+'''
+script_runner = "@shell"
+
+[tasks.install-client-npm]
+cwd = "${SPYGLASS_CLIENT_DIR}"
+private = true
+script = "npm -s install"
+script_runner = "@shell"
+
+[tasks.install-tauri-cli]
+install_crate = "tauri-cli"
+private = true
+
+[tasks.install-trunk]
+install_crate = "trunk"
+private = true
+
+[tasks.install-wasm32-unknown]
+args = ["target", "add", "wasm32-unknown-unknown"]
+command = "rustup"
+private = true
+
+[tasks.install-wasm32-wasi]
+args = ["target", "add", "wasm32-wasi"]
+command = "rustup"
+private = true
+
+[tasks.setup-linux]
+condition = { platforms = ["linux"] }
+install_script = '''
+sudo apt install libwebkit2gtk-4.0-dev \
+    build-essential \
+    curl \
+    wget \
+    libssl-dev \
+    libgtk-3-dev \
+    libayatana-appindicator3-dev \
+    librsvg2-dev
+'''
+private = true
+
+[tasks.setup-tauri]
+dependencies = [
+    "install-tauri-cli",
+    "install-trunk",
+    "install-wasm32-unknown",
+    "install-wasm32-wasi",
+]
+private = true
+script = '''mkdir -p ./crates/tauri/dist'''
+script_runner = "@shell"
+
+[tasks.setup-tauri.windows]
+script = '''IF NOT EXIST .\crates\tauri\dist mkdir .\crates\tauri\dist'''
+
+[tasks.prepare-env-file]
+private = true
+script = '''test -f .env || cp .env.template .env'''
+script_runner = "@shell"
+
+[tasks.prepare-env-file.windows]
+script = '''IF NOT EXIST .env COPY .env.template .env'''

VERSION.json

@@ -1,23 +1,23 @@
 {
-  "version": "23.4.1",
-  "notes": "See full release notes here: https://github.com/spyglass-search/spyglass/releases/tag/v2023.4.1",
-  "pub_date": "2023-04-05T16:23:51Z",
+  "version": "23.5.1",
+  "notes": "See full release notes here: https://github.com/spyglass-search/spyglass/releases/tag/v2023.5.1",
+  "pub_date": "2023-05-18T20:55:01Z",
   "platforms": {
     "darwin-x86_64": {
-      "signature": "dW50cnVzdGVkIGNvbW1lbnQ6IHNpZ25hdHVyZSBmcm9tIHRhdXJpIHNlY3JldCBrZXkKUlVSRk9WYXdDUXphYm5HSHRnVEdMNEJsYzZuVW5HM1RHL2VWZUtzTDNSbnptNkpoZXFDdGx6SElQM3hzbWhQK3ROdURFaFE2TXp3VG1EY0g2TjhqVGpsTXU3T09FRlBwSUFBPQp0cnVzdGVkIGNvbW1lbnQ6IHRpbWVzdGFtcDoxNjgwNjU2NTU0CWZpbGU6U3B5Z2xhc3MuYXBwLnRhci5negoyOXhpdWNNQnFDSnZGTzRoUjBPbG1jMTlDUGhOQlVESGNzRjB3Y3Y4WFBjajBPZWUvMkRHdlphbUphTUhBblp4SnVrMEhYUFBjYW5jSXcyTGRIeDNEZz09Cg==",
-      "url": "https://github.com/spyglass-search/spyglass/releases/download/v2023.4.1/Spyglass_universal.app.tar.gz"
+      "signature": "dW50cnVzdGVkIGNvbW1lbnQ6IHNpZ25hdHVyZSBmcm9tIHRhdXJpIHNlY3JldCBrZXkKUlVSRk9WYXdDUXphYnEyVmdPUCsrRGlWdWJWVHJJTkdaeXhnZDBlOXp6eVlVQkJGSUJBSTU3UUxkUE9VRVViYm5uU3IzUGN4Qll1a21ZQWs0dytvV1c1RWxIV1AzTVI3ekFRPQp0cnVzdGVkIGNvbW1lbnQ6IHRpbWVzdGFtcDoxNjg0NDQwMjQzCWZpbGU6U3B5Z2xhc3MuYXBwLnRhci5negp5U3BlY2VzdDlsck5MeExudUZ2c2RqekpOVWhEUmtuMk1BVTVjS1ZYZjc5YzV4aGNZazdiRjBObUxBaDNGaVErYS85VVBhbXF0MVgwUDQxRENKMnZCdz09Cg==",
+      "url": "https://github.com/spyglass-search/spyglass/releases/download/v2023.5.1/Spyglass_universal.app.tar.gz"
     },
     "darwin-aarch64": {
-      "signature": "dW50cnVzdGVkIGNvbW1lbnQ6IHNpZ25hdHVyZSBmcm9tIHRhdXJpIHNlY3JldCBrZXkKUlVSRk9WYXdDUXphYm5HSHRnVEdMNEJsYzZuVW5HM1RHL2VWZUtzTDNSbnptNkpoZXFDdGx6SElQM3hzbWhQK3ROdURFaFE2TXp3VG1EY0g2TjhqVGpsTXU3T09FRlBwSUFBPQp0cnVzdGVkIGNvbW1lbnQ6IHRpbWVzdGFtcDoxNjgwNjU2NTU0CWZpbGU6U3B5Z2xhc3MuYXBwLnRhci5negoyOXhpdWNNQnFDSnZGTzRoUjBPbG1jMTlDUGhOQlVESGNzRjB3Y3Y4WFBjajBPZWUvMkRHdlphbUphTUhBblp4SnVrMEhYUFBjYW5jSXcyTGRIeDNEZz09Cg==",
-      "url": "https://github.com/spyglass-search/spyglass/releases/download/v2023.4.1/Spyglass_universal.app.tar.gz"
+      "signature": "dW50cnVzdGVkIGNvbW1lbnQ6IHNpZ25hdHVyZSBmcm9tIHRhdXJpIHNlY3JldCBrZXkKUlVSRk9WYXdDUXphYnEyVmdPUCsrRGlWdWJWVHJJTkdaeXhnZDBlOXp6eVlVQkJGSUJBSTU3UUxkUE9VRVViYm5uU3IzUGN4Qll1a21ZQWs0dytvV1c1RWxIV1AzTVI3ekFRPQp0cnVzdGVkIGNvbW1lbnQ6IHRpbWVzdGFtcDoxNjg0NDQwMjQzCWZpbGU6U3B5Z2xhc3MuYXBwLnRhci5negp5U3BlY2VzdDlsck5MeExudUZ2c2RqekpOVWhEUmtuMk1BVTVjS1ZYZjc5YzV4aGNZazdiRjBObUxBaDNGaVErYS85VVBhbXF0MVgwUDQxRENKMnZCdz09Cg==",
+      "url": "https://github.com/spyglass-search/spyglass/releases/download/v2023.5.1/Spyglass_universal.app.tar.gz"
     },
     "linux-x86_64": {
-      "signature": "dW50cnVzdGVkIGNvbW1lbnQ6IHNpZ25hdHVyZSBmcm9tIHRhdXJpIHNlY3JldCBrZXkKUlVSRk9WYXdDUXphYnM2VDRla2k2VnBVY29zUlZ1b0k4ZkhWVGU1Q2tmMEpIQ0tFQ3RxUGEyRkFBZ3RtaUxLUGpqTThrcENaYnhEZE5OenorZHA3N2srb2QyZ0ZhTHhhZFFnPQp0cnVzdGVkIGNvbW1lbnQ6IHRpbWVzdGFtcDoxNjgwNzI1NjA3CWZpbGU6c3B5Z2xhc3NfMjMuNC4xX2FtZDY0LkFwcEltYWdlLnRhci5negp1MDZ2K3o3T2RNeUhpRHcyVXF6dnNxemo4dWZQMDNpT2R6TnhGOGdUSk5LNytWY0RSSzRNSThyVkV3VmphWGM4UlNlMm1mRnJGY1hhOGs5SW55MXlDUT09Cg==",
-      "url": "https://github.com/spyglass-search/spyglass/releases/download/v2023.4.1/spyglass_23.4.1_amd64.AppImage.tar.gz"
+      "signature": "dW50cnVzdGVkIGNvbW1lbnQ6IHNpZ25hdHVyZSBmcm9tIHRhdXJpIHNlY3JldCBrZXkKUlVSRk9WYXdDUXphYnBxZ1RZd1BSZEJWcXdaRm9GNSsxdlJWSEIwTnFEVHB0d1duZnhZdUU4WFlmaVBneDJjNVpWMjd5WUYvK2xlTGtPRklscGFpeVBCMDRzcFRKUmdJS0FVPQp0cnVzdGVkIGNvbW1lbnQ6IHRpbWVzdGFtcDoxNjg0NDM3MTQ0CWZpbGU6c3B5Z2xhc3NfMjMuNS4xX2FtZDY0LkFwcEltYWdlLnRhci5negpQUS9JaDN4Sk9NZFdCa3Q0a3drc3cwRFlFaURSREhBb1dXY2lrUTlDOFRoQVkzRTJpRGI0MjByZStDT2dQRmNiRnF5ZXczZ2dtZFV5ODlmUURzQ2tBdz09Cg==",
+      "url": "https://github.com/spyglass-search/spyglass/releases/download/v2023.5.1/spyglass_23.5.1_amd64.AppImage.tar.gz"
     },
     "windows-x86_64": {
-      "signature": "dW50cnVzdGVkIGNvbW1lbnQ6IHNpZ25hdHVyZSBmcm9tIHRhdXJpIHNlY3JldCBrZXkKUlVSRk9WYXdDUXphYmpmeDNOZFFDTE0yVURPMVhGMUVkNVNVSTE1b2FtbFlEYVpUT25Gb28xRkVKRjNhdUVGdWZkSFlPMEd0dmRIT2pubFVyd2N3T0puMnlITnUyb25NcVFRPQp0cnVzdGVkIGNvbW1lbnQ6IHRpbWVzdGFtcDoxNjgwNjU1NDk1CWZpbGU6U3B5Z2xhc3NfMjMuNC4xX3g2NF9lbi1VUy5tc2kuemlwCkpVNUdqcnp5WmdRYVRQWlFXdXdxcXNwaFMxcVpHRVNqck9QNENkNzNpeXZsWTNnVTNSRmxqWVZNNlV3cSttN0ZDWHVHTUkzRWtsRGdjVm80dllsZkRnPT0K",
-      "url": "https://github.com/spyglass-search/spyglass/releases/download/v2023.4.1/Spyglass_23.4.1_x64_en-US.msi.zip"
+      "signature": "dW50cnVzdGVkIGNvbW1lbnQ6IHNpZ25hdHVyZSBmcm9tIHRhdXJpIHNlY3JldCBrZXkKUlVSRk9WYXdDUXphYnB6cyt3ejFEWE1jU1hNMjRXVWI3RXZvQW04VGhoa29ETDl6ZlE5NUUyNnltWVpTOUJpaWdpbnVHNXB2b2podVZmTnNaaEZpWnVzdXRGYlNGdWRzUUFBPQp0cnVzdGVkIGNvbW1lbnQ6IHRpbWVzdGFtcDoxNjg0NDM3NzEwCWZpbGU6U3B5Z2xhc3NfMjMuNS4xX3g2NF9lbi1VUy5tc2kuemlwCkpwaDBCT3ZOM09GTnBZam9QVStVRGNyKzNocnozMTlUTGhkejU4VnpNUk5UeW5qWTZkWHZUU2VUeDlXOXVRSkV4WlpNUkNWWlZZbGFaTlhpUnJrV0F3PT0K",
+      "url": "https://github.com/spyglass-search/spyglass/releases/download/v2023.5.1/Spyglass_23.5.1_x64_en-US.msi.zip"
     }
   }
 }

apps/web/Cargo.toml

@@ -13,17 +13,19 @@ gloo = "0.8.0"
 js-sys = "0.3"
 log = "0.4"
 markdown = "1.0.0-alpha.7"
-reqwest = { version = "0.11", features = ["json", "stream"] }
+reqwest = { version = "0.11", features = ["json", "stream", "multipart"] }
 serde = "1.0"
 serde_json = "1.0"
 strum = "0.24"
 strum_macros = "0.24"
 thiserror = "1.0"
 ui-components = { path = "../../crates/ui-components" }
+url = "2.3"
 wasm-bindgen = "0.2.83"
 wasm-bindgen-futures = "0.4.33"
 serde-wasm-bindgen = "0.5"
 uuid = { version = "1.0.0", features = ["v4"], default-features = false }
 web-sys = { version = "0.3.60", features = ["History", "Navigator", "VisibilityState"] }
 yew = { version = "0.20.0", features = ["csr"] }
+yew-hooks = "0.2"
 yew-router = "0.17"

apps/web/index.html

@@ -4,11 +4,23 @@
     <title>Spyglass Search</title>
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1">
-    <link data-trunk rel="css" href="/public/main.css"/>
-    <link data-trunk rel="copy-dir" href="/public/icons"/>
+    <link rel="short icon" type="image/x-icon"  href="/favicon.png" />
+    <link rel="alternate icon" type="image/png" href="/favicon.png" />
+    <link rel="icon" type="image/svg+xml" href="/favicon.svg" />
+
+    <link data-trunk rel="css" href="/public/main.css" />
+    <link data-trunk rel="copy-dir" href="/public/icons" />
+    <link data-trunk rel="copy-file" href="/public/favicon.png" />
+    <link data-trunk rel="copy-file" href="/public/favicon.svg" />
+
     <script src="https://cdn.auth0.com/js/auth0-spa-js/2.0/auth0-spa-js.production.js"></script>
     <script src="https://apis.google.com/js/api.js"></script>
     <script src="https://accounts.google.com/gsi/client"></script>
+    <script defer data-domain="search.spyglass.fyi" src="https://plausible.io/js/script.js"></script>
+    <script
+      src="https://js.sentry-cdn.com/db7492dc52224b7cb6f451b2b81f0ef3.min.js"
+      crossorigin="anonymous"
+    ></script>
   </head>
   <body class="text-white bg-neutral-800"></body>
 </html>