Added indexing progress information

svera · Mar 16, 2024 · 4538cb9 · 4538cb9
1 parent 4ea10fb
commit 4538cb9
Show file tree

Hide file tree

Showing 27 changed files with 759 additions and 661 deletions.
diff --git a/internal/index/bleve.go b/internal/index/bleve.go
@@ -18,13 +18,22 @@ import (
 	"github.com/blevesearch/bleve/v2/analysis/token/porter"
 	"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
 	"github.com/blevesearch/bleve/v2/mapping"
+	"github.com/spf13/afero"
 	"github.com/svera/coreander/v3/internal/metadata"
 )
 
 // Version identifies the mapping used for indexing. Any changes in the mapping requires an increase
 // of version, to signal that a new index needs to be created.
 const Version = "v2"
 
+// Metadata fields
+var (
+	internalIndexStartTime   = []byte("index-start-time")
+	internalIndexedDocuments = []byte("indexed-documents")
+	internalLanguages        = []byte("languages")
+	internalVersion          = []byte("version")
+)
+
 var noStopWordsFilters = map[string][]string{
 	es.AnalyzerName: {es.NormalizeName, lowercase.Name, es.LightStemmerName},
 	en.AnalyzerName: {en.PossessiveName, lowercase.Name, porter.Name},
@@ -37,21 +46,32 @@ var noStopWordsFilters = map[string][]string{
 const defaultAnalyzer = "default_analyzer"
 
 type BleveIndexer struct {
+	fs          afero.Fs
 	idx         bleve.Index
 	libraryPath string
 	reader      map[string]metadata.Reader
 }
 
 // NewBleve creates a new BleveIndexer instance using the passed parameters
-func NewBleve(index bleve.Index, libraryPath string, read map[string]metadata.Reader) *BleveIndexer {
+func NewBleve(index bleve.Index, fs afero.Fs, libraryPath string, read map[string]metadata.Reader) *BleveIndexer {
 	return &BleveIndexer{
+		fs,
 		index,
 		strings.TrimSuffix(libraryPath, string(filepath.Separator)),
 		read,
 	}
 }
 
-func Mapping() mapping.IndexMapping {
+func Create(path string) bleve.Index {
+	indexFile, err := bleve.New(path, CreateMapping())
+	if err != nil {
+		log.Fatal(err)
+	}
+	indexFile.SetInternal(internalVersion, []byte(Version))
+	return indexFile
+}
+
+func CreateMapping() mapping.IndexMapping {
 	indexMapping := bleve.NewIndexMapping()
 
 	err := indexMapping.AddCustomAnalyzer(defaultAnalyzer,

diff --git a/internal/index/bleve_read.go b/internal/index/bleve_read.go
@@ -3,16 +3,69 @@ package index
 import (
 	"fmt"
 	"html/template"
+	"io/fs"
+	"math"
 	"net/url"
+	"strconv"
 	"strings"
+	"time"
 
 	"github.com/blevesearch/bleve/v2"
 	"github.com/blevesearch/bleve/v2/search/query"
 	"github.com/gosimple/slug"
+	"github.com/spf13/afero"
 	"github.com/svera/coreander/v3/internal/metadata"
 	"github.com/svera/coreander/v3/internal/result"
 )
 
+func (b *BleveIndexer) IndexingProgress() (Progress, error) {
+	var progress Progress
+
+	indexStartTime, err := b.idx.GetInternal([]byte(internalIndexStartTime))
+	if err != nil {
+		return progress, err
+	}
+	if indexStartTime == nil {
+		return progress, nil
+	}
+	indexedDocuments, err := b.idx.GetInternal([]byte(internalIndexedDocuments))
+	if err != nil {
+		return progress, err
+	}
+	startTime, err := strconv.ParseInt(string(indexStartTime), 10, 64)
+	if err != nil {
+		return progress, err
+	}
+	indexedAmount, err := strconv.ParseFloat(string(indexedDocuments), 64)
+	if err != nil {
+		return progress, err
+	}
+	ellapsedTime := float64(time.Now().UnixNano() - startTime)
+	libraryFiles, err := countFiles(b.libraryPath, b.fs)
+	if err != nil {
+		return progress, err
+	}
+	progress.RemainingTime = time.Duration((ellapsedTime * (libraryFiles - indexedAmount)) / indexedAmount)
+	progress.Percentage = math.Round((100 / libraryFiles) * indexedAmount)
+	return progress, nil
+}
+
+func countFiles(dir string, fileSystem afero.Fs) (float64, error) {
+	var total float64
+
+	afero.Walk(fileSystem, dir, func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+		total++
+		return nil
+	})
+	return total, nil
+}
+
 // Search look for documents which match with the passed keywords. Returns a maximum <resultsPerPage> documents, offset by <page>
 func (b *BleveIndexer) Search(keywords string, page, resultsPerPage int) (result.Paginated[[]Document], error) {
 	for _, prefix := range []string{"Authors:", "Series:", "Title:", "Subjects:", "\""} {

diff --git a/internal/index/bleve_test.go b/internal/index/bleve_test.go
@@ -15,7 +15,7 @@ import (
 func TestIndexAndSearch(t *testing.T) {
 	for _, tcase := range testCases() {
 		t.Run(tcase.name, func(t *testing.T) {
-			indexMem, err := bleve.NewMemOnly(index.Mapping())
+			indexMem, err := bleve.NewMemOnly(index.CreateMapping())
 			if err != nil {
 				t.Errorf("Error initialising index")
 			}
@@ -28,14 +28,14 @@ func TestIndexAndSearch(t *testing.T) {
 				},
 			}
 
-			idx := index.NewBleve(indexMem, "lib", mockMetadataReaders)
-
 			appFS := afero.NewMemMapFs()
+			idx := index.NewBleve(indexMem, appFS, "lib", mockMetadataReaders)
+
 			// create test files and directories
 			appFS.MkdirAll("lib", 0755)
 			afero.WriteFile(appFS, tcase.filename, []byte(""), 0644)
 
-			err = idx.AddLibrary(appFS, 1)
+			err = idx.AddLibrary(1)
 			if err != nil {
 				t.Errorf("Error indexing: %s", err.Error())
 			}

diff --git a/internal/index/bleve_write.go b/internal/index/bleve_write.go
@@ -6,7 +6,9 @@ import (
 	"os"
 	"path/filepath"
 	"slices"
+	"strconv"
 	"strings"
+	"time"
 
 	"github.com/gosimple/slug"
 	"github.com/spf13/afero"
@@ -44,11 +46,14 @@ func (b *BleveIndexer) RemoveFile(file string) error {
 }
 
 // AddLibrary scans <libraryPath> for documents and adds them to the index in batches of <bathSize>
-func (b *BleveIndexer) AddLibrary(fs afero.Fs, batchSize int) error {
+func (b *BleveIndexer) AddLibrary(batchSize int) error {
 	batch := b.idx.NewBatch()
 	batchSlugs := make(map[string]struct{}, batchSize)
 	languages := []string{}
-	e := afero.Walk(fs, b.libraryPath, func(fullPath string, f os.FileInfo, err error) error {
+	var progress int64
+	batch.SetInternal(internalIndexStartTime, []byte(strconv.FormatInt(time.Now().UnixNano(), 10)))
+	batch.SetInternal(internalIndexedDocuments, []byte(strconv.FormatInt(progress, 10)))
+	e := afero.Walk(b.fs, b.libraryPath, func(fullPath string, f os.FileInfo, err error) error {
 		ext := strings.ToLower(filepath.Ext(fullPath))
 		if _, ok := b.reader[ext]; !ok {
 			return nil
@@ -70,14 +75,16 @@ func (b *BleveIndexer) AddLibrary(fs afero.Fs, batchSize int) error {
 		}
 
 		if batch.Size() == batchSize {
+			progress += int64(batchSize)
+			batch.SetInternal(internalIndexedDocuments, []byte(strconv.FormatInt(progress, 10)))
 			b.idx.Batch(batch)
 			batch.Reset()
 			batchSlugs = make(map[string]struct{}, batchSize)
 		}
 		return nil
 	})
-
-	b.idx.SetInternal([]byte("languages"), []byte(strings.Join(languages, ",")))
+	batch.SetInternal(internalIndexStartTime, nil)
+	batch.SetInternal(internalLanguages, []byte(strings.Join(languages, ",")))
 	b.idx.Batch(batch)
 	return e
 }

diff --git a/internal/index/progress.go b/internal/index/progress.go
@@ -0,0 +1,8 @@
+package index
+
+import "time"
+
+type Progress struct {
+	RemainingTime time.Duration
+	Percentage    float64
+}
diff --git a/internal/webserver/embedded/css/bootstrap.min.css b/internal/webserver/embedded/css/bootstrap.min.css
diff --git a/internal/webserver/embedded/js/bootstrap.bundle.min.js b/internal/webserver/embedded/js/bootstrap.bundle.min.js
diff --git a/internal/webserver/embedded/translations/es.yml b/internal/webserver/embedded/translations/es.yml
@@ -116,3 +116,5 @@
 "Error uploading document": "Error al subir el documento"
 "Invalid file type": "Tipo de archivo no válido"
 "Upload": "Subir"
+"Indexing in progress, search results may not be accurate.": "Indexando documentos, los resultados de búsqueda pueden no ser precisos."
+"Remaining time: %s minutes": "Tiempo restante: %s minutos"
diff --git a/internal/webserver/embedded/translations/fr.yml b/internal/webserver/embedded/translations/fr.yml
@@ -116,3 +116,5 @@
 "Error uploading document": "Erreur lors du téléchargement du document"
 "Invalid file type": "Type de fichier invalide"
 "Upload": "Télécharger"
+"Indexing in progress, search results may not be accurate.": "Indexation en cours, les résultats de la recherche peuvent ne pas être précis."
+"Remaining time: %s minutes": "Temps restant: %s minutes"
diff --git a/internal/webserver/embedded/views/auth/edit-password.html b/internal/webserver/embedded/views/auth/edit-password.html
@@ -1,29 +1,27 @@
-<div class="container mt-5 text-start">
-    <h2 class="pb-5">{{t .Lang "Set new password"}}</h2>
-    <form method="post" action="/{{.Lang}}/reset-password">
-        <div class="mb-3">
-            <label for="password" class="form-label">{{t .Lang "New password"}}</label>
-            <input type="password" name="password"
-                class='form-control {{if ne (index .Errors "password") ""}}is-invalid{{end}}' id="password"
-                required="required" minlength="{{.MinPasswordLength}}">
-            {{if ne (index .Errors "password") ""}}
-            <div class="invalid-feedback">
-                {{t .Lang .Errors.password}}
-            </div>
-            {{end}}
+<h2 class="pb-5">{{t .Lang "Set new password"}}</h2>
+<form method="post" action="/{{.Lang}}/reset-password">
+    <div class="mb-3">
+        <label for="password" class="form-label">{{t .Lang "New password"}}</label>
+        <input type="password" name="password"
+            class='form-control {{if ne (index .Errors "password") ""}}is-invalid{{end}}' id="password"
+            required="required" minlength="{{.MinPasswordLength}}">
+        {{if ne (index .Errors "password") ""}}
+        <div class="invalid-feedback">
+            {{t .Lang .Errors.password}}
         </div>
-        <div class="mb-3">
-            <label for="confirm-password" class="form-label">{{t .Lang "Confirm new password"}}</label>
-            <input type="password" name="confirm-password"
-                class='form-control {{if ne (index .Errors "confirmpassword") ""}}is-invalid{{end}}'
-                id="confirm-password" required="required" minlength="{{.MinPasswordLength}}">
-            {{if ne (index .Errors "confirmpassword") ""}}
-            <div class="invalid-feedback">
-                {{t .Lang .Errors.confirmpassword}}
-            </div>
-            {{end}}
+        {{end}}
+    </div>
+    <div class="mb-3">
+        <label for="confirm-password" class="form-label">{{t .Lang "Confirm new password"}}</label>
+        <input type="password" name="confirm-password"
+            class='form-control {{if ne (index .Errors "confirmpassword") ""}}is-invalid{{end}}'
+            id="confirm-password" required="required" minlength="{{.MinPasswordLength}}">
+        {{if ne (index .Errors "confirmpassword") ""}}
+        <div class="invalid-feedback">
+            {{t .Lang .Errors.confirmpassword}}
         </div>
-        <input type="hidden" name="id" value="{{.Uuid}}">
-        <button type="submit" class="btn btn-primary">{{t .Lang "Update"}}</button>
-    </form>
-</div>
+        {{end}}
+    </div>
+    <input type="hidden" name="id" value="{{.Uuid}}">
+    <button type="submit" class="btn btn-primary">{{t .Lang "Update"}}</button>
+</form>
diff --git a/internal/webserver/embedded/views/auth/login.html b/internal/webserver/embedded/views/auth/login.html
@@ -1,34 +1,17 @@
-<div class="container mt-5">
-    {{if .Error}}
-    <div class="row mt-5 text-start">
-        <div class="alert alert-danger" role="alert">
-            {{t .Lang .Error}}
-        </div>
-    </div>
-    {{end}}
+<form method="post" action="/{{.Lang}}/login">
+    <h2 class="h3 mb-3 fw-normal">{{t .Lang "Please sign in"}}</h2>
 
-    {{if .Message}}
-    <div class="row mt-5 text-start">
-        <div class="alert alert-success" role="alert">
-            {{t .Lang .Message}}
-        </div>
+    <div class="form-floating">
+        <input type="email" class="form-control" id="floatingInput" placeholder="[email protected]" name="email">
+        <label for="floatingInput" class="form-label">{{t .Lang "Email"}}</label>
+    </div>
+    <div class="form-floating mt-3">
+        <input type="password" class="form-control" id="floatingPassword" placeholder="Password" name="password">
+        <label for="floatingPassword">{{t .Lang "Password"}}</label>
+        {{if .EmailSendingConfigured}}
+        <div id="passwordHelp" class="form-text text-end pb-5"><a href="/{{.Lang}}/recover">{{t .Lang "I don't remember my password"}}</a></div>
+        {{end}}
     </div>
-    {{end}}
-    <form method="post" action="/{{.Lang}}/login">
-        <h2 class="h3 mb-3 fw-normal">{{t .Lang "Please sign in"}}</h2>
-
-        <div class="form-floating">
-            <input type="email" class="form-control" id="floatingInput" placeholder="[email protected]" name="email">
-            <label for="floatingInput" class="form-label">{{t .Lang "Email"}}</label>
-        </div>
-        <div class="form-floating mt-3">
-            <input type="password" class="form-control" id="floatingPassword" placeholder="Password" name="password">
-            <label for="floatingPassword">{{t .Lang "Password"}}</label>
-            {{if .EmailSendingConfigured}}
-            <div id="passwordHelp" class="form-text text-end pb-5"><a href="/{{.Lang}}/recover">{{t .Lang "I don't remember my password"}}</a></div>
-            {{end}}
-        </div>
 
-        <button class="w-100 btn btn-lg btn-primary mt-3" type="submit">{{t .Lang "Sign in"}}</button>
-    </form>
-</div>
+    <button class="w-100 btn btn-lg btn-primary mt-3" type="submit">{{t .Lang "Sign in"}}</button>
+</form>
diff --git a/internal/webserver/embedded/views/auth/recover.html b/internal/webserver/embedded/views/auth/recover.html
@@ -1,21 +1,20 @@
-<div class="container mt-5 text-start">
-    <section class="pb-5">
-        <h2 class="pb-5">{{t .Lang "Recover password"}}</h2>
-        <form method="post" action="/{{.Lang}}/recover" autocomplete="off">
-            <div class="mb-3">
-                <label for="email" class="form-label">{{t .Lang "Email"}}</label>
-                <div class="input-group">
-                    <input type="email" name="email"
-                        class='form-control {{if ne (index .Errors "email") ""}}is-invalid{{end}}' id="email"
-                        required value="{{.User.Email}}">
-                    <button class="btn btn-primary" type="submit">{{t .Lang "Request"}}</button>
-                    {{if ne (index .Errors "email") ""}}
-                    <div class="invalid-feedback">
-                        {{t .Lang .Errors.email}}
-                    </div>
-                    {{end}}
-                    </div>
-            </div>
-        </form>
-    </section>
-</div>
+<section class="pb-5">
+    <h2 class="pb-5">{{t .Lang "Recover password"}}</h2>
+    <form method="post" action="/{{.Lang}}/recover" autocomplete="off">
+        <div class="mb-3">
+            <label for="email" class="form-label">{{t .Lang "Email"}}</label>
+            <div class="input-group">
+                <input type="email" name="email"
+                    class='form-control {{if ne (index .Errors "email") ""}}is-invalid{{end}}' id="email"
+                    required value="{{.User.Email}}">
+                <button class="btn btn-primary" type="submit">{{t .Lang "Request"}}</button>
+                {{if ne (index .Errors "email") ""}}
+                <div class="invalid-feedback">
+                    {{t .Lang .Errors.email}}
+                </div>
+                {{end}}
+                </div>
+        </div>
+    </form>
+</section>
+
diff --git a/internal/webserver/embedded/views/auth/request.html b/internal/webserver/embedded/views/auth/request.html
@@ -1,7 +1,5 @@
-<div class="container mt-5 text-start">
-    <section class="pb-5">
-        <h2 class="pb-5">{{t .Lang "Recover password"}}</h2>
-        <p>{{t .Lang "We've received your password recovery request. If the address you introduced is registered in our system, you'll receive an email with further instructions in your inbox."}}</p>
-        <p>{{t .Lang "Check your spam folder if you don't receive the recovery email after a while."}}</p>
-    </section>
-</div>
+<section class="pb-5">
+    <h2 class="pb-5">{{t .Lang "Recover password"}}</h2>
+    <p>{{t .Lang "We've received your password recovery request. If the address you introduced is registered in our system, you'll receive an email with further instructions in your inbox."}}</p>
+    <p>{{t .Lang "Check your spam folder if you don't receive the recovery email after a while."}}</p>
+</section>