From 6a08fa1ba4ff364c7fa7e192ddd0224eda25b5af Mon Sep 17 00:00:00 2001 From: Krzysztof Kiewicz Date: Fri, 20 Dec 2024 16:44:06 +0100 Subject: [PATCH] Cleanup 2 --- quesma/persistence/elastic_with_eviction.go | 10 ++-- quesma/persistence/evictor.go | 15 ++--- quesma/persistence/model.go | 65 ++++++++++----------- 3 files changed, 41 insertions(+), 49 deletions(-) diff --git a/quesma/persistence/elastic_with_eviction.go b/quesma/persistence/elastic_with_eviction.go index a615b50de..04d2865ee 100644 --- a/quesma/persistence/elastic_with_eviction.go +++ b/quesma/persistence/elastic_with_eviction.go @@ -20,7 +20,7 @@ import ( type ElasticDatabaseWithEviction struct { ctx context.Context *ElasticJSONDatabase // maybe remove and copy fields here - EvictorInterface + // EvictorInterface sizeInBytesLimit int64 } @@ -28,8 +28,8 @@ func NewElasticDatabaseWithEviction(cfg config.ElasticsearchConfiguration, index return &ElasticDatabaseWithEviction{ ctx: context.Background(), ElasticJSONDatabase: NewElasticJSONDatabase(cfg, indexName), - EvictorInterface: &Evictor{}, - sizeInBytesLimit: sizeInBytesLimit, + // EvictorInterface: &Evictor{}, + sizeInBytesLimit: sizeInBytesLimit, } } @@ -45,6 +45,7 @@ func (db *ElasticDatabaseWithEviction) Put(document *JSONWithSize) error { } bytesNeeded := dbSize + document.SizeInBytesTotal if bytesNeeded > db.SizeInBytesLimit() { + return errors.New("elastic database: is full, cannot put document") /* TODO: restore after eviction readded logger.Info().Msgf("elastic database: is full, need %d bytes more. Evicting documents", bytesNeeded-db.SizeInBytesLimit()) @@ -58,9 +59,6 @@ func (db *ElasticDatabaseWithEviction) Put(document *JSONWithSize) error { */ } - if bytesNeeded > db.SizeInBytesLimit() { - return errors.New("elastic database: is full, cannot put document") - } elasticsearchURL := fmt.Sprintf("%s/_update/%s", db.indexName, document.id) if printDebugElasticDB { diff --git a/quesma/persistence/evictor.go b/quesma/persistence/evictor.go index 88bfc5f90..fa84e9f12 100644 --- a/quesma/persistence/evictor.go +++ b/quesma/persistence/evictor.go @@ -2,21 +2,16 @@ // SPDX-License-Identifier: Elastic-2.0 package persistence -import "fmt" - type EvictorInterface interface { Evict(documents []*JSONWithSize, sizeNeeded int64) (bytesEvicted int64) } -// It's only 1 implementation, which looks well suited for ElasticSearch. -// It can be implemented differently. +// TODO: Find out how this might work. My old idea doesn't work now, +// as don't remove entire indices, but delete single documents. +// (It turned out consistency was too eventual to rely on it) +// old comment: It's only 1 implementation, which looks well suited for ElasticSearch. It can be implemented differently. type Evictor struct{} func (e *Evictor) Evict(documents []*JSONWithSize, sizeNeeded int64) (bytesEvicted int64) { - if sizeNeeded <= 0 { - return // check if it's empty array or nil - } - fmt.Println("kk dbg SelectToEvict() sizeNeeded:", sizeNeeded) - - return + panic("implement me (or remove)") } diff --git a/quesma/persistence/model.go b/quesma/persistence/model.go index 58767f8c5..55b3ca3b3 100644 --- a/quesma/persistence/model.go +++ b/quesma/persistence/model.go @@ -7,39 +7,38 @@ import ( "time" ) -// JSONDatabase is an interface for a database that stores JSON data. -// Treat it as `etcd` equivalent rather than `MongoDB`. -// The main usage is to store our configuration data, like -// - schema -// - user settings -// - statistics, etc -// -// For each case, we should have a separate database. -type JSONDatabase interface { - List() (keys []string, err error) - Get(key string) (string, bool, error) - Put(key string, data string) error -} - -type DatabaseWithEviction interface { // for sure JSON? maybe not only json? check - Put(doc JSONWithSize) error - Get(id string) (types.JSON, error) - Delete(id string) error - DeleteOld(time.Duration) error - DocCount() (int, error) - SizeInBytes() (int64, error) - SizeInBytesLimit() int64 -} - -type JSONWithSizeInterface interface { - SizeInBytes() int64 -} - -type JSONWithSize struct { - types.JSON - id string - SizeInBytesTotal int64 -} +type ( + // JSONDatabase is an interface for a database that stores JSON data. + // Treat it as `etcd` equivalent rather than `MongoDB`. + // The main usage is to store our configuration data, like + // - schema + // - user settings + // - statistics, etc + // + // For each case, we should have a separate database. + JSONDatabase interface { + List() (keys []string, err error) + Get(key string) (string, bool, error) + Put(key string, data string) error + } + DatabaseWithEviction interface { // for sure JSON? maybe not only json? check + Put(doc JSONWithSize) error + Get(id string) (types.JSON, error) + Delete(id string) error + DeleteOld(time.Duration) error + DocCount() (int, error) + SizeInBytes() (int64, error) + SizeInBytesLimit() int64 + } + JSONWithSizeInterface interface { + SizeInBytes() int64 + } + JSONWithSize struct { + types.JSON + id string + SizeInBytesTotal int64 + } +) func NewJSONWithSize(data types.JSON, id string, sizeInBytesTotal int64) *JSONWithSize { return &JSONWithSize{