From 2205c1af865d37a022a9001f0bcd0c57a897dbf5 Mon Sep 17 00:00:00 2001 From: Anna Lushnikova Date: Fri, 20 Sep 2024 09:14:22 -0400 Subject: [PATCH] [databases]: add support for Opensearch advanced configuration --- databases.go | 79 +++++++++++++++++++++++++++ databases_test.go | 134 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 213 insertions(+) diff --git a/databases.go b/databases.go index c0c91e2..83bdb9d 100644 --- a/databases.go +++ b/databases.go @@ -153,10 +153,12 @@ type DatabasesService interface { GetRedisConfig(context.Context, string) (*RedisConfig, *Response, error) GetMySQLConfig(context.Context, string) (*MySQLConfig, *Response, error) GetMongoDBConfig(context.Context, string) (*MongoDBConfig, *Response, error) + GetOpensearchConfig(context.Context, string) (*OpensearchConfig, *Response, error) UpdatePostgreSQLConfig(context.Context, string, *PostgreSQLConfig) (*Response, error) UpdateRedisConfig(context.Context, string, *RedisConfig) (*Response, error) UpdateMySQLConfig(context.Context, string, *MySQLConfig) (*Response, error) UpdateMongoDBConfig(context.Context, string, *MongoDBConfig) (*Response, error) + UpdateOpensearchConfig(context.Context, string, *OpensearchConfig) (*Response, error) ListOptions(todo context.Context) (*DatabaseOptions, *Response, error) UpgradeMajorVersion(context.Context, string, *UpgradeVersionRequest) (*Response, error) ListTopics(context.Context, string, *ListOptions) ([]DatabaseTopic, *Response, error) @@ -703,6 +705,10 @@ type databaseMongoDBConfigRoot struct { Config *MongoDBConfig `json:"config"` } +type databaseOpensearchConfigRoot struct { + Config *OpensearchConfig `json:"config"` +} + type databaseBackupsRoot struct { Backups []DatabaseBackup `json:"backups"` } @@ -1546,6 +1552,38 @@ func (svc *DatabasesServiceOp) UpdateMongoDBConfig(ctx context.Context, database return resp, nil } +// GetOpensearchConfig retrieves the config for a Opensearch database cluster. +func (svc *DatabasesServiceOp) GetOpensearchConfig(ctx context.Context, databaseID string) (*OpensearchConfig, *Response, error) { + path := fmt.Sprintf(databaseConfigPath, databaseID) + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(databaseOpensearchConfigRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Config, resp, nil +} + +// UpdateOpensearchConfig updates the config for a Opensearch database cluster. +func (svc *DatabasesServiceOp) UpdateOpensearchConfig(ctx context.Context, databaseID string, config *OpensearchConfig) (*Response, error) { + path := fmt.Sprintf(databaseConfigPath, databaseID) + root := &databaseOpensearchConfigRoot{ + Config: config, + } + req, err := svc.client.NewRequest(ctx, http.MethodPatch, path, root) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + return resp, nil +} + // ListOptions gets the database options available. func (svc *DatabasesServiceOp) ListOptions(ctx context.Context) (*DatabaseOptions, *Response, error) { root := new(databaseOptionsRoot) @@ -1817,3 +1855,44 @@ func (svc *DatabasesServiceOp) DeleteLogsink(ctx context.Context, databaseID, lo } return resp, nil } + +// OpensearchConfig holds advanced configurations for Opensearch database clusters. +type OpensearchConfig struct { + HttpMaxContentLengthBytes *int `json:"http_max_content_length_bytes,omitempty"` + HttpMaxHeaderSizeBytes *int `json:"http_max_header_size_bytes,omitempty"` + HttpMaxInitialLineLengthBytes *int `json:"http_max_initial_line_length_bytes,omitempty"` + IndicesQueryBoolMaxClauseCount *int `json:"indices_query_bool_max_clause_count,omitempty"` + IndicesFielddataCacheSizePercentage *int `json:"indices_fielddata_cache_size_percentage,omitempty"` + IndicesMemoryIndexBufferSizePercentage *int `json:"indices_memory_index_buffer_size_percentage,omitempty"` + IndicesMemoryMinIndexBufferSizeMb *int `json:"indices_memory_min_index_buffer_size_mb,omitempty"` + IndicesMemoryMaxIndexBufferSizeMb *int `json:"indices_memory_max_index_buffer_size_mb,omitempty"` + IndicesQueriesCacheSizePercentage *int `json:"indices_queries_cache_size_percentage,omitempty"` + IndicesRecoveryMaxMbPerSec *int `json:"indices_recovery_max_mb_per_sec,omitempty"` + IndicesRecoveryMaxConcurrentFileChunks *int `json:"indices_recovery_max_concurrent_file_chunks,omitempty"` + ThreadPoolSearchSize *int `json:"thread_pool_search_size,omitempty"` + ThreadPoolSearchThrottledSize *int `json:"thread_pool_search_throttled_size,omitempty"` + ThreadPoolGetSize *int `json:"thread_pool_get_size,omitempty"` + ThreadPoolAnalyzeSize *int `json:"thread_pool_analyze_size,omitempty"` + ThreadPoolWriteSize *int `json:"thread_pool_write_size,omitempty"` + ThreadPoolForceMergeSize *int `json:"thread_pool_force_merge_size,omitempty"` + ThreadPoolSearchQueueSize *int `json:"thread_pool_search_queue_size,omitempty"` + ThreadPoolSearchThrottledQueueSize *int `json:"thread_pool_search_throttled_queue_size,omitempty"` + ThreadPoolGetQueueSize *int `json:"thread_pool_get_queue_size,omitempty"` + ThreadPoolAnalyzeQueueSize *int `json:"thread_pool_analyze_queue_size,omitempty"` + ThreadPoolWriteQueueSize *int `json:"thread_pool_write_queue_size,omitempty"` + IsmEnabled *bool `json:"ism_enabled,omitempty"` + IsmHistoryEnabled *bool `json:"ism_history_enabled,omitempty"` + IsmHistoryMaxAgeHours *int `json:"ism_history_max_age_hours,omitempty"` + IsmHistoryMaxDocs *uint64 `json:"ism_history_max_docs,omitempty"` + IsmHistoryRolloverCheckPeriodHours *int `json:"ism_history_rollover_check_period_hours,omitempty"` + IsmHistoryRolloverRetentionPeriodDays *int `json:"ism_history_rollover_retention_period_days,omitempty"` + SearchMaxBuckets *int `json:"search_max_buckets,omitempty"` + ActionAutoCreateIndexEnabled *bool `json:"action_auto_create_index_enabled,omitempty"` + EnableSecurityAudit *bool `json:"enable_security_audit,omitempty"` + ActionDestructiveRequiresName *bool `json:"action_destructive_requires_name,omitempty"` + ClusterMaxShardsPerNode *int `json:"cluster_max_shards_per_node,omitempty"` + OverrideMainResponseVersion *bool `json:"override_main_response_version,omitempty"` + ScriptMaxCompilationsRate *string `json:"script_max_compilations_rate,omitempty"` + ClusterRoutingAllocationNodeConcurrentRecoveries *int `json:"cluster_routing_allocation_node_concurrent_recoveries,omitempty"` + ReindexRemoteWhitelist []string `json:"reindex_remote_whitelist,omitempty"` +} diff --git a/databases_test.go b/databases_test.go index 2721b9d..1365eb8 100644 --- a/databases_test.go +++ b/databases_test.go @@ -3064,6 +3064,140 @@ func TestDatabases_UpdateConfigMongoDB(t *testing.T) { require.NoError(t, err) } +func TestDatabases_GetConfigOpensearch(t *testing.T) { + setup() + defer teardown() + + var ( + dbSvc = client.Databases + dbID = "da4e0206-d019-41d7-b51f-deadbeefbb8f" + path = fmt.Sprintf("/v2/databases/%s/config", dbID) + + opensearchConfigJSON = `{ + "config": { + "ism_enabled": true, + "ism_history_enabled": true, + "ism_history_max_age_hours": 24, + "ism_history_max_docs": 2500000, + "ism_history_rollover_check_period_hours": 8, + "ism_history_rollover_retention_period_days": 30, + "http_max_content_length_bytes": 100000000, + "http_max_header_size_bytes": 8192, + "http_max_initial_line_length_bytes": 4096, + "indices_query_bool_max_clause_count": 1024, + "search_max_buckets": 10000, + "indices_fielddata_cache_size_percentage": 0, + "indices_memory_index_buffer_size_percentage": 10, + "indices_memory_min_index_buffer_size_mb": 48, + "indices_memory_max_index_buffer_size_mb": 0, + "indices_queries_cache_size_percentage": 10, + "indices_recovery_max_mb_per_sec": 40, + "indices_recovery_max_concurrent_file_chunks": 2, + "action_auto_create_index_enabled": true, + "action_destructive_requires_name": false, + "plugins_alerting_filter_by_backend_roles_enabled": false, + "enable_security_audit": false, + "thread_pool_search_size": 0, + "thread_pool_search_throttled_size": 0, + "thread_pool_search_throttled_queue_size": 0, + "thread_pool_search_queue_size": 0, + "thread_pool_get_size": 0, + "thread_pool_get_queue_size": 0, + "thread_pool_analyze_size": 0, + "thread_pool_analyze_queue_size": 0, + "thread_pool_write_size": 0, + "thread_pool_write_queue_size": 0, + "thread_pool_force_merge_size": 0, + "override_main_response_version": false, + "script_max_compilations_rate": "use-context", + "cluster_max_shards_per_node": 0, + "cluster_routing_allocation_node_concurrent_recoveries": 2 + } +}` + + kafkaConfig = OpensearchConfig{ + HttpMaxContentLengthBytes: PtrTo(100000000), + HttpMaxHeaderSizeBytes: PtrTo(8192), + HttpMaxInitialLineLengthBytes: PtrTo(4096), + IndicesQueryBoolMaxClauseCount: PtrTo(1024), + IndicesFielddataCacheSizePercentage: PtrTo(0), + IndicesMemoryIndexBufferSizePercentage: PtrTo(10), + IndicesMemoryMinIndexBufferSizeMb: PtrTo(48), + IndicesMemoryMaxIndexBufferSizeMb: PtrTo(0), + IndicesQueriesCacheSizePercentage: PtrTo(10), + IndicesRecoveryMaxMbPerSec: PtrTo(40), + IndicesRecoveryMaxConcurrentFileChunks: PtrTo(2), + ThreadPoolSearchSize: PtrTo(0), + ThreadPoolSearchThrottledSize: PtrTo(0), + ThreadPoolGetSize: PtrTo(0), + ThreadPoolAnalyzeSize: PtrTo(0), + ThreadPoolWriteSize: PtrTo(0), + ThreadPoolForceMergeSize: PtrTo(0), + ThreadPoolSearchQueueSize: PtrTo(0), + ThreadPoolSearchThrottledQueueSize: PtrTo(0), + ThreadPoolGetQueueSize: PtrTo(0), + ThreadPoolAnalyzeQueueSize: PtrTo(0), + ThreadPoolWriteQueueSize: PtrTo(0), + IsmEnabled: PtrTo(true), + IsmHistoryEnabled: PtrTo(true), + IsmHistoryMaxAgeHours: PtrTo(24), + IsmHistoryMaxDocs: PtrTo(uint64(2500000)), + IsmHistoryRolloverCheckPeriodHours: PtrTo(8), + IsmHistoryRolloverRetentionPeriodDays: PtrTo(30), + SearchMaxBuckets: PtrTo(10000), + ActionAutoCreateIndexEnabled: PtrTo(true), + EnableSecurityAudit: PtrTo(false), + ActionDestructiveRequiresName: PtrTo(false), + ClusterMaxShardsPerNode: PtrTo(0), + OverrideMainResponseVersion: PtrTo(false), + ScriptMaxCompilationsRate: PtrTo("use-context"), + ClusterRoutingAllocationNodeConcurrentRecoveries: PtrTo(2), + ReindexRemoteWhitelist: nil, + } + ) + + mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) { + testMethod(t, r, http.MethodGet) + fmt.Fprint(w, opensearchConfigJSON) + }) + + got, _, err := dbSvc.GetOpensearchConfig(ctx, dbID) + require.NoError(t, err) + require.Equal(t, &kafkaConfig, got) +} + +func TestDatabases_UpdateConfigOpensearch(t *testing.T) { + setup() + defer teardown() + + var ( + dbID = "deadbeef-dead-4aa5-beef-deadbeef347d" + path = fmt.Sprintf("/v2/databases/%s/config", dbID) + opensearchConfig = &OpensearchConfig{ + HttpMaxContentLengthBytes: PtrTo(1), + HttpMaxHeaderSizeBytes: PtrTo(0), + } + ) + + mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) { + testMethod(t, r, http.MethodPatch) + + var b databaseOpensearchConfigRoot + decoder := json.NewDecoder(r.Body) + err := decoder.Decode(&b) + require.NoError(t, err) + + assert.Equal(t, b.Config, opensearchConfig) + assert.Equal(t, 0, *b.Config.HttpMaxHeaderSizeBytes, "pointers to zero value should be sent") + assert.Nil(t, b.Config.HttpMaxInitialLineLengthBytes, "excluded value should not be sent") + + w.WriteHeader(http.StatusNoContent) + }) + + _, err := client.Databases.UpdateOpensearchConfig(ctx, dbID, opensearchConfig) + require.NoError(t, err) +} + func TestDatabases_UpgradeMajorVersion(t *testing.T) { setup() defer teardown()