From b3be40528303693f51332b726b7e0ff47ae22b90 Mon Sep 17 00:00:00 2001 From: Dmytro Feshchenko Date: Mon, 13 May 2024 18:52:17 +0300 Subject: [PATCH 1/3] Added EXPLAIN ANALYSE query plan into RecommendationService for more relevant recommendations. Added two settings: ignore_explain_queries, ignore_insert_queries. extractIndexesAndSchemaFromRecord was refactored to use schema and indexes from multiple tables from SQL query instead of SlowLog model schema and indexes. getTableNamesFromRawQuery function has been added. Default prompt has been updated. --- config/slower.php | 5 ++- src/Services/RecommendationService.php | 52 ++++++++++++++++++-------- src/SlowerServiceProvider.php | 8 ++++ 3 files changed, 49 insertions(+), 16 deletions(-) diff --git a/config/slower.php b/config/slower.php index e755ff6..0e7c3ba 100644 --- a/config/slower.php +++ b/config/slower.php @@ -13,10 +13,13 @@ ], 'ai_recommendation' => env('SLOWER_AI_RECOMMENDATION', true), 'recommendation_model' => env('SLOWER_AI_RECOMMENDATION_MODEL', 'gpt-4'), + 'recommendation_use_explain' => env('SLOWER_AI_RECOMMENDATION_USE_EXPLAIN', true), + 'ignore_explain_queries' => env('SLOWER_IGNORE_EXPLAIN_QUERIES', true), + 'ignore_insert_queries' => env('SLOWER_IGNORE_INSERT_QUERIES', true), 'open_ai' => [ 'api_key' => env('OPENAI_API_KEY'), 'organization' => env('OPENAI_ORGANIZATION'), 'request_timeout' => env('OPENAI_TIMEOUT'), ], - 'prompt' => env('SLOWER_PROMPT', 'As a distinguished database optimization expert, your expertise is invaluable for refining SQL queries to achieve maximum efficiency. Please examine the SQL statement provided below. Based on your analysis, could you recommend sophisticated indexing techniques or query modifications that could significantly improve performance and scalability?'), + 'prompt' => env('SLOWER_PROMPT', 'As a distinguished database optimization expert, your expertise is invaluable for refining SQL queries to achieve maximum efficiency. Schema json provide list of indexes and column definitions for each table in query. Also analyse the output of EXPLAIN ANALYSE and provide recommendations to optimize query. Please examine the SQL statement provided below including EXPLAIN ANALYSE query plan. Based on your analysis, could you recommend sophisticated indexing techniques or query modifications that could significantly improve performance and scalability?'), ]; diff --git a/src/Services/RecommendationService.php b/src/Services/RecommendationService.php index 3a99a4b..5186cf8 100644 --- a/src/Services/RecommendationService.php +++ b/src/Services/RecommendationService.php @@ -14,20 +14,24 @@ public function __construct(protected Client $client) public function getRecommendation($record): ?string { + $schema = $this->extractIndexesAndSchemaFromRecord($record); + $userMessage = 'The query execution took ' . $record->time . ' milliseconds.' . PHP_EOL . + 'Connection: ' . $record->connection . PHP_EOL . + 'Connection Name: ' . $record->connection_name . PHP_EOL . + 'Schema: ' . json_encode($schema, JSON_PRETTY_PRINT) . PHP_EOL . + 'Sql: ' . $record->raw_sql . PHP_EOL; - [$indexes, $schema] = $this->extractIndexesAndSchemaFromRecord($record); + if (config('slower.recommendation_use_explain', false)) { + $plan = collect(DB::select('explain analyse ' . $record->raw_sql))->implode('QUERY PLAN', PHP_EOL); + + $userMessage .= 'EXPLAIN ANALYSE output: ' . $plan . PHP_EOL; + } $result = $this->client->chat()->create([ 'model' => config('slower.recommendation_model', 'gpt-4'), 'messages' => [ ['role' => 'system', 'content' => config('slower.prompt')], - ['role' => 'user', 'content' => 'The query execution took '.$record->time.' milliseconds.'.PHP_EOL. - 'Connection: '.$record->connection.PHP_EOL. - 'Current Indexes: '.json_encode($indexes, JSON_PRETTY_PRINT).PHP_EOL. - 'Schema: '.json_encode($schema, JSON_PRETTY_PRINT).PHP_EOL. - 'Connection Name: '.$record->connection_name.PHP_EOL. - 'Sql: '.$record->raw_sql, - ], + ['role' => 'user', 'content' => $userMessage], ], ]); @@ -43,18 +47,36 @@ public function getRecommendation($record): ?string private function extractIndexesAndSchemaFromRecord($record): array { - $schemaBuilder = DB::connection($record->getConnectionName())->getSchemaBuilder(); + $schemaBuilder = DB::connection($record->connection_name)->getSchemaBuilder(); - $columns = $schemaBuilder->getColumnListing($record->getTable()); + $schema = []; - $indexes = $schemaBuilder->getIndexes($record->getTable()); + $tables = $this->getTableNamesFromRawQuery($record->raw_sql); + foreach ($tables as $tableName) { + $columns = $schemaBuilder->getColumnListing($tableName); + $schema[$tableName]['indexes'] = $schemaBuilder->getIndexes($tableName); - $schema = []; + foreach ($columns as $column) { + $schema[$tableName]['columns'][] = [$column => $schemaBuilder->getColumnType($tableName, $column)]; + } + } + + return $schema; + } + + private function getTableNamesFromRawQuery(string $sqlQuery): array + { + // Regular expression to match table names + $pattern = '/(?:FROM|JOIN|INTO|UPDATE)\s+(\S+)(?:\s+(?:AS\s+)?\w+)?(?:\s+ON\s+[^ ]+)?/i'; + + preg_match_all($pattern, $sqlQuery, $matches); - foreach ($columns as $column) { - $schema[$column] = $schemaBuilder->getColumnType($record->getTable(), $column); + // Extract table names from the matches + $tableNames = []; + foreach ($matches[1] as $tableName) { + $tableNames[] = str_replace(['`', '"'], '', $tableName); } - return [$indexes, $schema]; + return array_unique($tableNames); } } diff --git a/src/SlowerServiceProvider.php b/src/SlowerServiceProvider.php index 1658b25..531e4c4 100644 --- a/src/SlowerServiceProvider.php +++ b/src/SlowerServiceProvider.php @@ -60,6 +60,14 @@ private function registerDatabaseListener(): void { if (config('slower.enabled')) { DB::whenQueryingForLongerThan(config('slower.threshold', 10000), function (Connection $connection, QueryExecuted $event) { + if(config('slower.ignore_explain_queries', true) && Str::startsWith($event->sql, 'EXPLAIN')) { + return; + } + + if(config('slower.ignore_insert_queries', true) && stripos($event->sql, 'insert') === 0) { + return; + } + $this->createRecord($event, $connection); $this->notify($event, $connection); }); From ccc07bff57a38ab0337e3771021a9d1ac8ef60b1 Mon Sep 17 00:00:00 2001 From: Dmytro Feshchenko Date: Thu, 16 May 2024 15:58:42 +0300 Subject: [PATCH 2/3] Send sql instead of raw_sql to openai --- src/Services/RecommendationService.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Services/RecommendationService.php b/src/Services/RecommendationService.php index 5186cf8..b9e185d 100644 --- a/src/Services/RecommendationService.php +++ b/src/Services/RecommendationService.php @@ -19,7 +19,7 @@ public function getRecommendation($record): ?string 'Connection: ' . $record->connection . PHP_EOL . 'Connection Name: ' . $record->connection_name . PHP_EOL . 'Schema: ' . json_encode($schema, JSON_PRETTY_PRINT) . PHP_EOL . - 'Sql: ' . $record->raw_sql . PHP_EOL; + 'Sql: ' . $record->sql . PHP_EOL; if (config('slower.recommendation_use_explain', false)) { $plan = collect(DB::select('explain analyse ' . $record->raw_sql))->implode('QUERY PLAN', PHP_EOL); From aaea16694fb80ca39517448c665b26a26eedc671 Mon Sep 17 00:00:00 2001 From: Dmytro Feshchenko Date: Thu, 16 May 2024 17:12:45 +0300 Subject: [PATCH 3/3] Logging logic for slow queries has been changed. Listen to all queries and log slow queries by individual request time instead of the aggregate time of all queries per request. --- src/SlowerServiceProvider.php | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/SlowerServiceProvider.php b/src/SlowerServiceProvider.php index 531e4c4..934ea7d 100644 --- a/src/SlowerServiceProvider.php +++ b/src/SlowerServiceProvider.php @@ -59,7 +59,11 @@ public function packageRegistered(): void private function registerDatabaseListener(): void { if (config('slower.enabled')) { - DB::whenQueryingForLongerThan(config('slower.threshold', 10000), function (Connection $connection, QueryExecuted $event) { + DB::listen(function (QueryExecuted $event) { + if($event->time < config('slower.threshold', 10000)) { + return; + } + if(config('slower.ignore_explain_queries', true) && Str::startsWith($event->sql, 'EXPLAIN')) { return; } @@ -68,8 +72,8 @@ private function registerDatabaseListener(): void return; } - $this->createRecord($event, $connection); - $this->notify($event, $connection); + $this->createRecord($event, $event->connection); + $this->notify($event, $event->connection); }); } }