diff --git a/README.md b/README.md index f957e653..2c665031 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Clickhouse::Activerecord -A Ruby database ActiveRecord driver for ClickHouse. Support Rails >= 5.2. +A Ruby database ActiveRecord driver for ClickHouse. Support Rails >= 7.1. Support ClickHouse version from 22.0 LTS. ## Installation @@ -50,41 +50,31 @@ class ActionView < ActiveRecord::Base end ``` -## Usage in Rails 5 +## Usage in Rails Add your `database.yml` connection information with postfix `_clickhouse` for you environment: ```yml -development_clickhouse: +development: adapter: clickhouse database: database ``` -Add to your model: +Your model example: ```ruby class Action < ActiveRecord::Base - establish_connection "#{Rails.env}_clickhouse".to_sym end ``` For materialized view model add: ```ruby class ActionView < ActiveRecord::Base - establish_connection "#{Rails.env}_clickhouse".to_sym self.is_view = true end ``` -Or global connection: - -```yml -development: - adapter: clickhouse - database: database -``` - -## Usage in Rails 6 with second database +## Usage in Rails with second database Add your `database.yml` connection information for you environment: @@ -102,31 +92,31 @@ Connection [Multiple Databases with Active Record](https://guides.rubyonrails.or ```ruby class Action < ActiveRecord::Base - connects_to database: { writing: :clickhouse, reading: :clickhouse } + establish_connection :clickhouse end ``` ### Rake tasks -**Note!** For Rails 6 you can use default rake tasks if you configure `migrations_paths` in your `database.yml`, for example: `rake db:migrate` - Create / drop / purge / reset database: - $ rake clickhouse:create - $ rake clickhouse:drop - $ rake clickhouse:purge - $ rake clickhouse:reset + $ rake db:create + $ rake db:drop + $ rake db:purge + $ rake db:reset -Prepare system tables for rails: +Or with multiple databases: - $ rake clickhouse:prepare_schema_migration_table - $ rake clickhouse:prepare_internal_metadata_table + $ rake db:create:clickhouse + $ rake db:drop:clickhouse + $ rake db:purge:clickhouse + $ rake db:reset:clickhouse Migration: $ rails g clickhouse_migration MIGRATION_NAME COLUMNS - $ rake clickhouse:migrate - $ rake clickhouse:rollback + $ rake db:migrate + $ rake db:rollback ### Dump / Load for multiple using databases @@ -195,20 +185,20 @@ User.joins(:actions).using(:group_id) Integer types are unsigned by default. Specify signed values with `:unsigned => false`. The default integer is `UInt32` -| Type (bit size) | Range | :limit (byte size) | -| :--- | :----: | ---: | -| Int8 | -128 to 127 | 1 | -| Int16 | -32768 to 32767 | 2 | -| Int32 | -2147483648 to 2,147,483,647 | 3,4 | -| Int64 | -9223372036854775808 to 9223372036854775807] | 5,6,7,8 | -| Int128 | ... | 9 - 15 | -| Int256 | ... | 16+ | -| UInt8 | 0 to 255 | 1 | -| UInt16 | 0 to 65,535 | 2 | -| UInt32 | 0 to 4,294,967,295 | 3,4 | -| UInt64 | 0 to 18446744073709551615 | 5,6,7,8 | -| UInt256 | 0 to ... | 8+ | -| Array | ... | ... | +| Type (bit size) | Range | :limit (byte size) | +|:----------------|:--------------------------------------------:|-------------------:| +| Int8 | -128 to 127 | 1 | +| Int16 | -32768 to 32767 | 2 | +| Int32 | -2147483648 to 2,147,483,647 | 3,4 | +| Int64 | -9223372036854775808 to 9223372036854775807] | 5,6,7,8 | +| Int128 | ... | 9 - 15 | +| Int256 | ... | 16+ | +| UInt8 | 0 to 255 | 1 | +| UInt16 | 0 to 65,535 | 2 | +| UInt32 | 0 to 4,294,967,295 | 3,4 | +| UInt64 | 0 to 18446744073709551615 | 5,6,7,8 | +| UInt256 | 0 to ... | 8+ | +| Array | ... | ... | Example: diff --git a/lib/arel/visitors/clickhouse.rb b/lib/arel/visitors/clickhouse.rb index c59b4b47..a63471a9 100644 --- a/lib/arel/visitors/clickhouse.rb +++ b/lib/arel/visitors/clickhouse.rb @@ -15,7 +15,7 @@ def aggregate(name, o, collector) def visit_Arel_Table o, collector collector = super - collector << ' FINAL ' if o.final + collector << ' FINAL' if o.final collector end diff --git a/lib/clickhouse-activerecord/migration.rb b/lib/clickhouse-activerecord/migration.rb index 21c41a82..330bdfaa 100644 --- a/lib/clickhouse-activerecord/migration.rb +++ b/lib/clickhouse-activerecord/migration.rb @@ -25,8 +25,21 @@ def create_table end end - def all_versions - final.where(active: 1).order(:version).pluck(:version) + def versions + table = arel_table.dup + table.final = true + sm = Arel::SelectManager.new(table) + sm.project(arel_table[primary_key]) + sm.order(arel_table[primary_key].asc) + sm.where([arel_table['active'].eq(1)]) + + connection.select_values(sm, "#{self.class} Load") + end + + def delete_version(version) + im = Arel::InsertManager.new(arel_table) + im.insert(arel_table[primary_key] => version.to_s, arel_table['active'] => 0) + connection.insert(im, "#{self.class} Create Rollback Version", primary_key, version) end end @@ -77,29 +90,9 @@ def select_entry(key) class MigrationContext < ::ActiveRecord::MigrationContext #:nodoc: - def up(target_version = nil) - selected_migrations = if block_given? - migrations.select { |m| yield m } - else - migrations - end - - ClickhouseActiverecord::Migrator.new(:up, selected_migrations, schema_migration, internal_metadata, target_version).migrate - end - - def down(target_version = nil) - selected_migrations = if block_given? - migrations.select { |m| yield m } - else - migrations - end - - ClickhouseActiverecord::Migrator.new(:down, selected_migrations, schema_migration, internal_metadata, target_version).migrate - end - def get_all_versions if schema_migration.table_exists? - schema_migration.all_versions.map(&:to_i) + schema_migration.versions.map(&:to_i) else [] end @@ -107,16 +100,4 @@ def get_all_versions end - class Migrator < ::ActiveRecord::Migrator - - def record_version_state_after_migrating(version) - if down? - migrated.delete(version) - @schema_migration.create!(version: version.to_s, active: 0) - else - super - end - end - - end end diff --git a/lib/clickhouse-activerecord/version.rb b/lib/clickhouse-activerecord/version.rb index 901480fe..fd3b523d 100644 --- a/lib/clickhouse-activerecord/version.rb +++ b/lib/clickhouse-activerecord/version.rb @@ -1,3 +1,3 @@ module ClickhouseActiverecord - VERSION = '0.6.1' + VERSION = '1.0.0' end diff --git a/lib/core_extensions/active_record/relation.rb b/lib/core_extensions/active_record/relation.rb index 7a938d1d..35c7a50d 100644 --- a/lib/core_extensions/active_record/relation.rb +++ b/lib/core_extensions/active_record/relation.rb @@ -11,21 +11,65 @@ def reverse_order! self end + # Define settings in the SETTINGS clause of the SELECT query. The setting value is applied only to that query and is reset to the default or previous value after the query is executed. + # For example: + # + # users = User.settings(optimize_read_in_order: 1, cast_keep_nullable: 1).where(name: 'John') + # # SELECT users.* FROM users WHERE users.name = 'John' SETTINGS optimize_read_in_order = 1, cast_keep_nullable = 1 + # + # An ActiveRecord::ActiveRecordError will be raised if database not ClickHouse. # @param [Hash] opts def settings(**opts) + spawn.settings!(**opts) + end + + # @param [Hash] opts + def settings!(**opts) + assert_mutability! check_command('SETTINGS') @values[:settings] = (@values[:settings] || {}).merge opts self end + # When FINAL is specified, ClickHouse fully merges the data before returning the result and thus performs all data transformations that happen during merges for the given table engine. + # For example: + # + # users = User.final.all + # # SELECT users.* FROM users FINAL + # + # An ActiveRecord::ActiveRecordError will be raised if database not ClickHouse. # @param [Boolean] final def final(final = true) + spawn.final!(final) + end + + # @param [Boolean] final + def final!(final = true) + assert_mutability! check_command('FINAL') @table = @table.dup @table.final = final self end + # The USING clause specifies one or more columns to join, which establishes the equality of these columns. For example: + # + # users = User.joins(:joins).using(:event_name, :date) + # # SELECT users.* FROM users INNER JOIN joins USING event_name,date + # + # An ActiveRecord::ActiveRecordError will be raised if database not ClickHouse. + # @param [Array] opts + def using(*opts) + spawn.using!(*opts) + end + + # @param [Array] opts + def using!(*opts) + assert_mutability! + @values[:using] = opts + self + end + private def check_command(cmd) @@ -36,6 +80,7 @@ def build_arel(aliases = nil) arel = super arel.settings(@values[:settings]) if @values[:settings].present? + arel.using(@values[:using]) if @values[:using].present? arel end diff --git a/lib/tasks/clickhouse.rake b/lib/tasks/clickhouse.rake index ee29f99b..181ac42c 100644 --- a/lib/tasks/clickhouse.rake +++ b/lib/tasks/clickhouse.rake @@ -2,17 +2,13 @@ namespace :clickhouse do task prepare_schema_migration_table: :environment do - ClickhouseActiverecord::SchemaMigration.create_table unless ENV['simple'] || ARGV.any? { |a| a.include?('--simple') } + connection = ActiveRecord::Tasks::DatabaseTasks.migration_connection + connection.schema_migration.create_table unless ENV['simple'] || ARGV.any? { |a| a.include?('--simple') } end task prepare_internal_metadata_table: :environment do - ClickhouseActiverecord::InternalMetadata.create_table unless ENV['simple'] || ARGV.any? { |a| a.include?('--simple') } - end - - task load_config: :environment do - ENV['SCHEMA'] = 'db/clickhouse_schema.rb' - ActiveRecord::Migrator.migrations_paths = %w[db/migrate_clickhouse] - ActiveRecord::Base.establish_connection(:clickhouse) + connection = ActiveRecord::Tasks::DatabaseTasks.migration_connection + connection.internal_metadata.create_table unless ENV['simple'] || ARGV.any? { |a| a.include?('--simple') } end namespace :schema do @@ -37,52 +33,55 @@ namespace :clickhouse do namespace :structure do desc 'Load database structure' - task load: [:load_config, 'db:check_protected_environments'] do + task load: ['db:check_protected_environments'] do config = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: 'clickhouse') ClickhouseActiverecord::Tasks.new(config).structure_load(Rails.root.join('db/clickhouse_structure.sql')) end desc 'Dump database structure' - task dump: [:load_config, 'db:check_protected_environments'] do + task dump: ['db:check_protected_environments'] do config = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: 'clickhouse') ClickhouseActiverecord::Tasks.new(config).structure_dump(Rails.root.join('db/clickhouse_structure.sql')) end end desc 'Creates the database from DATABASE_URL or config/database.yml' - task create: [:load_config] do + task create: [] do config = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: 'clickhouse') ActiveRecord::Tasks::DatabaseTasks.create(config) end desc 'Drops the database from DATABASE_URL or config/database.yml' - task drop: [:load_config, 'db:check_protected_environments'] do + task drop: ['db:check_protected_environments'] do config = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: 'clickhouse') ActiveRecord::Tasks::DatabaseTasks.drop(config) end desc 'Empty the database from DATABASE_URL or config/database.yml' - task purge: [:load_config, 'db:check_protected_environments'] do + task purge: ['db:check_protected_environments'] do config = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: 'clickhouse') ActiveRecord::Tasks::DatabaseTasks.purge(config) end # desc 'Resets your database using your migrations for the current environment' - task reset: :load_config do + task :reset do Rake::Task['clickhouse:purge'].execute Rake::Task['clickhouse:migrate'].execute end desc 'Migrate the clickhouse database' - task migrate: %i[load_config prepare_schema_migration_table prepare_internal_metadata_table] do - Rake::Task['db:migrate'].execute + task migrate: %i[prepare_schema_migration_table prepare_internal_metadata_table] do + Rake::Task['db:migrate:clickhouse'].execute if File.exists? "#{Rails.root}/db/clickhouse_schema_simple.rb" Rake::Task['clickhouse:schema:dump'].execute(simple: true) end end desc 'Rollback the clickhouse database' - task rollback: %i[load_config prepare_schema_migration_table prepare_internal_metadata_table] do - Rake::Task['db:rollback'].execute + task rollback: %i[prepare_schema_migration_table prepare_internal_metadata_table] do + Rake::Task['db:rollback:clickhouse'].execute + if File.exists? "#{Rails.root}/db/clickhouse_schema_simple.rb" + Rake::Task['clickhouse:schema:dump'].execute(simple: true) + end end end diff --git a/spec/cases/model_spec.rb b/spec/cases/model_spec.rb index e6e440de..fcf8b323 100644 --- a/spec/cases/model_spec.rb +++ b/spec/cases/model_spec.rb @@ -137,6 +137,7 @@ class Model < ActiveRecord::Base it 'select' do expect(model.count).to eq(2) expect(model.final.count).to eq(1) + expect(model.final.where(date: '2023-07-21').to_sql).to eq('SELECT sample.* FROM sample FINAL WHERE sample.date = \'2023-07-21\'') end end end