From c91066a817782c430a7909b9af043dcfb206b647 Mon Sep 17 00:00:00 2001 From: nixx Date: Mon, 22 Jan 2024 18:08:37 +0300 Subject: [PATCH] refactoring `final` method --- README.md | 5 +- .../connection_adapters/clickhouse_adapter.rb | 17 +-- lib/arel/nodes/final.rb | 7 ++ lib/arel/visitors/clickhouse.rb | 12 +- lib/clickhouse-activerecord.rb | 6 + lib/clickhouse-activerecord/migration.rb | 103 ------------------ lib/clickhouse-activerecord/version.rb | 2 +- .../active_record/internal_metadata.rb | 46 ++++++++ lib/core_extensions/active_record/relation.rb | 12 +- .../active_record/schema_migration.rb | 48 ++++++++ lib/core_extensions/arel/nodes/select_core.rb | 19 ++++ lib/core_extensions/arel/select_manager.rb | 5 + lib/core_extensions/arel/table.rb | 2 - lib/tasks/clickhouse.rake | 7 +- spec/cases/migration_spec.rb | 6 +- spec/cases/model_spec.rb | 5 +- 16 files changed, 158 insertions(+), 144 deletions(-) create mode 100644 lib/arel/nodes/final.rb delete mode 100644 lib/clickhouse-activerecord/migration.rb create mode 100644 lib/core_extensions/active_record/internal_metadata.rb create mode 100644 lib/core_extensions/active_record/schema_migration.rb create mode 100644 lib/core_extensions/arel/nodes/select_core.rb diff --git a/README.md b/README.md index 2c665031..d4644f57 100644 --- a/README.md +++ b/README.md @@ -124,11 +124,11 @@ If you using multiple databases, for example: PostgreSQL, Clickhouse. Schema dump to `db/clickhouse_schema.rb` file: - $ rake clickhouse:schema:dump + $ rake db:schema:dump:clickhouse Schema load from `db/clickhouse_schema.rb` file: - $ rake clickhouse:schema:load + $ rake db:schema:load:clickhouse For export schema to PostgreSQL, you need use: @@ -238,6 +238,7 @@ Donations to this project are going directly to [PNixx](https://github.com/PNixx * BTC address: `1H3rhpf7WEF5JmMZ3PVFMQc7Hm29THgUfN` * ETH address: `0x6F094365A70fe7836A633d2eE80A1FA9758234d5` * XMR address: `42gP71qLB5M43RuDnrQ3vSJFFxis9Kw9VMURhpx9NLQRRwNvaZRjm2TFojAMC8Fk1BQhZNKyWhoyJSn5Ak9kppgZPjE17Zh` +* TON address: `UQBt0-s1igIpJoEup0B1yAUkZ56rzbpruuAjNhQ26MVCaNlC` ## Development diff --git a/lib/active_record/connection_adapters/clickhouse_adapter.rb b/lib/active_record/connection_adapters/clickhouse_adapter.rb index e89ace0d..f8cb1f56 100644 --- a/lib/active_record/connection_adapters/clickhouse_adapter.rb +++ b/lib/active_record/connection_adapters/clickhouse_adapter.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true require 'arel/visitors/clickhouse' +require 'arel/nodes/final' require 'arel/nodes/settings' require 'arel/nodes/using' -require 'clickhouse-activerecord/migration' require 'active_record/connection_adapters/clickhouse/oid/array' require 'active_record/connection_adapters/clickhouse/oid/date' require 'active_record/connection_adapters/clickhouse/oid/date_time' @@ -64,7 +64,7 @@ def is_view module ModelSchema module ClassMethods - delegate :final, :settings, to: :all + delegate :final, :final!, :settings, :settings!, to: :all def is_view @is_view || false @@ -132,23 +132,10 @@ def initialize(logger, connection_parameters, config, full_config) connect end - # Support SchemaMigration from v5.2.2 to v6+ - def schema_migration # :nodoc: - ClickhouseActiverecord::SchemaMigration.new(self) - end - - def internal_metadata # :nodoc: - ClickhouseActiverecord::InternalMetadata.new(self) - end - def migrations_paths @full_config[:migrations_paths] || 'db/migrate_clickhouse' end - def migration_context # :nodoc: - ClickhouseActiverecord::MigrationContext.new(migrations_paths, schema_migration, internal_metadata) - end - def arel_visitor # :nodoc: Arel::Visitors::Clickhouse.new(self) end diff --git a/lib/arel/nodes/final.rb b/lib/arel/nodes/final.rb new file mode 100644 index 00000000..664a859f --- /dev/null +++ b/lib/arel/nodes/final.rb @@ -0,0 +1,7 @@ +module Arel # :nodoc: all + module Nodes + class Final < Arel::Nodes::Unary + delegate :empty?, to: :expr + end + end +end diff --git a/lib/arel/visitors/clickhouse.rb b/lib/arel/visitors/clickhouse.rb index 8b6a1d62..e9a180aa 100644 --- a/lib/arel/visitors/clickhouse.rb +++ b/lib/arel/visitors/clickhouse.rb @@ -13,12 +13,6 @@ def aggregate(name, o, collector) end end - def visit_Arel_Table o, collector - collector = super - collector << ' FINAL' if o.final - collector - end - def visit_Arel_Nodes_SelectOptions(o, collector) maybe_visit o.settings, super end @@ -34,6 +28,12 @@ def visit_Arel_Nodes_UpdateStatement(o, collector) maybe_visit o.limit, collector end + def visit_Arel_Nodes_Final(o, collector) + visit o.expr, collector + collector << ' FINAL' + collector + end + def visit_Arel_Nodes_Settings(o, collector) return collector if o.expr.empty? diff --git a/lib/clickhouse-activerecord.rb b/lib/clickhouse-activerecord.rb index 94a32f1c..67f95bda 100644 --- a/lib/clickhouse-activerecord.rb +++ b/lib/clickhouse-activerecord.rb @@ -2,8 +2,11 @@ require 'active_record/connection_adapters/clickhouse_adapter' +require 'core_extensions/active_record/internal_metadata' require 'core_extensions/active_record/relation' +require 'core_extensions/active_record/schema_migration' +require 'core_extensions/arel/nodes/select_core' require 'core_extensions/arel/nodes/select_statement' require 'core_extensions/arel/select_manager' require 'core_extensions/arel/table' @@ -21,8 +24,11 @@ module ClickhouseActiverecord def self.load + ActiveRecord::InternalMetadata.singleton_class.prepend(CoreExtensions::ActiveRecord::InternalMetadata::ClassMethods) ActiveRecord::Relation.prepend(CoreExtensions::ActiveRecord::Relation) + ActiveRecord::SchemaMigration.singleton_class.prepend(CoreExtensions::ActiveRecord::SchemaMigration::ClassMethods) + Arel::Nodes::SelectCore.prepend(CoreExtensions::Arel::Nodes::SelectCore) Arel::Nodes::SelectStatement.prepend(CoreExtensions::Arel::Nodes::SelectStatement) Arel::SelectManager.prepend(CoreExtensions::Arel::SelectManager) Arel::Table.prepend(CoreExtensions::Arel::Table) diff --git a/lib/clickhouse-activerecord/migration.rb b/lib/clickhouse-activerecord/migration.rb deleted file mode 100644 index 330bdfaa..00000000 --- a/lib/clickhouse-activerecord/migration.rb +++ /dev/null @@ -1,103 +0,0 @@ -require 'active_record/migration' - -module ClickhouseActiverecord - - class SchemaMigration < ::ActiveRecord::SchemaMigration - def create_table - return if table_exists? - - version_options = connection.internal_string_options_for_primary_key - table_options = { - id: false, options: 'ReplacingMergeTree(ver) ORDER BY (version)', if_not_exists: true - } - full_config = connection.instance_variable_get(:@full_config) || {} - - if full_config[:distributed_service_tables] - table_options.merge!(with_distributed: table_name, sharding_key: 'cityHash64(version)') - - distributed_suffix = "_#{full_config[:distributed_service_tables_suffix] || 'distributed'}" - end - - connection.create_table(table_name + distributed_suffix.to_s, **table_options) do |t| - t.string :version, **version_options - t.column :active, 'Int8', null: false, default: '1' - t.datetime :ver, null: false, default: -> { 'now()' } - end - end - - def versions - table = arel_table.dup - table.final = true - sm = Arel::SelectManager.new(table) - sm.project(arel_table[primary_key]) - sm.order(arel_table[primary_key].asc) - sm.where([arel_table['active'].eq(1)]) - - connection.select_values(sm, "#{self.class} Load") - end - - def delete_version(version) - im = Arel::InsertManager.new(arel_table) - im.insert(arel_table[primary_key] => version.to_s, arel_table['active'] => 0) - connection.insert(im, "#{self.class} Create Rollback Version", primary_key, version) - end - end - - class InternalMetadata < ::ActiveRecord::InternalMetadata - - def create_table - return if table_exists? || !enabled? - - key_options = connection.internal_string_options_for_primary_key - table_options = { - id: false, - options: connection.adapter_name.downcase == 'clickhouse' ? 'ReplacingMergeTree(created_at) PARTITION BY key ORDER BY key' : '', - if_not_exists: true - } - full_config = connection.instance_variable_get(:@full_config) || {} - - if full_config[:distributed_service_tables] - table_options.merge!(with_distributed: table_name, sharding_key: 'cityHash64(created_at)') - - distributed_suffix = "_#{full_config[:distributed_service_tables_suffix] || 'distributed'}" - end - - connection.create_table(table_name + distributed_suffix.to_s, **table_options) do |t| - t.string :key, **key_options - t.string :value - t.timestamps - end - end - - private - - def update_entry(key, new_value) - create_entry(key, new_value) - end - - def select_entry(key) - table = arel_table.dup - table.final = true - sm = Arel::SelectManager.new(table) - sm.project(Arel::Nodes::SqlLiteral.new("*")) - sm.where(table[primary_key].eq(Arel::Nodes::BindParam.new(key))) - sm.order(table[primary_key].asc) - sm.limit = 1 - - connection.select_all(sm, "#{self.class} Load").first - end - end - - class MigrationContext < ::ActiveRecord::MigrationContext #:nodoc: - - def get_all_versions - if schema_migration.table_exists? - schema_migration.versions.map(&:to_i) - else - [] - end - end - - end - -end diff --git a/lib/clickhouse-activerecord/version.rb b/lib/clickhouse-activerecord/version.rb index c4f4a61e..f7d40eb2 100644 --- a/lib/clickhouse-activerecord/version.rb +++ b/lib/clickhouse-activerecord/version.rb @@ -1,3 +1,3 @@ module ClickhouseActiverecord - VERSION = '1.0.2' + VERSION = '1.0.3' end diff --git a/lib/core_extensions/active_record/internal_metadata.rb b/lib/core_extensions/active_record/internal_metadata.rb new file mode 100644 index 00000000..fb7f4d4f --- /dev/null +++ b/lib/core_extensions/active_record/internal_metadata.rb @@ -0,0 +1,46 @@ +module CoreExtensions + module ActiveRecord + module InternalMetadata + module ClassMethods + + def []=(key, value) + row = final.find_by(key: key) + if row.nil? || row.value != value + create!(key: key, value: value) + end + end + + def [](key) + final.where(key: key).pluck(:value).first + end + + def create_table + return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter) + return if table_exists? || !enabled? + + key_options = connection.internal_string_options_for_primary_key + table_options = { + id: false, + options: connection.adapter_name.downcase == 'clickhouse' ? 'ReplacingMergeTree(created_at) PARTITION BY key ORDER BY key' : '', + if_not_exists: true + } + full_config = connection.instance_variable_get(:@full_config) || {} + + if full_config[:distributed_service_tables] + table_options.merge!(with_distributed: table_name, sharding_key: 'cityHash64(created_at)') + + distributed_suffix = "_#{full_config[:distributed_service_tables_suffix] || 'distributed'}" + else + distributed_suffix = '' + end + + connection.create_table(table_name + distributed_suffix.to_s, **table_options) do |t| + t.string :key, **key_options + t.string :value + t.timestamps + end + end + end + end + end +end diff --git a/lib/core_extensions/active_record/relation.rb b/lib/core_extensions/active_record/relation.rb index 35c7a50d..8256bf8e 100644 --- a/lib/core_extensions/active_record/relation.rb +++ b/lib/core_extensions/active_record/relation.rb @@ -38,17 +38,14 @@ def settings!(**opts) # # SELECT users.* FROM users FINAL # # An ActiveRecord::ActiveRecordError will be raised if database not ClickHouse. - # @param [Boolean] final - def final(final = true) - spawn.final!(final) + def final + spawn.final! end - # @param [Boolean] final - def final!(final = true) + def final! assert_mutability! check_command('FINAL') - @table = @table.dup - @table.final = final + @values[:final] = true self end @@ -79,6 +76,7 @@ def check_command(cmd) def build_arel(aliases = nil) arel = super + arel.final! if @values[:final].present? arel.settings(@values[:settings]) if @values[:settings].present? arel.using(@values[:using]) if @values[:using].present? diff --git a/lib/core_extensions/active_record/schema_migration.rb b/lib/core_extensions/active_record/schema_migration.rb new file mode 100644 index 00000000..3ff978e2 --- /dev/null +++ b/lib/core_extensions/active_record/schema_migration.rb @@ -0,0 +1,48 @@ +module CoreExtensions + module ActiveRecord + module SchemaMigration + module ClassMethods + + def create_table + return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter) + + return if table_exists? + + version_options = connection.internal_string_options_for_primary_key + table_options = { + id: false, options: 'ReplacingMergeTree(ver) ORDER BY (version)', if_not_exists: true + } + full_config = connection.instance_variable_get(:@full_config) || {} + + if full_config[:distributed_service_tables] + table_options.merge!(with_distributed: table_name, sharding_key: 'cityHash64(version)') + + distributed_suffix = "_#{full_config[:distributed_service_tables_suffix] || 'distributed'}" + else + distributed_suffix = '' + end + + connection.create_table(table_name + distributed_suffix.to_s, **table_options) do |t| + t.string :version, **version_options + t.column :active, 'Int8', null: false, default: '1' + t.datetime :ver, null: false, default: -> { 'now()' } + end + end + + def delete_version(version) + return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter) + + im = Arel::InsertManager.new(arel_table) + im.insert(arel_table[primary_key] => version.to_s, arel_table['active'] => 0) + connection.insert(im, "#{self.class} Create Rollback Version", primary_key, version) + end + + def all_versions + return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter) + + final.where(active: 1).order(:version).pluck(:version) + end + end + end + end +end diff --git a/lib/core_extensions/arel/nodes/select_core.rb b/lib/core_extensions/arel/nodes/select_core.rb new file mode 100644 index 00000000..50be451c --- /dev/null +++ b/lib/core_extensions/arel/nodes/select_core.rb @@ -0,0 +1,19 @@ +module CoreExtensions + module Arel # :nodoc: all + module Nodes + module SelectCore + attr_accessor :final + + def source + return super unless final + + ::Arel::Nodes::Final.new(super) + end + + def eql?(other) + super && final == other.final + end + end + end + end +end diff --git a/lib/core_extensions/arel/select_manager.rb b/lib/core_extensions/arel/select_manager.rb index d5ea3f7a..a8592a06 100644 --- a/lib/core_extensions/arel/select_manager.rb +++ b/lib/core_extensions/arel/select_manager.rb @@ -2,6 +2,11 @@ module CoreExtensions module Arel module SelectManager + def final! + @ctx.final = true + self + end + # @param [Hash] values def settings(values) @ast.settings = ::Arel::Nodes::Settings.new(values) diff --git a/lib/core_extensions/arel/table.rb b/lib/core_extensions/arel/table.rb index 89233430..a49f2461 100644 --- a/lib/core_extensions/arel/table.rb +++ b/lib/core_extensions/arel/table.rb @@ -1,8 +1,6 @@ module CoreExtensions module Arel module Table - attr_accessor :final - def is_view type_caster.is_view end diff --git a/lib/tasks/clickhouse.rake b/lib/tasks/clickhouse.rake index 5bc8378b..eb37aea5 100644 --- a/lib/tasks/clickhouse.rake +++ b/lib/tasks/clickhouse.rake @@ -12,15 +12,16 @@ namespace :clickhouse do end namespace :schema do - # TODO: not testing + # TODO: deprecated desc 'Load database schema' task load: %i[prepare_internal_metadata_table] do simple = ENV['simple'] || ARGV.any? { |a| a.include?('--simple') } ? '_simple' : nil - config = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: 'clickhouse') - ClickhouseActiverecord::SchemaMigration.new(ActiveRecord::Base.establish_connection(config).connection).drop_table + ActiveRecord::Base.establish_connection(:clickhouse) + ActiveRecord::SchemaMigration.drop_table load(Rails.root.join("db/clickhouse_schema#{simple}.rb")) end + # TODO: deprecated desc 'Dump database schema' task dump: :environment do |_, args| simple = ENV['simple'] || args[:simple] || ARGV.any? { |a| a.include?('--simple') } ? '_simple' : nil diff --git a/spec/cases/migration_spec.rb b/spec/cases/migration_spec.rb index 6485ad87..6fb590a6 100644 --- a/spec/cases/migration_spec.rb +++ b/spec/cases/migration_spec.rb @@ -9,7 +9,7 @@ end let(:directory) { raise 'NotImplemented' } let(:migrations_dir) { File.join(FIXTURES_PATH, 'migrations', directory) } - let(:migration_context) { ClickhouseActiverecord::MigrationContext.new(migrations_dir, model.connection.schema_migration, model.connection.internal_metadata) } + let(:migration_context) { ActiveRecord::MigrationContext.new(migrations_dir, model.connection.schema_migration, model.connection.internal_metadata) } if ActiveRecord::version >= Gem::Version.new('6.1') connection_config = ActiveRecord::Base.connection_db_config.configuration_hash @@ -313,11 +313,11 @@ describe 'drop table sync' do it 'drops table' do migrations_dir = File.join(FIXTURES_PATH, 'migrations', 'dsl_drop_table_sync') - quietly { ClickhouseActiverecord::MigrationContext.new(migrations_dir, model.connection.schema_migration).up(1) } + quietly { ActiveRecord::MigrationContext.new(migrations_dir, model.connection.schema_migration).up(1) } expect(ActiveRecord::Base.connection.tables).to include('some') - quietly { ClickhouseActiverecord::MigrationContext.new(migrations_dir, model.connection.schema_migration).up(2) } + quietly { ActiveRecord::MigrationContext.new(migrations_dir, model.connection.schema_migration).up(2) } expect(ActiveRecord::Base.connection.tables).not_to include('some') end diff --git a/spec/cases/model_spec.rb b/spec/cases/model_spec.rb index 9fccd0a1..fafd3993 100644 --- a/spec/cases/model_spec.rb +++ b/spec/cases/model_spec.rb @@ -19,7 +19,7 @@ class Model < ActiveRecord::Base before do migrations_dir = File.join(FIXTURES_PATH, 'migrations', 'add_sample_data') - quietly { ClickhouseActiverecord::MigrationContext.new(migrations_dir, model.connection.schema_migration).up } + quietly { ActiveRecord::MigrationContext.new(migrations_dir, model.connection.schema_migration).up } end @@ -161,6 +161,7 @@ class Model < ActiveRecord::Base it 'select' do expect(model.count).to eq(2) expect(model.final.count).to eq(1) + expect(model.final!.count).to eq(1) expect(model.final.where(date: '2023-07-21').to_sql).to eq('SELECT sample.* FROM sample FINAL WHERE sample.date = \'2023-07-21\'') end end @@ -176,7 +177,7 @@ class Model < ActiveRecord::Base before do migrations_dir = File.join(FIXTURES_PATH, 'migrations', 'add_array_datetime') - quietly { ClickhouseActiverecord::MigrationContext.new(migrations_dir, model.connection.schema_migration).up } + quietly { ActiveRecord::MigrationContext.new(migrations_dir, model.connection.schema_migration).up } end describe '#create' do