fix schema migration final #105

README.md

@@ -1,6 +1,6 @@
 # Clickhouse::Activerecord
 
-A Ruby database ActiveRecord driver for ClickHouse. Support Rails >= 5.2.
+A Ruby database ActiveRecord driver for ClickHouse. Support Rails >= 7.1.
 Support ClickHouse version from 22.0 LTS.
 
 ## Installation
@@ -50,41 +50,31 @@ class ActionView < ActiveRecord::Base
 end
 ```
 
-## Usage in Rails 5
+## Usage in Rails
 
 Add your `database.yml` connection information with postfix `_clickhouse` for you environment:
 
 ```yml
-development_clickhouse:
+development:
   adapter: clickhouse
   database: database
 ```
 
-Add to your model:
+Your model example:
 
 ```ruby
 class Action < ActiveRecord::Base
-  establish_connection "#{Rails.env}_clickhouse".to_sym
 end
 ```
 
 For materialized view model add:
 ```ruby
 class ActionView < ActiveRecord::Base
-  establish_connection "#{Rails.env}_clickhouse".to_sym
   self.is_view = true
 end
 ```
 
-Or global connection:
-
-```yml
-development:
-  adapter: clickhouse
-  database: database
-```
-
-## Usage in Rails 6 with second database
+## Usage in Rails with second database
 
 Add your `database.yml` connection information for you environment:
 
@@ -102,31 +92,31 @@ Connection [Multiple Databases with Active Record](https://guides.rubyonrails.or
 
 ```ruby
 class Action < ActiveRecord::Base
-  connects_to database: { writing: :clickhouse, reading: :clickhouse }
+  establish_connection :clickhouse
 end
 ```
 
 ### Rake tasks
 
-**Note!** For Rails 6 you can use default rake tasks if you configure `migrations_paths` in your `database.yml`, for example: `rake db:migrate`
-
 Create / drop / purge / reset database:
 
-    $ rake clickhouse:create
-    $ rake clickhouse:drop
-    $ rake clickhouse:purge
-    $ rake clickhouse:reset
+    $ rake db:create
+    $ rake db:drop
+    $ rake db:purge
+    $ rake db:reset
 
-Prepare system tables for rails:
+Or with multiple databases:
 
-    $ rake clickhouse:prepare_schema_migration_table
-    $ rake clickhouse:prepare_internal_metadata_table
+    $ rake db:create:clickhouse
+    $ rake db:drop:clickhouse
+    $ rake db:purge:clickhouse
+    $ rake db:reset:clickhouse
 
 Migration:
 
     $ rails g clickhouse_migration MIGRATION_NAME COLUMNS
-    $ rake clickhouse:migrate
-    $ rake clickhouse:rollback
+    $ rake db:migrate
+    $ rake db:rollback
 
 ### Dump / Load for multiple using databases
 
@@ -195,20 +185,20 @@ User.joins(:actions).using(:group_id)
 Integer types are unsigned by default. Specify signed values with `:unsigned =>
 false`. The default integer is `UInt32`
 
-| Type (bit size)    | Range | :limit (byte size) |
-| :---        |    :----:   |          ---: |
-| Int8 | -128 to 127 | 1 | 
-| Int16 | -32768 to 32767 | 2 |
-| Int32 | -2147483648 to 2,147,483,647 | 3,4 |
-| Int64 | -9223372036854775808 to 9223372036854775807] |  5,6,7,8 |
-| Int128 | ... | 9 - 15 |
-| Int256 | ... | 16+ |
-| UInt8 | 0 to 255 | 1 |
-| UInt16 | 0 to 65,535 | 2 |
-| UInt32 | 0 to 4,294,967,295 | 3,4 |
-| UInt64 | 0 to 18446744073709551615 | 5,6,7,8 |
-| UInt256 | 0 to ... | 8+ |
-| Array | ... | ... |
+| Type (bit size) |                    Range                     | :limit (byte size) |
+|:----------------|:--------------------------------------------:|-------------------:|
+| Int8            |                 -128 to 127                  |                  1 | 
+| Int16           |               -32768 to 32767                |                  2 |
+| Int32           |         -2147483648 to 2,147,483,647         |                3,4 |
+| Int64           | -9223372036854775808 to 9223372036854775807] |            5,6,7,8 |
+| Int128          |                     ...                      |             9 - 15 |
+| Int256          |                     ...                      |                16+ |
+| UInt8           |                   0 to 255                   |                  1 |
+| UInt16          |                 0 to 65,535                  |                  2 |
+| UInt32          |              0 to 4,294,967,295              |                3,4 |
+| UInt64          |          0 to 18446744073709551615           |            5,6,7,8 |
+| UInt256         |                   0 to ...                   |                 8+ |
+| Array           |                     ...                      |                ... |
 
 Example:
 

lib/arel/visitors/clickhouse.rb

@@ -15,7 +15,7 @@ def aggregate(name, o, collector)
 
       def visit_Arel_Table o, collector
         collector = super
-        collector << ' FINAL ' if o.final
+        collector << ' FINAL' if o.final
         collector
       end
 

lib/clickhouse-activerecord/migration.rb

@@ -25,8 +25,21 @@ def create_table
       end
     end
 
-    def all_versions
-      final.where(active: 1).order(:version).pluck(:version)
+    def versions
+      table = arel_table.dup
+      table.final = true
+      sm = Arel::SelectManager.new(table)
+      sm.project(arel_table[primary_key])
+      sm.order(arel_table[primary_key].asc)
+      sm.where([arel_table['active'].eq(1)])
+
+      connection.select_values(sm, "#{self.class} Load")
+    end
+
+    def delete_version(version)
+      im = Arel::InsertManager.new(arel_table)
+      im.insert(arel_table[primary_key] => version.to_s, arel_table['active'] => 0)
+      connection.insert(im, "#{self.class} Create Rollback Version", primary_key, version)
     end
   end
 
@@ -77,46 +90,14 @@ def select_entry(key)
 
   class MigrationContext < ::ActiveRecord::MigrationContext #:nodoc:
 
-    def up(target_version = nil)
-      selected_migrations = if block_given?
-        migrations.select { |m| yield m }
-      else
-        migrations
-      end
-
-      ClickhouseActiverecord::Migrator.new(:up, selected_migrations, schema_migration, internal_metadata, target_version).migrate
-    end
-
-    def down(target_version = nil)
-      selected_migrations = if block_given?
-        migrations.select { |m| yield m }
-      else
-        migrations
-      end
-
-      ClickhouseActiverecord::Migrator.new(:down, selected_migrations, schema_migration, internal_metadata, target_version).migrate
-    end
-
     def get_all_versions
       if schema_migration.table_exists?
-        schema_migration.all_versions.map(&:to_i)
+        schema_migration.versions.map(&:to_i)
       else
         []
       end
     end
 
   end
 
-  class Migrator < ::ActiveRecord::Migrator
-
-    def record_version_state_after_migrating(version)
-      if down?
-        migrated.delete(version)
-        @schema_migration.create!(version: version.to_s, active: 0)
-      else
-        super
-      end
-    end
-
-  end
 end

lib/clickhouse-activerecord/version.rb

@@ -1,3 +1,3 @@
 module ClickhouseActiverecord
-  VERSION = '0.6.1'
+  VERSION = '1.0.0'
 end

lib/core_extensions/active_record/relation.rb

@@ -11,21 +11,65 @@ def reverse_order!
         self
       end
 
+      # Define settings in the SETTINGS clause of the SELECT query. The setting value is applied only to that query and is reset to the default or previous value after the query is executed.
+      # For example:
+      #
+      #   users = User.settings(optimize_read_in_order: 1, cast_keep_nullable: 1).where(name: 'John')
+      #   # SELECT users.* FROM users WHERE users.name = 'John' SETTINGS optimize_read_in_order = 1, cast_keep_nullable = 1
+      #
+      # An <tt>ActiveRecord::ActiveRecordError</tt> will be raised if database not ClickHouse.
       # @param [Hash] opts
       def settings(**opts)
+        spawn.settings!(**opts)
+      end
+
+      # @param [Hash] opts
+      def settings!(**opts)
+        assert_mutability!
         check_command('SETTINGS')
         @values[:settings] = (@values[:settings] || {}).merge opts
         self
       end
 
+      # When FINAL is specified, ClickHouse fully merges the data before returning the result and thus performs all data transformations that happen during merges for the given table engine.
+      # For example:
+      #
+      #   users = User.final.all
+      #   # SELECT users.* FROM users FINAL
+      #
+      # An <tt>ActiveRecord::ActiveRecordError</tt> will be raised if database not ClickHouse.
       # @param [Boolean] final
       def final(final = true)
+        spawn.final!(final)
+      end
+
+      # @param [Boolean] final
+      def final!(final = true)
+        assert_mutability!
         check_command('FINAL')
         @table = @table.dup
         @table.final = final
         self
       end
 
+      # The USING clause specifies one or more columns to join, which establishes the equality of these columns. For example:
+      #
+      #   users = User.joins(:joins).using(:event_name, :date)
+      #   # SELECT users.* FROM users INNER JOIN joins USING event_name,date
+      #
+      # An <tt>ActiveRecord::ActiveRecordError</tt> will be raised if database not ClickHouse.
+      # @param [Array] opts
+      def using(*opts)
+        spawn.using!(*opts)
+      end
+
+      # @param [Array] opts
+      def using!(*opts)
+        assert_mutability!
+        @values[:using] = opts
+        self
+      end
+
       private
 
       def check_command(cmd)
@@ -36,6 +80,7 @@ def build_arel(aliases = nil)
         arel = super
 
         arel.settings(@values[:settings]) if @values[:settings].present?
+        arel.using(@values[:using]) if @values[:using].present?
 
         arel
       end

lib/tasks/clickhouse.rake

@@ -2,17 +2,13 @@
 
 namespace :clickhouse do
   task prepare_schema_migration_table: :environment do
-    ClickhouseActiverecord::SchemaMigration.create_table unless ENV['simple'] || ARGV.any? { |a| a.include?('--simple') }
+    connection = ActiveRecord::Tasks::DatabaseTasks.migration_connection
+    connection.schema_migration.create_table unless ENV['simple'] || ARGV.any? { |a| a.include?('--simple') }
   end
 
   task prepare_internal_metadata_table: :environment do
-    ClickhouseActiverecord::InternalMetadata.create_table unless ENV['simple'] || ARGV.any? { |a| a.include?('--simple') }
-  end
-
-  task load_config: :environment do
-    ENV['SCHEMA'] = 'db/clickhouse_schema.rb'
-    ActiveRecord::Migrator.migrations_paths = %w[db/migrate_clickhouse]
-    ActiveRecord::Base.establish_connection(:clickhouse)
+    connection = ActiveRecord::Tasks::DatabaseTasks.migration_connection
+    connection.internal_metadata.create_table unless ENV['simple'] || ARGV.any? { |a| a.include?('--simple') }
   end
 
   namespace :schema do
@@ -37,52 +33,55 @@ namespace :clickhouse do
 
   namespace :structure do
     desc 'Load database structure'
-    task load: [:load_config, 'db:check_protected_environments'] do
+    task load: ['db:check_protected_environments'] do
       config = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: 'clickhouse')
       ClickhouseActiverecord::Tasks.new(config).structure_load(Rails.root.join('db/clickhouse_structure.sql'))
     end
 
     desc 'Dump database structure'
-    task dump: [:load_config, 'db:check_protected_environments'] do
+    task dump: ['db:check_protected_environments'] do
       config = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: 'clickhouse')
       ClickhouseActiverecord::Tasks.new(config).structure_dump(Rails.root.join('db/clickhouse_structure.sql'))
     end
   end
 
   desc 'Creates the database from DATABASE_URL or config/database.yml'
-  task create: [:load_config] do
+  task create: [] do
     config = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: 'clickhouse')
     ActiveRecord::Tasks::DatabaseTasks.create(config)
   end
 
   desc 'Drops the database from DATABASE_URL or config/database.yml'
-  task drop: [:load_config, 'db:check_protected_environments'] do
+  task drop: ['db:check_protected_environments'] do
     config = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: 'clickhouse')
     ActiveRecord::Tasks::DatabaseTasks.drop(config)
   end
 
   desc 'Empty the database from DATABASE_URL or config/database.yml'
-  task purge: [:load_config, 'db:check_protected_environments'] do
+  task purge: ['db:check_protected_environments'] do
     config = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: 'clickhouse')
     ActiveRecord::Tasks::DatabaseTasks.purge(config)
   end
 
   # desc 'Resets your database using your migrations for the current environment'
-  task reset: :load_config do
+  task :reset  do
     Rake::Task['clickhouse:purge'].execute
     Rake::Task['clickhouse:migrate'].execute
   end
 
   desc 'Migrate the clickhouse database'
-  task migrate: %i[load_config prepare_schema_migration_table prepare_internal_metadata_table] do
-    Rake::Task['db:migrate'].execute
+  task migrate: %i[prepare_schema_migration_table prepare_internal_metadata_table] do
+    Rake::Task['db:migrate:clickhouse'].execute
     if File.exists? "#{Rails.root}/db/clickhouse_schema_simple.rb"
       Rake::Task['clickhouse:schema:dump'].execute(simple: true)
     end
   end
 
   desc 'Rollback the clickhouse database'
-  task rollback: %i[load_config prepare_schema_migration_table prepare_internal_metadata_table] do
-    Rake::Task['db:rollback'].execute
+  task rollback: %i[prepare_schema_migration_table prepare_internal_metadata_table] do
+    Rake::Task['db:rollback:clickhouse'].execute
+    if File.exists? "#{Rails.root}/db/clickhouse_schema_simple.rb"
+      Rake::Task['clickhouse:schema:dump'].execute(simple: true)
+    end
   end
 end

spec/cases/model_spec.rb

@@ -137,6 +137,7 @@ class Model < ActiveRecord::Base
       it 'select' do
         expect(model.count).to eq(2)
         expect(model.final.count).to eq(1)
+        expect(model.final.where(date: '2023-07-21').to_sql).to eq('SELECT sample.* FROM sample FINAL WHERE sample.date = \'2023-07-21\'')
       end
     end
   end