From 7170fbf96c1060d691ee4d14e70fc41116c1862a Mon Sep 17 00:00:00 2001 From: Greg Rychlewski Date: Fri, 21 Jul 2023 11:34:53 -0400 Subject: [PATCH] allow a single prefix for mysqldump --- integration_test/myxql/storage_test.exs | 39 ++---- lib/ecto/adapters/myxql.ex | 172 +++++++++++++----------- lib/mix/tasks/ecto.dump.ex | 12 +- 3 files changed, 114 insertions(+), 109 deletions(-) diff --git a/integration_test/myxql/storage_test.exs b/integration_test/myxql/storage_test.exs index 9f45badb..47e2ccc8 100644 --- a/integration_test/myxql/storage_test.exs +++ b/integration_test/myxql/storage_test.exs @@ -146,57 +146,44 @@ defmodule Ecto.Integration.StorageTest do end test "structure dump and load with migrations table" do + default_db = "ecto_test" num = @base_migration + System.unique_integer([:positive]) :ok = Ecto.Migrator.up(PoolRepo, num, Migration, log: false) {:ok, path} = Ecto.Adapters.MyXQL.structure_dump(tmp_path(), TestRepo.config()) contents = File.read!(path) - assert contents =~ "INSERT INTO `ecto_test`.`schema_migrations` (version) VALUES (#{num})" + assert contents =~ "Database: #{default_db}" + assert contents =~ "INSERT INTO `schema_migrations` (version) VALUES (#{num})" end - test "dumps structure and schema_migration records from multiple prefixes" do - # Create the test_schema schema + test "raises when attempting to dump multiple prefixes" do create_database() prefix = params()[:database] - - # Run migrations - version = @base_migration + System.unique_integer([:positive]) - :ok = Ecto.Migrator.up(PoolRepo, version, Migration, log: false) - :ok = Ecto.Migrator.up(PoolRepo, version, Migration, log: false, prefix: prefix) - config = Keyword.put(TestRepo.config(), :dump_prefixes, ["ecto_test", prefix]) - {:ok, path} = Ecto.Adapters.MyXQL.structure_dump(tmp_path(), config) - contents = File.read!(path) + msg = "cannot dump multiple prefixes with MySQL. Please run the command separately for each prefix." - assert contents =~ "Current Database: `#{prefix}`" - assert contents =~ "Current Database: `ecto_test`" - assert contents =~ "CREATE TABLE `schema_migrations`" - assert contents =~ ~s[INSERT INTO `#{prefix}`.`schema_migrations` (version) VALUES (#{version})] - assert contents =~ ~s[INSERT INTO `ecto_test`.`schema_migrations` (version) VALUES (#{version})] + assert_raise ArgumentError, msg, fn -> + Ecto.Adapters.MyXQL.structure_dump(tmp_path(), config) + end after drop_database() end test "dumps structure and schema_migration records only from queried prefix" do - # Create the test_schema schema + # Create the storage_mgt database create_database() prefix = params()[:database] # Run migrations version = @base_migration + System.unique_integer([:positive]) - :ok = Ecto.Migrator.up(PoolRepo, version, Migration, log: false) :ok = Ecto.Migrator.up(PoolRepo, version, Migration, log: false, prefix: prefix) - config = Keyword.put(TestRepo.config(), :dump_prefixes, ["ecto_test"]) + config = Keyword.put(TestRepo.config(), :dump_prefixes, [prefix]) {:ok, path} = Ecto.Adapters.MyXQL.structure_dump(tmp_path(), config) contents = File.read!(path) - refute contents =~ "Current Database: `#{prefix}`" - assert contents =~ "Current Database: `ecto_test`" - assert contents =~ "CREATE TABLE `schema_migrations`" - refute contents =~ ~s[INSERT INTO `#{prefix}`.`schema_migrations` (version) VALUES (#{version})] - assert contents =~ ~s[INSERT INTO `ecto_test`.`schema_migrations` (version) VALUES (#{version})] - after - drop_database() + refute contents =~ "USE `#{prefix}`" + assert contents =~ "Database: #{prefix}" + assert contents =~ "INSERT INTO `schema_migrations` (version) VALUES (#{version})" end defp strip_timestamp(dump) do diff --git a/lib/ecto/adapters/myxql.ex b/lib/ecto/adapters/myxql.ex index 44295cb4..3dce5dbc 100644 --- a/lib/ecto/adapters/myxql.ex +++ b/lib/ecto/adapters/myxql.ex @@ -51,9 +51,11 @@ defmodule Ecto.Adapters.MyXQL do * `:collation` - the collation order * `:dump_path` - where to place dumped structures * `:dump_prefixes` - list of prefixes that will be included in the - structure dump. When specified, the prefixes will have their definitions - dumped along with the data in their migration table. When it is not - specified, only the configured database and its migration table are dumped. + structure dump. For MySQL, this list must be of length 1. Multiple + prefixes are not supported. When specified, the prefixes will have + their definitions dumped along with the data in their migration table. + When it is not specified, only the configured database and its migration + table are dumped. ### After connect callback @@ -147,12 +149,12 @@ defmodule Ecto.Adapters.MyXQL do ## Custom MySQL types @impl true - def loaders({:map, _}, type), do: [&json_decode/1, &Ecto.Type.embedded_load(type, &1, :json)] - def loaders(:map, type), do: [&json_decode/1, type] - def loaders(:float, type), do: [&float_decode/1, type] - def loaders(:boolean, type), do: [&bool_decode/1, type] - def loaders(:binary_id, type), do: [Ecto.UUID, type] - def loaders(_, type), do: [type] + def loaders({:map, _}, type), do: [&json_decode/1, &Ecto.Type.embedded_load(type, &1, :json)] + def loaders(:map, type), do: [&json_decode/1, type] + def loaders(:float, type), do: [&float_decode/1, type] + def loaders(:boolean, type), do: [&bool_decode/1, type] + def loaders(:binary_id, type), do: [Ecto.UUID, type] + def loaders(_, type), do: [type] defp bool_decode(<<0>>), do: {:ok, false} defp bool_decode(<<1>>), do: {:ok, true} @@ -172,14 +174,19 @@ defmodule Ecto.Adapters.MyXQL do @impl true def storage_up(opts) do - database = Keyword.fetch!(opts, :database) || raise ":database is nil in repository configuration" + database = + Keyword.fetch!(opts, :database) || raise ":database is nil in repository configuration" + opts = Keyword.delete(opts, :database) charset = opts[:charset] || "utf8mb4" - check_existence_command = "SELECT TRUE FROM information_schema.schemata WHERE schema_name = '#{database}'" + check_existence_command = + "SELECT TRUE FROM information_schema.schemata WHERE schema_name = '#{database}'" + case run_query(check_existence_command, opts) do {:ok, %{num_rows: 1}} -> {:error, :already_up} + _ -> create_command = ~s(CREATE DATABASE `#{database}` DEFAULT CHARACTER SET = #{charset}) @@ -188,36 +195,46 @@ defmodule Ecto.Adapters.MyXQL do case run_query(create_command, opts) do {:ok, _} -> :ok + {:error, %{mysql: %{name: :ER_DB_CREATE_EXISTS}}} -> {:error, :already_up} + {:error, error} -> {:error, Exception.message(error)} + {:exit, exit} -> {:error, exit_to_exception(exit)} end end end - defp concat_if(content, nil, _fun), do: content + defp concat_if(content, nil, _fun), do: content defp concat_if(content, value, fun), do: content <> " " <> fun.(value) @impl true def storage_down(opts) do - database = Keyword.fetch!(opts, :database) || raise ":database is nil in repository configuration" + database = + Keyword.fetch!(opts, :database) || raise ":database is nil in repository configuration" + opts = Keyword.delete(opts, :database) command = "DROP DATABASE `#{database}`" case run_query(command, opts) do {:ok, _} -> :ok + {:error, %{mysql: %{name: :ER_DB_DROP_EXISTS}}} -> {:error, :already_down} + {:error, %{mysql: %{name: :ER_BAD_DB_ERROR}}} -> {:error, :already_down} + {:error, error} -> {:error, Exception.message(error)} + {:exit, :killed} -> {:error, :already_down} + {:exit, exit} -> {:error, exit_to_exception(exit)} end @@ -225,10 +242,13 @@ defmodule Ecto.Adapters.MyXQL do @impl Ecto.Adapter.Storage def storage_status(opts) do - database = Keyword.fetch!(opts, :database) || raise ":database is nil in repository configuration" + database = + Keyword.fetch!(opts, :database) || raise ":database is nil in repository configuration" + opts = Keyword.delete(opts, :database) - check_database_query = "SELECT schema_name FROM information_schema.schemata WHERE schema_name = '#{database}'" + check_database_query = + "SELECT schema_name FROM information_schema.schemata WHERE schema_name = '#{database}'" case run_query(check_database_query, opts) do {:ok, %{num_rows: 0}} -> :down @@ -250,7 +270,7 @@ defmodule Ecto.Adapters.MyXQL do Ecto.Adapters.SQL.raise_migration_pool_size_error() end - opts = Keyword.merge(opts, [timeout: :infinity, telemetry_options: [schema_migration: true]]) + opts = Keyword.merge(opts, timeout: :infinity, telemetry_options: [schema_migration: true]) {:ok, result} = transaction(meta, opts, fn -> @@ -275,11 +295,13 @@ defmodule Ecto.Adapters.MyXQL do key = primary_key!(schema_meta, returning) {fields, values} = :lists.unzip(params) sql = @conn.insert(prefix, source, fields, [fields], on_conflict, [], []) - opts = if is_nil(Keyword.get(opts, :cache_statement)) do - [{:cache_statement, "ecto_insert_#{source}_#{length(fields)}"} | opts] - else - opts - end + + opts = + if is_nil(Keyword.get(opts, :cache_statement)) do + [{:cache_statement, "ecto_insert_#{source}_#{length(fields)}"} | opts] + else + opts + end case Ecto.Adapters.SQL.query(adapter_meta, sql, values ++ query_params, opts) do {:ok, %{num_rows: 0}} -> @@ -295,7 +317,7 @@ defmodule Ecto.Adapters.MyXQL do {:error, err} -> case @conn.to_constraints(err, source: source) do - [] -> raise err + [] -> raise err constraints -> {:invalid, constraints} end end @@ -303,9 +325,11 @@ defmodule Ecto.Adapters.MyXQL do defp primary_key!(%{autogenerate_id: {_, key, _type}}, [key]), do: key defp primary_key!(_, []), do: nil + defp primary_key!(%{schema: schema}, returning) do - raise ArgumentError, "MySQL does not support :read_after_writes in schemas for non-primary keys. " <> - "The following fields in #{inspect schema} are tagged as such: #{inspect returning}" + raise ArgumentError, + "MySQL does not support :read_after_writes in schemas for non-primary keys. " <> + "The following fields in #{inspect(schema)} are tagged as such: #{inspect(returning)}" end defp last_insert_id(nil, _last_insert_id), do: [] @@ -315,11 +339,11 @@ defmodule Ecto.Adapters.MyXQL do @impl true def structure_dump(default, config) do table = config[:migration_source] || "schema_migrations" - path = config[:dump_path] || Path.join(default, "structure.sql") - prefixes = config[:dump_prefixes] || [config[:database]] + path = config[:dump_path] || Path.join(default, "structure.sql") + database = dump_database!(config[:dump_prefixes], config[:database]) - with {:ok, versions} <- select_versions(prefixes, table, config), - {:ok, contents} <- mysql_dump(prefixes, config), + with {:ok, versions} <- select_versions(database, table, config), + {:ok, contents} <- mysql_dump(database, config), {:ok, contents} <- append_versions(table, versions, contents) do File.mkdir_p!(Path.dirname(path)) File.write!(path, contents) @@ -327,25 +351,25 @@ defmodule Ecto.Adapters.MyXQL do end end - defp select_versions(prefixes, table, config) do - result = - Enum.reduce_while(prefixes, [], fn prefix, versions -> - case run_query(~s[SELECT version FROM `#{prefix}`.`#{table}` ORDER BY version], config) do - {:ok, %{rows: rows}} -> {:cont, Enum.map(rows, &{prefix, hd(&1)}) ++ versions} - {:error, %{mysql: %{name: :ER_NO_SUCH_TABLE}}} -> {:cont, versions} - {:error, _} = error -> {:halt, error} - {:exit, exit} -> {:halt, {:error, exit_to_exception(exit)}} - end - end) + defp dump_database!([prefix], _), do: prefix + defp dump_database!(nil, config_database), do: config_database + + defp dump_database!(_, _) do + raise ArgumentError, + "cannot dump multiple prefixes with MySQL. Please run the command separately for each prefix." + end - case result do + defp select_versions(database, table, config) do + case run_query(~s[SELECT version FROM `#{database}`.`#{table}` ORDER BY version], config) do + {:ok, %{rows: rows}} -> {:ok, Enum.map(rows, &hd/1)} + {:error, %{mysql: %{name: :ER_NO_SUCH_TABLE}}} -> {:ok, []} {:error, _} = error -> error - versions -> {:ok, versions} + {:exit, exit} -> {:error, exit_to_exception(exit)} end end - defp mysql_dump(prefixes, config) do - args = ["--no-data", "--routines", "--databases" | prefixes] + defp mysql_dump(database, config) do + args = ["--no-data", "--routines", "--no-create-db", database] case run_with_cmd("mysqldump", config, args) do {output, 0} -> {:ok, output} @@ -358,29 +382,28 @@ defmodule Ecto.Adapters.MyXQL do end defp append_versions(table, versions, contents) do - sql_statements = - Enum.map_join(versions, fn {prefix, version} -> - ~s[INSERT INTO `#{prefix}`.`#{table}` (version) VALUES (#{version});\n] - end) - - {:ok, contents <> sql_statements} + {:ok, + contents <> + Enum.map_join(versions, &~s[INSERT INTO `#{table}` (version) VALUES (#{&1});\n])} end @impl true def structure_load(default, config) do path = config[:dump_path] || Path.join(default, "structure.sql") - args = ["--execute", "SET FOREIGN_KEY_CHECKS = 0; SOURCE #{path}; SET FOREIGN_KEY_CHECKS = 1"] + args = ["--execute", "SET FOREIGN_KEY_CHECKS = 0; SOURCE #{path}; SET FOREIGN_KEY_CHECKS = 1", "--database", config[:database]] case run_with_cmd("mysql", config, args) do {_output, 0} -> {:ok, path} - {output, _} -> {:error, output} + {output, _} -> {:error, output} end end @impl true - def dump_cmd(args, opts \\ [], config) when is_list(config) and is_list(args), - do: run_with_cmd("mysqldump", config, args, opts) + def dump_cmd(args, opts \\ [], config) when is_list(config) and is_list(args) do + args = args ++ [config[:database]] + run_with_cmd("mysqldump", config, args, opts) + end ## Helpers @@ -394,23 +417,27 @@ defmodule Ecto.Adapters.MyXQL do |> Keyword.put(:backoff_type, :stop) |> Keyword.put(:max_restarts, 0) - task = Task.Supervisor.async_nolink(Ecto.Adapters.SQL.StorageSupervisor, fn -> - {:ok, conn} = MyXQL.start_link(opts) + task = + Task.Supervisor.async_nolink(Ecto.Adapters.SQL.StorageSupervisor, fn -> + {:ok, conn} = MyXQL.start_link(opts) - value = MyXQL.query(conn, sql, [], opts) - GenServer.stop(conn) - value - end) + value = MyXQL.query(conn, sql, [], opts) + GenServer.stop(conn) + value + end) timeout = Keyword.get(opts, :timeout, 15_000) case Task.yield(task, timeout) || Task.shutdown(task) do {:ok, {:ok, result}} -> {:ok, result} + {:ok, {:error, error}} -> {:error, error} + {:exit, exit} -> {:exit, exit} + nil -> {:error, RuntimeError.exception("command timed out")} end @@ -425,7 +452,7 @@ defmodule Ecto.Adapters.MyXQL do defp run_with_cmd(cmd, opts, opt_args, cmd_opts \\ []) do unless System.find_executable(cmd) do raise "could not find executable `#{cmd}` in path, " <> - "please guarantee it is available before running ecto commands" + "please guarantee it is available before running ecto commands" end env = @@ -435,8 +462,8 @@ defmodule Ecto.Adapters.MyXQL do [] end - host = opts[:hostname] || System.get_env("MYSQL_HOST") || "localhost" - port = opts[:port] || System.get_env("MYSQL_TCP_PORT") || "3306" + host = opts[:hostname] || System.get_env("MYSQL_HOST") || "localhost" + port = opts[:port] || System.get_env("MYSQL_TCP_PORT") || "3306" protocol = opts[:cli_protocol] || System.get_env("MYSQL_CLI_PROTOCOL") || "tcp" user_args = @@ -446,30 +473,21 @@ defmodule Ecto.Adapters.MyXQL do [] end - database_args = - if database = opts[:database] do - if cmd == "mysqldump" do - ["--databases", database] - else - ["--database", database] - end - else - [] - end - args = [ - "--host", host, - "--port", to_string(port), - "--protocol", protocol - ] ++ user_args ++ database_args ++ opt_args + "--host", + host, + "--port", + to_string(port), + "--protocol", + protocol + ] ++ user_args ++ opt_args cmd_opts = cmd_opts |> Keyword.put_new(:stderr_to_stdout, true) |> Keyword.update(:env, env, &Enum.concat(env, &1)) - System.cmd(cmd, args, cmd_opts) end end diff --git a/lib/mix/tasks/ecto.dump.ex b/lib/mix/tasks/ecto.dump.ex index 62cdb4fc..aab99f7b 100644 --- a/lib/mix/tasks/ecto.dump.ex +++ b/lib/mix/tasks/ecto.dump.ex @@ -48,11 +48,11 @@ defmodule Mix.Tasks.Ecto.Dump do * `--no-compile` - does not compile applications before dumping * `--no-deps-check` - does not check dependencies before dumping * `--prefix` - prefix that will be included in the structure dump. - Can include multiple prefixes (ex. `--prefix foo --prefix bar`). - When specified, the prefixes will have their definitions dumped along - with the data in their migration table. The default behavior is - dependent on the adapter for backwards compatibility reasons. - For PostgreSQL, the configured database has the definitions dumped + Can include multiple prefixes (ex. `--prefix foo --prefix bar`) with + PostgreSQL but not MySQL. When specified, the prefixes will have + their definitions dumped along with the data in their migration table. + The default behavior is dependent on the adapter for backwards compatibility + reasons. For PostgreSQL, the configured database has the definitions dumped from all of its schemas but only the data from the migration table from the `public` schema is included. For MySQL, only the configured database and its migration table are dumped. @@ -97,7 +97,7 @@ defmodule Mix.Tasks.Ecto.Dump do end end end - + defp format_time(microsec) when microsec < 1_000, do: "#{microsec} μs" defp format_time(microsec) when microsec < 1_000_000, do: "#{div(microsec, 1_000)} ms" defp format_time(microsec), do: "#{Float.round(microsec / 1_000_000.0)} s"