Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix mysqldump generating use database #541

Merged
merged 5 commits into from
Jul 21, 2023
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 13 additions & 26 deletions integration_test/myxql/storage_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -146,57 +146,44 @@ defmodule Ecto.Integration.StorageTest do
end

test "structure dump and load with migrations table" do
default_db = "ecto_test"
num = @base_migration + System.unique_integer([:positive])
:ok = Ecto.Migrator.up(PoolRepo, num, Migration, log: false)
{:ok, path} = Ecto.Adapters.MyXQL.structure_dump(tmp_path(), TestRepo.config())
contents = File.read!(path)
assert contents =~ "INSERT INTO `ecto_test`.`schema_migrations` (version) VALUES (#{num})"
assert contents =~ "Database: #{default_db}"
assert contents =~ "INSERT INTO `schema_migrations` (version) VALUES (#{num})"
end

test "dumps structure and schema_migration records from multiple prefixes" do
# Create the test_schema schema
test "raises when attempting to dump multiple prefixes" do
create_database()
prefix = params()[:database]

# Run migrations
version = @base_migration + System.unique_integer([:positive])
:ok = Ecto.Migrator.up(PoolRepo, version, Migration, log: false)
:ok = Ecto.Migrator.up(PoolRepo, version, Migration, log: false, prefix: prefix)

config = Keyword.put(TestRepo.config(), :dump_prefixes, ["ecto_test", prefix])
{:ok, path} = Ecto.Adapters.MyXQL.structure_dump(tmp_path(), config)
contents = File.read!(path)
msg = "cannot dump multiple prefixes with MySQL. Please run the command separately for each prefix."

assert contents =~ "Current Database: `#{prefix}`"
assert contents =~ "Current Database: `ecto_test`"
assert contents =~ "CREATE TABLE `schema_migrations`"
assert contents =~ ~s[INSERT INTO `#{prefix}`.`schema_migrations` (version) VALUES (#{version})]
assert contents =~ ~s[INSERT INTO `ecto_test`.`schema_migrations` (version) VALUES (#{version})]
assert_raise ArgumentError, msg, fn ->
Ecto.Adapters.MyXQL.structure_dump(tmp_path(), config)
end
after
drop_database()
end

test "dumps structure and schema_migration records only from queried prefix" do
# Create the test_schema schema
# Create the storage_mgt database
create_database()
prefix = params()[:database]

# Run migrations
version = @base_migration + System.unique_integer([:positive])
:ok = Ecto.Migrator.up(PoolRepo, version, Migration, log: false)
:ok = Ecto.Migrator.up(PoolRepo, version, Migration, log: false, prefix: prefix)

config = Keyword.put(TestRepo.config(), :dump_prefixes, ["ecto_test"])
config = Keyword.put(TestRepo.config(), :dump_prefixes, [prefix])
{:ok, path} = Ecto.Adapters.MyXQL.structure_dump(tmp_path(), config)
contents = File.read!(path)

refute contents =~ "Current Database: `#{prefix}`"
assert contents =~ "Current Database: `ecto_test`"
assert contents =~ "CREATE TABLE `schema_migrations`"
refute contents =~ ~s[INSERT INTO `#{prefix}`.`schema_migrations` (version) VALUES (#{version})]
assert contents =~ ~s[INSERT INTO `ecto_test`.`schema_migrations` (version) VALUES (#{version})]
after
drop_database()
refute contents =~ "USE `#{prefix}`"
assert contents =~ "Database: #{prefix}"
assert contents =~ "INSERT INTO `schema_migrations` (version) VALUES (#{version})"
end

defp strip_timestamp(dump) do
Expand Down
97 changes: 46 additions & 51 deletions lib/ecto/adapters/myxql.ex
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,11 @@ defmodule Ecto.Adapters.MyXQL do
* `:collation` - the collation order
* `:dump_path` - where to place dumped structures
* `:dump_prefixes` - list of prefixes that will be included in the
structure dump. When specified, the prefixes will have their definitions
dumped along with the data in their migration table. When it is not
specified, only the configured database and its migration table are dumped.
structure dump. For MySQL, this list must be of length 1. Multiple
prefixes are not supported. When specified, the prefixes will have
their definitions dumped along with the data in their migration table.
When it is not specified, only the configured database and its migration
table are dumped.

### After connect callback

Expand Down Expand Up @@ -147,12 +149,12 @@ defmodule Ecto.Adapters.MyXQL do
## Custom MySQL types

@impl true
def loaders({:map, _}, type), do: [&json_decode/1, &Ecto.Type.embedded_load(type, &1, :json)]
def loaders(:map, type), do: [&json_decode/1, type]
def loaders(:float, type), do: [&float_decode/1, type]
def loaders(:boolean, type), do: [&bool_decode/1, type]
def loaders(:binary_id, type), do: [Ecto.UUID, type]
def loaders(_, type), do: [type]
def loaders({:map, _}, type), do: [&json_decode/1, &Ecto.Type.embedded_load(type, &1, :json)]
def loaders(:map, type), do: [&json_decode/1, type]
def loaders(:float, type), do: [&float_decode/1, type]
def loaders(:boolean, type), do: [&bool_decode/1, type]
def loaders(:binary_id, type), do: [Ecto.UUID, type]
def loaders(_, type), do: [type]

defp bool_decode(<<0>>), do: {:ok, false}
defp bool_decode(<<1>>), do: {:ok, true}
Expand All @@ -173,10 +175,12 @@ defmodule Ecto.Adapters.MyXQL do
@impl true
def storage_up(opts) do
database = Keyword.fetch!(opts, :database) || raise ":database is nil in repository configuration"

opts = Keyword.delete(opts, :database)
charset = opts[:charset] || "utf8mb4"

check_existence_command = "SELECT TRUE FROM information_schema.schemata WHERE schema_name = '#{database}'"

case run_query(check_existence_command, opts) do
{:ok, %{num_rows: 1}} ->
{:error, :already_up}
Expand All @@ -198,12 +202,13 @@ defmodule Ecto.Adapters.MyXQL do
end
end

defp concat_if(content, nil, _fun), do: content
defp concat_if(content, nil, _fun), do: content
defp concat_if(content, value, fun), do: content <> " " <> fun.(value)

@impl true
def storage_down(opts) do
database = Keyword.fetch!(opts, :database) || raise ":database is nil in repository configuration"

opts = Keyword.delete(opts, :database)
command = "DROP DATABASE `#{database}`"

Expand All @@ -226,6 +231,7 @@ defmodule Ecto.Adapters.MyXQL do
@impl Ecto.Adapter.Storage
def storage_status(opts) do
database = Keyword.fetch!(opts, :database) || raise ":database is nil in repository configuration"

opts = Keyword.delete(opts, :database)

check_database_query = "SELECT schema_name FROM information_schema.schemata WHERE schema_name = '#{database}'"
Expand Down Expand Up @@ -275,6 +281,7 @@ defmodule Ecto.Adapters.MyXQL do
key = primary_key!(schema_meta, returning)
{fields, values} = :lists.unzip(params)
sql = @conn.insert(prefix, source, fields, [fields], on_conflict, [], [])

opts = if is_nil(Keyword.get(opts, :cache_statement)) do
[{:cache_statement, "ecto_insert_#{source}_#{length(fields)}"} | opts]
else
Expand Down Expand Up @@ -303,6 +310,7 @@ defmodule Ecto.Adapters.MyXQL do

defp primary_key!(%{autogenerate_id: {_, key, _type}}, [key]), do: key
defp primary_key!(_, []), do: nil

defp primary_key!(%{schema: schema}, returning) do
raise ArgumentError, "MySQL does not support :read_after_writes in schemas for non-primary keys. " <>
"The following fields in #{inspect schema} are tagged as such: #{inspect returning}"
Expand All @@ -315,37 +323,37 @@ defmodule Ecto.Adapters.MyXQL do
@impl true
def structure_dump(default, config) do
table = config[:migration_source] || "schema_migrations"
path = config[:dump_path] || Path.join(default, "structure.sql")
prefixes = config[:dump_prefixes] || [config[:database]]
path = config[:dump_path] || Path.join(default, "structure.sql")
database = dump_database!(config[:dump_prefixes], config[:database])

with {:ok, versions} <- select_versions(prefixes, table, config),
{:ok, contents} <- mysql_dump(prefixes, config),
with {:ok, versions} <- select_versions(database, table, config),
{:ok, contents} <- mysql_dump(database, config),
{:ok, contents} <- append_versions(table, versions, contents) do
File.mkdir_p!(Path.dirname(path))
File.write!(path, contents)
{:ok, path}
end
end

defp select_versions(prefixes, table, config) do
result =
Enum.reduce_while(prefixes, [], fn prefix, versions ->
case run_query(~s[SELECT version FROM `#{prefix}`.`#{table}` ORDER BY version], config) do
{:ok, %{rows: rows}} -> {:cont, Enum.map(rows, &{prefix, hd(&1)}) ++ versions}
{:error, %{mysql: %{name: :ER_NO_SUCH_TABLE}}} -> {:cont, versions}
{:error, _} = error -> {:halt, error}
{:exit, exit} -> {:halt, {:error, exit_to_exception(exit)}}
end
end)
defp dump_database!([prefix], _), do: prefix
defp dump_database!(nil, config_database), do: config_database

defp dump_database!(_, _) do
raise ArgumentError,
"cannot dump multiple prefixes with MySQL. Please run the command separately for each prefix."
end

case result do
defp select_versions(database, table, config) do
case run_query(~s[SELECT version FROM `#{database}`.`#{table}` ORDER BY version], config) do
{:ok, %{rows: rows}} -> {:ok, Enum.map(rows, &hd/1)}
{:error, %{mysql: %{name: :ER_NO_SUCH_TABLE}}} -> {:ok, []}
{:error, _} = error -> error
versions -> {:ok, versions}
{:exit, exit} -> {:error, exit_to_exception(exit)}
end
end

defp mysql_dump(prefixes, config) do
args = ["--no-data", "--routines", "--databases" | prefixes]
defp mysql_dump(database, config) do
args = ["--no-data", "--routines", "--no-create-db", database]

case run_with_cmd("mysqldump", config, args) do
{output, 0} -> {:ok, output}
Expand All @@ -358,29 +366,28 @@ defmodule Ecto.Adapters.MyXQL do
end

defp append_versions(table, versions, contents) do
sql_statements =
Enum.map_join(versions, fn {prefix, version} ->
~s[INSERT INTO `#{prefix}`.`#{table}` (version) VALUES (#{version});\n]
end)

{:ok, contents <> sql_statements}
{:ok,
contents <>
Enum.map_join(versions, &~s[INSERT INTO `#{table}` (version) VALUES (#{&1});\n])}
end

@impl true
def structure_load(default, config) do
path = config[:dump_path] || Path.join(default, "structure.sql")

args = ["--execute", "SET FOREIGN_KEY_CHECKS = 0; SOURCE #{path}; SET FOREIGN_KEY_CHECKS = 1"]
args = ["--execute", "SET FOREIGN_KEY_CHECKS = 0; SOURCE #{path}; SET FOREIGN_KEY_CHECKS = 1", "--database", config[:database]]

case run_with_cmd("mysql", config, args) do
{_output, 0} -> {:ok, path}
{output, _} -> {:error, output}
{output, _} -> {:error, output}
end
end

@impl true
def dump_cmd(args, opts \\ [], config) when is_list(config) and is_list(args),
do: run_with_cmd("mysqldump", config, args, opts)
def dump_cmd(args, opts \\ [], config) when is_list(config) and is_list(args) do
args = args ++ [config[:database]]
run_with_cmd("mysqldump", config, args, opts)
end

## Helpers

Expand Down Expand Up @@ -425,7 +432,7 @@ defmodule Ecto.Adapters.MyXQL do
defp run_with_cmd(cmd, opts, opt_args, cmd_opts \\ []) do
unless System.find_executable(cmd) do
raise "could not find executable `#{cmd}` in path, " <>
"please guarantee it is available before running ecto commands"
"please guarantee it is available before running ecto commands"
end

env =
Expand All @@ -446,17 +453,6 @@ defmodule Ecto.Adapters.MyXQL do
[]
end

database_args =
if database = opts[:database] do
if cmd == "mysqldump" do
["--databases", database]
else
["--database", database]
end
else
[]
end

args =
[
"--host", host,
Expand All @@ -469,7 +465,6 @@ defmodule Ecto.Adapters.MyXQL do
|> Keyword.put_new(:stderr_to_stdout, true)
|> Keyword.update(:env, env, &Enum.concat(env, &1))


System.cmd(cmd, args, cmd_opts)
end
end
12 changes: 6 additions & 6 deletions lib/mix/tasks/ecto.dump.ex
Original file line number Diff line number Diff line change
Expand Up @@ -48,11 +48,11 @@ defmodule Mix.Tasks.Ecto.Dump do
* `--no-compile` - does not compile applications before dumping
* `--no-deps-check` - does not check dependencies before dumping
* `--prefix` - prefix that will be included in the structure dump.
Can include multiple prefixes (ex. `--prefix foo --prefix bar`).
When specified, the prefixes will have their definitions dumped along
with the data in their migration table. The default behavior is
dependent on the adapter for backwards compatibility reasons.
For PostgreSQL, the configured database has the definitions dumped
Can include multiple prefixes (ex. `--prefix foo --prefix bar`) with
PostgreSQL but not MySQL. When specified, the prefixes will have
their definitions dumped along with the data in their migration table.
The default behavior is dependent on the adapter for backwards compatibility
reasons. For PostgreSQL, the configured database has the definitions dumped
from all of its schemas but only the data from the migration table
from the `public` schema is included. For MySQL, only the configured
database and its migration table are dumped.
Expand Down Expand Up @@ -97,7 +97,7 @@ defmodule Mix.Tasks.Ecto.Dump do
end
end
end

defp format_time(microsec) when microsec < 1_000, do: "#{microsec} μs"
defp format_time(microsec) when microsec < 1_000_000, do: "#{div(microsec, 1_000)} ms"
defp format_time(microsec), do: "#{Float.round(microsec / 1_000_000.0)} s"
Expand Down
Loading