From 441cb074c16941b32606ae3ac3d140b258de16cb Mon Sep 17 00:00:00 2001 From: Manuel Rubio Date: Fri, 23 Feb 2018 07:17:55 +0100 Subject: [PATCH 1/3] support for Elixir --- README.md | 125 ++++++++++++++++++++++++++++++++++++++++--- lib/dbi.ex | 33 ++++++++++++ mix.exs | 67 +++++++++++++++++++++++ mix.lock | 5 ++ test/dbi_test.exs | 57 ++++++++++++++++++++ test/test_helper.exs | 1 + 6 files changed, 282 insertions(+), 6 deletions(-) create mode 100644 lib/dbi.ex create mode 100644 mix.exs create mode 100644 mix.lock create mode 100644 test/dbi_test.exs create mode 100644 test/test_helper.exs diff --git a/README.md b/README.md index 3badeb3..e09e46e 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ DBI for Erlang [![License: LGPL 2.1](https://img.shields.io/badge/License-GNU%20Lesser%20General%20Public%20License%20v2.1-blue.svg)](https://raw.githubusercontent.com/altenwald/dbi/master/COPYING) [![Hex](https://img.shields.io/hexpm/v/dbi.svg)](https://hex.pm/packages/dbi) -Database Interface for Erlang. This is an abstract implementation to use the most common database libraries ([p1_mysql][1], [epgsql][2] and [esqlite][4], and others you want) to use with standard SQL in your programs and don't worry about if you need to change between the main databases in the market. +Database Interface for Erlang and Elixir. This is an abstract implementation to use the most common database libraries ([p1_mysql][1], [epgsql][2] and [esqlite][4], and others you want) to use with standard SQL in your programs and don't worry about if you need to change between the main databases in the market. ### Install (rebar3) @@ -14,10 +14,18 @@ To use it, with rebar, you only need to add the dependency to the rebar.config f ```erlang {deps, [ - {dbi, "0.1.0"} + {dbi, "0.2.0"} ]} ``` +### Install (mix) + +To use it, with mix, you only need to add the dependency to the mix.exs file: + +```elixir +{:dbi, "~> 0.2.0"} +``` + ### Configuration The configuration is made in the configuration file (`sys.config` or `app.config`) so, you can add a new block for config the database connection as follow: @@ -49,21 +57,59 @@ The configuration is made in the configuration file (`sys.config` or `app.config The available types in this moment are: `mysql`, `pgsql` and `sqlite`. +In case you're using Elixir, you can define the configuration for your project in this way: + +```elixir +confg :dbi, mydatabase: [ + type: :mysql, + host: 'localhost', + user: 'root', + pass: 'root', + database: 'mydatabase', + poolsize: 10 + ], + mylocaldb: [ + type: :sqlite, + database: ':memory' + ], + mystrongdb: [ + type: :pgsql, + host: 'localhost', + user: 'root', + pass: 'root', + database: 'mystrongdb', + poolsize: 100 + ] +``` + ### Using DBI -To do a query: +To do a query (Erlang): ```erlang {ok, Count, Rows} = dbi:do_query(mydatabase, "SELECT * FROM users", []), ``` -Or with params: +Elixir: + +```elixir +{:ok, count, rows} = DBI.do_query(:mydatabase, "SELECT * FROM users", []) +``` + +Or with params (Erlang): ```erlang {ok, Count, Rows} = dbi:do_query(mydatabase, "SELECT * FROM users WHERE id = $1", [12]), ``` +Elixir: + +```elixir +{:ok, count, rows} = DBI.do_query(:mydatabase, + "SELECT * FROM users WHERE id = $1", [12]) +``` + Rows has the format: `[{field1, field2, ..., fieldN}, ...]` **IMPORTANT** the use of $1..$100 in the query is extracted from pgsql, in mysql and sqlite is converted to the `?` syntax so, if you write this query: @@ -73,11 +119,18 @@ Rows has the format: `[{field1, field2, ..., fieldN}, ...]` "UPDATE users SET name = $2 WHERE id = $1", [12, "Mike"]), ``` +Elixir: + +```elixir +{:ok, count, rows} = DBI.do_query(:mydatabase, + "UPDATE users SET name = $2 WHERE id = $1", [12, "Mike"]) +``` + That should works well in pgsql, but **NOT for mysql and NOT for sqlite**. For avoid this situations, the best to do is **always keep the order of the params**. ### Delayed or Queued queries -If you want to create a connection to send only commands like INSERT, UPDATE or DELETE but without saturate the database (and run out database connections in the pool) you can use `dbi_delayed`: +If you want to create a connection to send only commands like INSERT, UPDATE or DELETE but without saturate the database (and run out database connections in the pool) you can use `dbi_delayed` (Erlang): ```erlang {ok, PID} = dbi_delayed:start_link(delay_myconn, myconn), @@ -85,7 +138,15 @@ dbi_delayed:do_query(delay_myconn, "INSERT INTO my tab VALUES ($1, $2)", [N1, N2]), ``` -This use only one connection from the pool `myconn`, when the query ends then `dbi_delayed` gets another query to run from the queue. You get statistics about the progress and the queue size: +Elixir: + +```elixir +{:ok, pid} = DBI.Delayed.start_link(:delay_myconn, :myconn) +DBI.Delayed.do_query(:delay_myconn, + "INSERT INTO my tab VALUES ($1, $2)", [n1, n2]) +``` + +This use only one connection from the pool `myconn`, when the query ends then `dbi_delayed` gets another query to run from the queue. You get statistics about the progress and the queue size (Erlang): ```erlang dbi_delayed:stats(delay_myconn). @@ -96,6 +157,17 @@ dbi_delayed:stats(delay_myconn). ] ``` +Elixir: + +```elixir +DBI.Delayed.stats(:delay_myconn) +[ + size: 0, + query_error: 0, + query_ok: 1 +] +``` + The delayed can be added to the configuration: ```erlang @@ -112,6 +184,20 @@ The delayed can be added to the configuration: ]} ``` +Elixir: + +```elixir +config :dbi, mydatabase: [ + type: :mysql, + host: 'localhost', + user: 'root', + pass: 'root', + database: 'mydatabase', + poolsize: 10, + delayed: :delay_myconn + ] +``` + ### Cache queries Another thing you can do is use a cache for SQL queries. The cache store the SQL as `key` and the result as `value` and keep the values for the time you specify in the configuration file: @@ -130,6 +216,20 @@ Another thing you can do is use a cache for SQL queries. The cache store the SQL ]} ``` +Elixir: + +```elixir +config :dbi, mydatabase: [ + type: :mysql, + host: 'localhost', + user: 'root', + pass: 'root', + database: 'mydatabase', + poolsize: 10, + cache: 5 + ] +``` + The cache param is in seconds. The ideal time to keep the cache values depends on the size of your tables, the data to store in the cache and how frequent are the changes in that data. For avoid flood and other issues due to fast queries or a lot of queries in little time you can use 5 or 10 seconds. To store the information about constants or other data without frequent changes you can use 3600 (one hour) or more time. To use the cache you should to use the following function from `dbi_cache`: @@ -138,6 +238,12 @@ To use the cache you should to use the following function from `dbi_cache`: dbi_cache:do_query(mydatabase, "SELECT items FROM table"), ``` +Elixir: + +```elixir +DBI.Cache.do_query(:mydatabase, "SELECT items FROM table") +``` + You can use `do_query/2` or `do_query/3` if you want to use params. And if you want to use a specific TTL (time-to-live) for your query, you can use `do_query/4`: ```erlang @@ -145,6 +251,13 @@ dbi_cache:do_query(mydatabase, "SELECT items FROM table", [], 3600), ``` +Elixir: + +```elixir +DBI.Cache.do_query(:mydatabase, + "SELECT items FROM table", [], 3600) +``` + Enjoy! [1]: https://github.com/processone/p1_mysql diff --git a/lib/dbi.ex b/lib/dbi.ex new file mode 100644 index 0000000..2d2ff3c --- /dev/null +++ b/lib/dbi.ex @@ -0,0 +1,33 @@ +defmodule DBI do + use Application + + # wrap from dbi_app.erl + def start(type, args), do: :dbi_app.start(type, args) + def stop(modules), do: :dbi_app.stop(modules) + + # wrap from dbi.erl + def start(), do: :dbi.start() + def do_query(pool, query), do: :dbi.do_query(pool, query) + def do_query(pool, query, args), do: :dbi.do_query(pool, query, args) + def connect(type, host, port, user, pass, database, poolname) do + :dbi.connect(type, host, port, user, pass, database, poolname) + end + def connect(type, host, port, user, pass, db, poolname, poolsize, extra) do + :dbi.connect(type, host, port, user, pass, db, poolname, poolsize, extra) + end + + defmodule Cache do + def do_query(ref, query), do: :dbi_cache.do_query(ref, query) + def do_query(ref, query, args), do: :dbi_cache.do_query(ref, query, args) + def do_query(ref, query, args, ttl) do + :dbi_cache.do_query(ref, query, args, ttl) + end + end + + defmodule Delayed do + def start_link(ref, conn), do: :dbi_delayed.start_link(ref, conn) + def do_query(ref, query, args), do: :dbi_delayed.do_query(ref, query, args) + def do_query(ref, query), do: :dbi_delayed.do_query(ref, query) + def stats(ref), do: :dbi_delayed.stats(ref) + end +end diff --git a/mix.exs b/mix.exs new file mode 100644 index 0000000..06839f4 --- /dev/null +++ b/mix.exs @@ -0,0 +1,67 @@ +defmodule DBI.Mixfile do + use Mix.Project + + def project do + [app: :dbi, + version: get_version(), + name: "DBI", + description: "DataBase Interface for Erlang", + package: package(), + source_url: "https://github.com/altenwald/dbi", + elixir: "~> 1.3", + compilers: Mix.compilers, + build_embedded: Mix.env == :prod, + start_permanent: Mix.env == :prod, + deps: deps()] + end + + def application do + env = if Mix.env == :test do + [testdb1: [type: :sqlite, + database: ':memory:'], + testdb2: [type: :sqlite, + database: ':memory:', + cache: 3], + testdb3: [type: :sqlite, + database: ':memory:', + delayed: :mydelayed] + ] + else + [] + end + [applications: [:crypto, :public_key, :asn1, :ssl], + mod: {DBI, []}, + env: env] + end + + defp deps do + [{:epgsql, "~> 3.4.0"}, + {:p1_mysql, "~> 1.0.4"}, + {:esqlite, "~> 0.2.3"}, + {:cache, "~> 2.2.0"}, + {:poolboy, "~> 1.5.1"}] + end + + defp package do + [files: ["lib", "mix.exs", "README*", "LICENSE*"], + maintainers: ["Manuel Rubio"], + licenses: ["LGPL 2.1"], + links: %{"GitHub" => "https://github.com/altenwald/dbi"}] + end + + defp get_version do + retrieve_version_from_git() + |> String.split("-") + |> case do + [tag] -> tag + [tag, _num_commits, commit] -> "#{tag}-#{commit}" + end + end + + defp retrieve_version_from_git do + System.cmd("git", ["describe", "--always", "--tags"]) + |> Tuple.to_list + |> List.first + |> String.strip + end +end diff --git a/mix.lock b/mix.lock new file mode 100644 index 0000000..823701d --- /dev/null +++ b/mix.lock @@ -0,0 +1,5 @@ +%{"cache": {:hex, :cache, "2.2.0", "3c11dbf4cd8fcd5787c95a5fb2a04038e3729cfca0386016eea8c953ab48a5ab", [:rebar3], [], "hexpm"}, + "epgsql": {:hex, :epgsql, "3.4.0", "39d473b83d74329a88f8fb1f99ad24c7a2329fd753957c488808cb7c0fc8164e", [:rebar3], [], "hexpm"}, + "esqlite": {:hex, :esqlite, "0.2.3", "1a8b60877fdd3d50a8a84b342db04032c0231cc27ecff4ddd0d934485d4c0cd5", [:rebar3], [], "hexpm"}, + "p1_mysql": {:hex, :p1_mysql, "1.0.4", "7b9d7957a9d031813a0e6bcea5a7f5e91b54db805a92709a445cf75cf934bc1d", [:rebar3], [], "hexpm"}, + "poolboy": {:hex, :poolboy, "1.5.1", "6b46163901cfd0a1b43d692657ed9d7e599853b3b21b95ae5ae0a777cf9b6ca8", [:rebar], [], "hexpm"}} diff --git a/test/dbi_test.exs b/test/dbi_test.exs new file mode 100644 index 0000000..d0396d1 --- /dev/null +++ b/test/dbi_test.exs @@ -0,0 +1,57 @@ +defmodule SkirnirImapParserTest do + use ExUnit.Case + + test "basic" do + assert :ok == DBI.start() + assert {:ok, 0, []} == DBI.do_query(:testdb1, + "CREATE TABLE testing ( id int primary key );") + assert {:ok, 1, []} == DBI.do_query(:testdb1, + "INSERT INTO testing(id) VALUES (1)") + assert {:ok, 4, []} == DBI.do_query(:testdb1, + "INSERT INTO testing(id) VALUES (2),(3),(4),(5)") + {:ok, 5, _} = DBI.do_query(:testdb1, "SELECT * FROM testing") + :ok = Application.stop(:dbi) + end + + test "delayed" do + assert :ok == DBI.start() + assert :ok == DBI.Delayed.do_query(:mydelayed, + "CREATE TABLE testing ( id int primary key );") + assert :ok == DBI.Delayed.do_query(:mydelayed, + "INSERT INTO testing(id) VALUES (1)") + assert :ok == DBI.Delayed.do_query(:mydelayed, + "INSERT INTO testing(id) VALUES (2),(3),(4),(5)") + assert {:error, {:sqlite_error, 'no such table: testing'}} == + DBI.do_query(:testdb3, "SELECT * FROM testing") + :timer.sleep(:timer.seconds(1)) + {:ok, 5, _} = DBI.do_query(:testdb3, "SELECT * FROM testing") + :ok = Application.stop(:dbi) + end + + test "cache" do + assert :ok == DBI.start() + assert {:ok, 0, []} == DBI.do_query(:testdb2, + "CREATE TABLE testing ( id int primary key );") + assert {:ok, 1, []} == DBI.do_query(:testdb2, + "INSERT INTO testing(id) VALUES (1)") + assert {:ok, 1, [{1}]} == DBI.Cache.do_query(:testdb2, "SELECT * FROM testing") + DBI.do_query(:testdb2, "UPDATE testing SET id = 2") + assert {:ok, 1, [{1}]} == DBI.Cache.do_query(:testdb2, "SELECT * FROM testing") + :timer.sleep(:timer.seconds(3)) + {:ok, 1, [{2}]} == DBI.Cache.do_query(:testdb2, "SELECT * FROM testing") + :ok = Application.stop(:dbi) + end + + test "args" do + assert :ok == DBI.start() + assert {:ok, 0, []} = DBI.do_query(:testdb1, + "CREATE TABLE testing ( id int primary key, comment varchar(100) );") + Enum.each(1..10, fn(i) -> + assert {:ok, 1, []} == DBI.do_query(:testdb1, + "INSERT INTO testing(id, comment) VALUES ($1, $2)", + [i, "Data that requires sanitization! ' --"]) + end) + {:ok, 10, [{_,_}|_]} = DBI.do_query(:testdb1, "SELECT * FROM testing") + :ok = Application.stop(:dbi) + end +end diff --git a/test/test_helper.exs b/test/test_helper.exs new file mode 100644 index 0000000..869559e --- /dev/null +++ b/test/test_helper.exs @@ -0,0 +1 @@ +ExUnit.start() From a5e7fda79e4b42d8fac8332400e5d53e9a53b65c Mon Sep 17 00:00:00 2001 From: Manuel Rubio Date: Fri, 23 Feb 2018 07:18:25 +0100 Subject: [PATCH 2/3] remove unused `run` function from connect --- src/dbi.erl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/dbi.erl b/src/dbi.erl index 7fa41c7..dfbfd8b 100644 --- a/src/dbi.erl +++ b/src/dbi.erl @@ -67,8 +67,7 @@ connect(Type, Host, Port, User, Pass, Database, Poolname, Poolsize, Extra) -> {database, Database}, {poolsize, Poolsize} | Extra ], application:set_env(dbi, Poolname, DBConf), - Module:init(Host, Port, User, Pass, Database, Poolname, Poolsize, Extra), - Module:run(). + Module:init(Host, Port, User, Pass, Database, Poolname, Poolsize, Extra). -spec connect( Type::atom(), Host :: string(), Port :: integer(), User :: string(), From c3952cb1ab6942194de8fa40916b36b4cbceca92 Mon Sep 17 00:00:00 2001 From: Manuel Rubio Date: Fri, 23 Feb 2018 07:18:48 +0100 Subject: [PATCH 3/3] change dbi_utils name to dbi_query --- src/dbi_mysql.erl | 6 +++--- src/dbi_pgsql.erl | 4 ++-- src/{dbi_utils.erl => dbi_query.erl} | 2 +- src/dbi_sqlite.erl | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) rename src/{dbi_utils.erl => dbi_query.erl} (98%) diff --git a/src/dbi_mysql.erl b/src/dbi_mysql.erl index a3b7512..551ffbb 100644 --- a/src/dbi_mysql.erl +++ b/src/dbi_mysql.erl @@ -29,10 +29,10 @@ start_link(ConnData) -> init(Host, Port, User, Pass, Database, Poolname, Poolsize, Extra) -> application:start(p1_mysql), MaxOverflow = proplists:get_value(max_overflow, Extra, ?DEFAULT_MAX_OVERFLOW), - ConnData = [Host, dbi_utils:default(Port, ?DEFAULT_PORT), + ConnData = [Host, dbi_query:default(Port, ?DEFAULT_PORT), User, Pass, Database, undefined], PoolArgs = [{name, {local, Poolname}}, {worker_module, ?MODULE}, - {size, dbi_utils:default(Poolsize, ?DEFAULT_POOLSIZE)}, + {size, dbi_query:default(Poolsize, ?DEFAULT_POOLSIZE)}, {max_overflow, MaxOverflow}], ChildSpec = poolboy:child_spec(Poolname, PoolArgs, ConnData), supervisor:start_child(?DBI_SUP, ChildSpec), @@ -53,7 +53,7 @@ do_query(PoolDB, SQL, Params) when is_list(SQL) -> do_query(PoolDB, list_to_binary(SQL), Params); do_query(PoolDB, RawSQL, Params) when is_binary(RawSQL) -> - SQL = dbi_utils:resolve(RawSQL), + SQL = dbi_query:resolve(RawSQL), poolboy:transaction(PoolDB, fun(PID) -> case p1_mysql_conn:squery(PID, SQL, self(), Params) of {data, Result} -> diff --git a/src/dbi_pgsql.erl b/src/dbi_pgsql.erl index a2d884d..2f2d8f9 100644 --- a/src/dbi_pgsql.erl +++ b/src/dbi_pgsql.erl @@ -31,9 +31,9 @@ init(Host, Port, User, Pass, Database, Poolname, Poolsize, Extra) -> MaxOverflow = proplists:get_value(max_overflow, Extra, ?DEFAULT_MAX_OVERFLOW), DataConn = [Host, User, Pass, [{database, Database}, - {port, dbi_utils:default(Port, ?DEFAULT_PORT)}] ++ Extra], + {port, dbi_query:default(Port, ?DEFAULT_PORT)}] ++ Extra], PoolArgs = [{name, {local, Poolname}}, {worker_module, ?MODULE}, - {size, dbi_utils:default(Poolsize, ?DEFAULT_POOLSIZE)}, + {size, dbi_query:default(Poolsize, ?DEFAULT_POOLSIZE)}, {max_overflow, MaxOverflow}], ChildSpec = poolboy:child_spec(Poolname, PoolArgs, DataConn), supervisor:start_child(?DBI_SUP, ChildSpec), diff --git a/src/dbi_utils.erl b/src/dbi_query.erl similarity index 98% rename from src/dbi_utils.erl rename to src/dbi_query.erl index 54e7a86..5b12c0e 100644 --- a/src/dbi_utils.erl +++ b/src/dbi_query.erl @@ -1,4 +1,4 @@ --module(dbi_utils). +-module(dbi_query). -author('manuel@altenwald.com'). -export([ diff --git a/src/dbi_sqlite.erl b/src/dbi_sqlite.erl index 03b7237..4816930 100644 --- a/src/dbi_sqlite.erl +++ b/src/dbi_sqlite.erl @@ -35,9 +35,9 @@ do_query(PoolDB, SQL, Params) when is_list(SQL) -> do_query(PoolDB, list_to_binary(SQL), Params); do_query(PoolDB, RawSQL, Params) when is_binary(RawSQL) -> - SQL = dbi_utils:resolve(RawSQL), + SQL = dbi_query:resolve(RawSQL), {ok, Conn} = dbi_sqlite_server:get_database(PoolDB), - case dbi_utils:sql_type(SQL) of + case dbi_query:sql_type(SQL) of dql -> case catch esqlite3:q(SQL, Params, Conn) of Rows when is_list(Rows) -> {ok, length(Rows), Rows};