Skip to content

Commit

Permalink
tests passing
Browse files Browse the repository at this point in the history 
Signed-off-by: Tim Vaillancourt <[email protected]>
  • Loading branch information
@timvaillancourt
timvaillancourt committed Oct 30, 2024
1 parent 02fcfe0 commit 03f9170
Show file tree
Hide file tree
Showing 6 changed files with 165 additions and 444 deletions.
85 changes: 1 addition & 84 deletions go/vt/external/golib/sqlutils/sqlite_dialect.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,70 +18,9 @@
This file has been copied over from VTOrc package
*/

// What's this about?
// This is a brute-force regular-expression based conversion from MySQL syntax to sqlite3 syntax.
// It is NOT meant to be a general purpose solution and is only expected & confirmed to run on
// queries issued by orchestrator. There are known limitations to this design.
// It's not even pretty.
// In fact...
// Well, it gets the job done at this time. Call it debt.

package sqlutils

import (
"regexp"
)

var sqlite3CreateTableConversions = []regexpMap{
rmap(`(?i) (character set|charset) [\S]+`, ``),
rmap(`(?i)int unsigned`, `int`),
rmap(`(?i)int[\s]*[(][\s]*([0-9]+)[\s]*[)] unsigned`, `int`),
rmap(`(?i)engine[\s]*=[\s]*(innodb|myisam|ndb|memory|tokudb)`, ``),
rmap(`(?i)DEFAULT CHARSET[\s]*=[\s]*[\S]+`, ``),
rmap(`(?i)[\S]*int( not null|) auto_increment`, `integer`),
rmap(`(?i)comment '[^']*'`, ``),
rmap(`(?i)after [\S]+`, ``),
rmap(`(?i)alter table ([\S]+) add (index|key) ([\S]+) (.+)`, `create index ${3}_${1} on $1 $4`),
rmap(`(?i)alter table ([\S]+) add unique (index|key) ([\S]+) (.+)`, `create unique index ${3}_${1} on $1 $4`),
rmap(`(?i)([\S]+) enum[\s]*([(].*?[)])`, `$1 text check($1 in $2)`),
rmap(`(?i)([\s\S]+[/][*] sqlite3-skip [*][/][\s\S]+)`, ``),
rmap(`(?i)timestamp default current_timestamp`, `timestamp default ('')`),
rmap(`(?i)timestamp not null default current_timestamp`, `timestamp not null default ('')`),

rmap(`(?i)add column (.*int) not null[\s]*$`, `add column $1 not null default 0`),
rmap(`(?i)add column (.* text) not null[\s]*$`, `add column $1 not null default ''`),
rmap(`(?i)add column (.* varchar.*) not null[\s]*$`, `add column $1 not null default ''`),
}

var sqlite3InsertConversions = []regexpMap{
rmap(`(?i)insert ignore ([\s\S]+) on duplicate key update [\s\S]+`, `insert or ignore $1`),
rmap(`(?i)insert ignore`, `insert or ignore`),
rmap(`(?i)now[(][)]`, `datetime('now')`),
rmap(`(?i)insert into ([\s\S]+) on duplicate key update [\s\S]+`, `replace into $1`),
}

var sqlite3GeneralConversions = []regexpMap{
rmap(`(?i)now[(][)][\s]*[-][\s]*interval [?] ([\w]+)`, `datetime('now', printf('-%d $1', ?))`),
rmap(`(?i)now[(][)][\s]*[+][\s]*interval [?] ([\w]+)`, `datetime('now', printf('+%d $1', ?))`),
rmap(`(?i)now[(][)][\s]*[-][\s]*interval ([0-9.]+) ([\w]+)`, `datetime('now', '-${1} $2')`),
rmap(`(?i)now[(][)][\s]*[+][\s]*interval ([0-9.]+) ([\w]+)`, `datetime('now', '+${1} $2')`),

rmap(`(?i)[=<>\s]([\S]+[.][\S]+)[\s]*[-][\s]*interval [?] ([\w]+)`, ` datetime($1, printf('-%d $2', ?))`),
rmap(`(?i)[=<>\s]([\S]+[.][\S]+)[\s]*[+][\s]*interval [?] ([\w]+)`, ` datetime($1, printf('+%d $2', ?))`),

rmap(`(?i)unix_timestamp[(][)]`, `strftime('%s', 'now')`),
rmap(`(?i)unix_timestamp[(]([^)]+)[)]`, `strftime('%s', $1)`),
rmap(`(?i)now[(][)]`, `datetime('now')`),
rmap(`(?i)cast[(][\s]*([\S]+) as signed[\s]*[)]`, `cast($1 as integer)`),

rmap(`(?i)\bconcat[(][\s]*([^,)]+)[\s]*,[\s]*([^,)]+)[\s]*[)]`, `($1 || $2)`),
rmap(`(?i)\bconcat[(][\s]*([^,)]+)[\s]*,[\s]*([^,)]+)[\s]*,[\s]*([^,)]+)[\s]*[)]`, `($1 || $2 || $3)`),

rmap(`(?i) rlike `, ` like `),

rmap(`(?i)create index([\s\S]+)[(][\s]*[0-9]+[\s]*[)]([\s\S]+)`, `create index ${1}${2}`),
rmap(`(?i)drop index ([\S]+) on ([\S]+)`, `drop index if exists $1`),
}
import "regexp"

var (
sqlite3IdentifyCreateTableStatement = regexp.MustCompile(regexpSpaces(`(?i)^[\s]*create table`))
Expand Down Expand Up @@ -110,25 +49,3 @@ func IsDropIndex(statement string) bool {
func IsAlterTable(statement string) bool {
return sqlite3IdentifyAlterTableStatement.MatchString(statement)
}

func ToSqlite3CreateTable(statement string) string {
return applyConversions(statement, sqlite3CreateTableConversions)
}

func ToSqlite3Insert(statement string) string {
return applyConversions(statement, sqlite3InsertConversions)
}

func ToSqlite3Dialect(statement string) (translated string) {
if IsCreateTable(statement) {
return ToSqlite3CreateTable(statement)
}
if IsAlterTable(statement) {
return ToSqlite3CreateTable(statement)
}
statement = applyConversions(statement, sqlite3GeneralConversions)
if IsInsert(statement) {
return ToSqlite3Insert(statement)
}
return statement
}
195 changes: 0 additions & 195 deletions go/vt/external/golib/sqlutils/sqlite_dialect_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,98 +51,6 @@ func TestIsCreateTable(t *testing.T) {
require.False(t, IsCreateTable("insert"))
}

func TestToSqlite3CreateTable(t *testing.T) {
{
statement := "create table t(id int)"
result := ToSqlite3CreateTable(statement)
require.Equal(t, result, statement)
}
{
statement := "create table t(id int, v varchar(123) CHARACTER SET ascii NOT NULL default '')"
result := ToSqlite3CreateTable(statement)
require.Equal(t, result, "create table t(id int, v varchar(123) NOT NULL default '')")
}
{
statement := "create table t(id int, v varchar ( 123 ) CHARACTER SET ascii NOT NULL default '')"
result := ToSqlite3CreateTable(statement)
require.Equal(t, result, "create table t(id int, v varchar ( 123 ) NOT NULL default '')")
}
{
statement := "create table t(i smallint unsigned)"
result := ToSqlite3CreateTable(statement)
require.Equal(t, result, "create table t(i smallint)")
}
{
statement := "create table t(i smallint(5) unsigned)"
result := ToSqlite3CreateTable(statement)
require.Equal(t, result, "create table t(i smallint)")
}
{
statement := "create table t(i smallint ( 5 ) unsigned)"
result := ToSqlite3CreateTable(statement)
require.Equal(t, result, "create table t(i smallint)")
}
}

func TestToSqlite3AlterTable(t *testing.T) {
{
statement := `
ALTER TABLE
database_instance
ADD COLUMN sql_delay INT UNSIGNED NOT NULL AFTER replica_lag_seconds
`
result := stripSpaces(ToSqlite3Dialect(statement))
require.Equal(t, result, stripSpaces(`
ALTER TABLE
database_instance
add column sql_delay int not null default 0
`))
}
{
statement := `
ALTER TABLE
database_instance
ADD INDEX source_host_port_idx (source_host, source_port)
`
result := stripSpaces(ToSqlite3Dialect(statement))
require.Equal(t, result, stripSpaces(`
create index
source_host_port_idx_database_instance
on database_instance (source_host, source_port)
`))
}
{
statement := `
ALTER TABLE
topology_recovery
ADD KEY last_detection_idx (last_detection_id)
`
result := stripSpaces(ToSqlite3Dialect(statement))
require.Equal(t, result, stripSpaces(`
create index
last_detection_idx_topology_recovery
on topology_recovery (last_detection_id)
`))
}

}

func TestCreateIndex(t *testing.T) {
{
statement := `
create index
source_host_port_idx_database_instance
on database_instance (source_host(128), source_port)
`
result := stripSpaces(ToSqlite3Dialect(statement))
require.Equal(t, result, stripSpaces(`
create index
source_host_port_idx_database_instance
on database_instance (source_host, source_port)
`))
}
}

func TestIsInsert(t *testing.T) {
require.True(t, IsInsert("insert into t"))
require.True(t, IsInsert("insert ignore into t"))
Expand All @@ -162,90 +70,6 @@ func TestIsInsert(t *testing.T) {
`))
}

func TestToSqlite3Insert(t *testing.T) {
{
statement := `
insert into
cluster_domain_name (cluster_name, domain_name, last_registered)
values
(?, ?, datetime('now'))
on duplicate key update
domain_name=values(domain_name),
last_registered=values(last_registered)
`
result := stripSpaces(ToSqlite3Dialect(statement))
require.Equal(t, result, stripSpaces(`
replace into
cluster_domain_name (cluster_name, domain_name, last_registered)
values
(?, ?, datetime('now'))
`))
}
}

func TestToSqlite3GeneralConversions(t *testing.T) {
{
statement := "select now()"
result := ToSqlite3Dialect(statement)
require.Equal(t, result, "select datetime('now')")
}
{
statement := "select now() - interval ? second"
result := ToSqlite3Dialect(statement)
require.Equal(t, result, "select datetime('now', printf('-%d second', ?))")
}
{
statement := "select now() + interval ? minute"
result := ToSqlite3Dialect(statement)
require.Equal(t, result, "select datetime('now', printf('+%d minute', ?))")
}
{
statement := "select now() + interval 5 minute"
result := ToSqlite3Dialect(statement)
require.Equal(t, result, "select datetime('now', '+5 minute')")
}
{
statement := "select some_table.some_column + interval ? minute"
result := ToSqlite3Dialect(statement)
require.Equal(t, result, "select datetime(some_table.some_column, printf('+%d minute', ?))")
}
{
statement := "AND primary_instance.last_attempted_check <= primary_instance.last_seen + interval ? minute"
result := ToSqlite3Dialect(statement)
require.Equal(t, result, "AND primary_instance.last_attempted_check <= datetime(primary_instance.last_seen, printf('+%d minute', ?))")
}
{
statement := "select concat(primary_instance.port, '') as port"
result := ToSqlite3Dialect(statement)
require.Equal(t, result, "select (primary_instance.port || '') as port")
}
{
statement := "select concat( 'abc' , 'def') as s"
result := ToSqlite3Dialect(statement)
require.Equal(t, result, "select ('abc' || 'def') as s")
}
{
statement := "select concat( 'abc' , 'def', last.col) as s"
result := ToSqlite3Dialect(statement)
require.Equal(t, result, "select ('abc' || 'def' || last.col) as s")
}
{
statement := "select concat(myself.only) as s"
result := ToSqlite3Dialect(statement)
require.Equal(t, result, "select concat(myself.only) as s")
}
{
statement := "select concat(1, '2', 3, '4') as s"
result := ToSqlite3Dialect(statement)
require.Equal(t, result, "select concat(1, '2', 3, '4') as s")
}
{
statement := "select group_concat( 'abc' , 'def') as s"
result := ToSqlite3Dialect(statement)
require.Equal(t, result, "select group_concat( 'abc' , 'def') as s")
}
}

func TestIsCreateIndex(t *testing.T) {
tests := []struct {
input string
Expand Down Expand Up @@ -293,22 +117,3 @@ func TestIsDropIndex(t *testing.T) {
})
}
}

func TestToSqlite3Dialect(t *testing.T) {
tests := []struct {
input string
expected string
}{
{"create table my_table(id int);", "create table my_table(id int);"},
{"alter table my_table add column new_col int;", "alter table my_table add column new_col int;"},
{"insert into my_table values (1);", "insert into my_table values (1);"},
{"", ""},
}

for _, test := range tests {
t.Run(test.input, func(t *testing.T) {
result := ToSqlite3Dialect(test.input)
assert.Equal(t, test.expected, result)
})
}
}
Loading

0 comments on commit 03f9170

Please sign in to comment.