diff --git a/nix/tools/devTools.nix b/nix/tools/devTools.nix index 114e5e4665..6185865843 100644 --- a/nix/tools/devTools.nix +++ b/nix/tools/devTools.nix @@ -77,6 +77,7 @@ let ${tests}/bin/postgrest-test-doctests ${tests}/bin/postgrest-test-io ${tests}/bin/postgrest-test-big-schema + ${tests}/bin/postgrest-test-replica ${style}/bin/postgrest-lint ${style}/bin/postgrest-style-check ''; diff --git a/nix/tools/tests.nix b/nix/tools/tests.nix index 77010f88d9..c50a388302 100644 --- a/nix/tools/tests.nix +++ b/nix/tools/tests.nix @@ -84,7 +84,7 @@ let '' ${cabal-install}/bin/cabal v2-build ${devCabalOptions} ${cabal-install}/bin/cabal v2-exec -- ${withTools.withPg} -f test/io/fixtures.sql \ - ${ioTestPython}/bin/pytest --ignore=test/io/test_big_schema.py -v test/io "''${_arg_leftovers[@]}" + ${ioTestPython}/bin/pytest --ignore=test/io/test_big_schema.py --ignore=test/io/test_replica.py -v test/io "''${_arg_leftovers[@]}" ''; testBigSchema = @@ -102,6 +102,21 @@ let ${ioTestPython}/bin/pytest -v test/io/test_big_schema.py "''${_arg_leftovers[@]}" ''; + testReplica = + checkedShellScript + { + name = "postgrest-test-replica"; + docs = "Run a pytest-based IO test on a replica. Add -k to run tests that match a given expression."; + args = [ "ARG_LEFTOVERS([pytest arguments])" ]; + workingDir = "/"; + withEnv = postgrest.env; + } + '' + ${cabal-install}/bin/cabal v2-build ${devCabalOptions} + ${cabal-install}/bin/cabal v2-exec -- ${withTools.withPg} --replica -f test/io/replica.sql \ + ${ioTestPython}/bin/pytest -v test/io/test_replica.py "''${_arg_leftovers[@]}" + ''; + dumpSchema = checkedShellScript { @@ -150,12 +165,16 @@ let # collect all tests HPCTIXFILE="$tmpdir"/io.tix \ ${withTools.withPg} -f test/io/fixtures.sql \ - ${cabal-install}/bin/cabal v2-exec ${devCabalOptions} -- ${ioTestPython}/bin/pytest --ignore=test/io/test_big_schema.py -v test/io + ${cabal-install}/bin/cabal v2-exec ${devCabalOptions} -- ${ioTestPython}/bin/pytest --ignore=test/io/test_big_schema.py --ignore=test/io/test_replica.py -v test/io HPCTIXFILE="$tmpdir"/big_schema.tix \ ${withTools.withPg} -f test/io/big_schema.sql \ ${cabal-install}/bin/cabal v2-exec ${devCabalOptions} -- ${ioTestPython}/bin/pytest -v test/io/test_big_schema.py + HPCTIXFILE="$tmpdir"/replica.tix \ + ${withTools.withPg} --replica -f test/io/replica.sql \ + ${cabal-install}/bin/cabal v2-exec ${devCabalOptions} -- ${ioTestPython}/bin/pytest -v test/io/test_replica.py + HPCTIXFILE="$tmpdir"/spec.tix \ ${withTools.withPg} -f test/spec/fixtures/load.sql \ ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:spec @@ -164,7 +183,7 @@ let # collect all the tix files ${ghc}/bin/hpc sum --union --exclude=Paths_postgrest --output="$tmpdir"/tests.tix \ - "$tmpdir"/io*.tix "$tmpdir"/big_schema*.tix "$tmpdir"/spec.tix + "$tmpdir"/io*.tix "$tmpdir"/big_schema*.tix "$tmpdir"/replica*.tix "$tmpdir"/spec.tix # prepare the overlay ${ghc}/bin/hpc overlay --output="$tmpdir"/overlay.tix test/coverage.overlay @@ -220,6 +239,7 @@ buildToolbox testSpecIdempotence testIO testBigSchema + testReplica dumpSchema coverage coverageDraftOverlay; diff --git a/nix/tools/withTools.nix b/nix/tools/withTools.nix index 82781b7697..4cda526ce9 100644 --- a/nix/tools/withTools.nix +++ b/nix/tools/withTools.nix @@ -28,6 +28,7 @@ let "ARG_USE_ENV([PGRST_DB_SCHEMAS], [test], [Schema to expose])" "ARG_USE_ENV([PGTZ], [utc], [Timezone to use])" "ARG_USE_ENV([PGOPTIONS], [-c search_path=public,test], [PG options to use])" + "ARG_OPTIONAL_BOOLEAN([replica],, [Enable a replica for the database])" ]; positionalCompletion = "_command"; workingDir = "/"; @@ -61,6 +62,7 @@ let HBA_FILE="$tmpdir/pg_hba.conf" echo "local $PGDATABASE some_protected_user password" > "$HBA_FILE" echo "local $PGDATABASE all trust" >> "$HBA_FILE" + echo "local replication all trust" >> "$HBA_FILE" log "Initializing database cluster..." # We try to make the database cluster as independent as possible from the host @@ -74,19 +76,50 @@ let pg_ctl -l "$tmpdir/db.log" -w start -o "-F -c listen_addresses=\"\" -c hba_file=$HBA_FILE -k $PGHOST -c log_statement=\"all\" " \ >> "$setuplog" + log "Creating a minimally privileged $PGUSER connection role..." + createuser "$PGUSER" -U postgres --host="$tmpdir/socket" --no-createdb --no-inherit --no-superuser --no-createrole --no-replication --login + + >&2 echo "${commandName}: You can connect with: psql 'postgres:///$PGDATABASE?host=$PGHOST' -U postgres" + >&2 echo "${commandName}: You can tail the logs with: tail -f $tmpdir/db.log" + + if test "$_arg_replica" = "on"; then + replica_slot="replica_$RANDOM" + replica_dir="$tmpdir/$replica_slot" + replica_host="$tmpdir/socket_$replica_slot" + + mkdir -p "$replica_host" + + replica_dblog="$tmpdir/db_$replica_slot.log" + + log "Running pg_basebackup for $replica_slot" + + pg_basebackup -v -h "$PGHOST" -U postgres --wal-method=stream --create-slot --slot="$replica_slot" --write-recovery-conf -D "$replica_dir" \ + >> "$setuplog" 2>&1 + + log "Starting replica on $replica_host" + + pg_ctl -D "$replica_dir" -l "$replica_dblog" -w start -o "-F -c listen_addresses=\"\" -c hba_file=$HBA_FILE -k $replica_host -c log_statement=\"all\" " \ + >> "$setuplog" + + >&2 echo "${commandName}: Replica enabled. You can connect to it with: psql 'postgres:///$PGDATABASE?host=$replica_host' -U postgres" + >&2 echo "${commandName}: You can tail the replica logs with: tail -f $replica_dblog" + + export PGREPLICAHOST="$replica_host" + export PGREPLICASLOT="$replica_slot" + fi + # shellcheck disable=SC2317 stop () { log "Stopping the database cluster..." - pg_ctl stop -m i >> "$setuplog" + pg_ctl stop --mode=immediate >> "$setuplog" rm -rf "$tmpdir/db" + if test "$_arg_replica" = "on"; then + log "Stopping the replica cluster..." + pg_ctl -D "$replica_dir" stop --mode=immediate >> "$setuplog" + rm -rf "$replica_dir" + fi } trap stop EXIT - - log "Creating a minimally privileged $PGUSER connection role..." - createuser "$PGUSER" -U postgres --host="$tmpdir/socket" --no-createdb --no-inherit --no-superuser --no-createrole --no-replication --login - - >&2 echo "${commandName}: You can connect with: psql 'postgres:///$PGDATABASE?host=$tmpdir/socket' -U postgres" - >&2 echo "${commandName}: You can tail the logs with: tail -f $tmpdir/db.log" fi if test "$_arg_fixtures"; then diff --git a/test/io/config.py b/test/io/config.py index 733c8873ec..ea6b125fef 100644 --- a/test/io/config.py +++ b/test/io/config.py @@ -45,6 +45,27 @@ def defaultenv(baseenv): } +@pytest.fixture +def replicaenv(defaultenv): + "Default environment for a PostgREST replica." + conf = { + "PGRST_DB_ANON_ROLE": "postgrest_test_anonymous", + "PGRST_DB_SCHEMAS": "replica", + } + return { + "primary": { + **defaultenv, + **conf, + }, + "replica": { + **defaultenv, + **conf, + "PGHOST": os.environ["PGREPLICAHOST"], + "PGREPLICASLOT": os.environ["PGREPLICASLOT"], + }, + } + + @pytest.fixture def slow_schema_cache_env(defaultenv): "Slow schema cache load environment PostgREST." diff --git a/test/io/replica.sql b/test/io/replica.sql new file mode 100644 index 0000000000..deffbec069 --- /dev/null +++ b/test/io/replica.sql @@ -0,0 +1,21 @@ +create schema replica; + +create or replace function replica.is_replica() returns bool as $$ + select pg_is_in_recovery(); +$$ language sql; + +create or replace function replica.get_replica_slot() returns name as $$ + select slot_name from pg_replication_slots limit 1; +$$ language sql; + +create table replica.items as select x as id from generate_series(1, 10) x; + +DROP ROLE IF EXISTS postgrest_test_anonymous; +CREATE ROLE postgrest_test_anonymous; + +GRANT postgrest_test_anonymous TO :PGUSER; + +GRANT USAGE ON SCHEMA replica TO postgrest_test_anonymous; + +GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA replica +TO postgrest_test_anonymous; diff --git a/test/io/test_replica.py b/test/io/test_replica.py new file mode 100644 index 0000000000..24b996253e --- /dev/null +++ b/test/io/test_replica.py @@ -0,0 +1,28 @@ +"IO tests for PostgREST started on replicas" + +import pytest + +from config import * +from util import * +from postgrest import * + + +def test_sanity_replica(replicaenv): + "Test that primary and replica are working as intended" + + with run(env=replicaenv["primary"]) as postgrest: + response = postgrest.session.get("/rpc/is_replica") + assert response.text == "false" + + response = postgrest.session.get("/rpc/get_replica_slot") + assert response.text == '"' + replicaenv["replica"]["PGREPLICASLOT"] + '"' + + response = postgrest.session.get("/items?select=count") + assert response.text == '[{"count":10}]' + + with run(env=replicaenv["replica"]) as postgrest: + response = postgrest.session.get("/rpc/is_replica") + assert response.text == "true" + + response = postgrest.session.get("/items?select=count") + assert response.text == '[{"count":10}]'