From 47e9a2d1341e49e0fb945a8081ebdd39511c3601 Mon Sep 17 00:00:00 2001 From: steve-chavez Date: Thu, 23 May 2024 17:59:20 -0500 Subject: [PATCH 1/9] refactor: Listener to own module --- docs/explanations/architecture.rst | 11 ++++ postgrest.cabal | 1 + src/PostgREST/App.hs | 7 +- src/PostgREST/AppState.hs | 95 ++++----------------------- src/PostgREST/Auth.hs | 6 +- src/PostgREST/Listener.hs | 101 +++++++++++++++++++++++++++++ 6 files changed, 132 insertions(+), 89 deletions(-) create mode 100644 src/PostgREST/Listener.hs diff --git a/docs/explanations/architecture.rst b/docs/explanations/architecture.rst index e926f2c19e..71af6a0bb8 100644 --- a/docs/explanations/architecture.rst +++ b/docs/explanations/architecture.rst @@ -61,3 +61,14 @@ Admin ----- `Admin.hs `_ is in charge of the :ref:`admin_server`. + +HTTP +---- + +The HTTP server is provided by `Warp `_. + +Listener +-------- + +`Listener.hs `_ is in charge of maintaining a `LISTEN session `_ +that keeps the :ref:`schema_cache` and the :ref:`in_db_config` up to date. diff --git a/postgrest.cabal b/postgrest.cabal index fd7744f294..b98c0272a2 100644 --- a/postgrest.cabal +++ b/postgrest.cabal @@ -62,6 +62,7 @@ library PostgREST.SchemaCache.Representations PostgREST.SchemaCache.Table PostgREST.Error + PostgREST.Listener PostgREST.Logger PostgREST.MediaType PostgREST.Metrics diff --git a/src/PostgREST/App.hs b/src/PostgREST/App.hs index ca681f50a0..d7a3d4eac8 100644 --- a/src/PostgREST/App.hs +++ b/src/PostgREST/App.hs @@ -33,6 +33,7 @@ import qualified PostgREST.AppState as AppState import qualified PostgREST.Auth as Auth import qualified PostgREST.Cors as Cors import qualified PostgREST.Error as Error +import qualified PostgREST.Listener as Listener import qualified PostgREST.Logger as Logger import qualified PostgREST.Plan as Plan import qualified PostgREST.Query as Query @@ -67,10 +68,10 @@ run appState = do observer $ AppStartObs prettyVersion - AppState.connectionWorker appState -- Loads the initial SchemaCache + AppState.connectionWorker appState Unix.installSignalHandlers (AppState.getMainThreadId appState) (AppState.connectionWorker appState) (AppState.reReadConfig False appState) - -- reload schema cache + config on NOTIFY - AppState.runListener appState + + Listener.runListener appState Admin.runAdmin appState (serverSettings conf) diff --git a/src/PostgREST/AppState.hs b/src/PostgREST/AppState.hs index a9c5f1947a..62d63f8e99 100644 --- a/src/PostgREST/AppState.hs +++ b/src/PostgREST/AppState.hs @@ -1,5 +1,4 @@ {-# LANGUAGE LambdaCase #-} -{-# LANGUAGE MultiWayIf #-} {-# LANGUAGE NamedFieldPuns #-} {-# LANGUAGE RecordWildCards #-} @@ -21,10 +20,10 @@ module PostgREST.AppState , initWithPool , putSchemaCache , putPgVersion + , putIsListenerOn , usePool , reReadConfig , connectionWorker - , runListener , getObserver , isLoaded , isPending @@ -36,8 +35,6 @@ import qualified Data.ByteString.Char8 as BS import qualified Data.Cache as C import Data.Either.Combinators (whenLeft) import qualified Data.Text as T (unpack) -import Hasql.Connection (acquire) -import qualified Hasql.Notifications as SQL import qualified Hasql.Pool as SQL import qualified Hasql.Pool.Config as SQL import qualified Hasql.Session as SQL @@ -54,9 +51,8 @@ import System.TimeIt (timeItT) import Control.AutoUpdate (defaultUpdateSettings, mkAutoUpdate, updateAction) import Control.Debounce -import Control.Exception (throw) import Control.Retry (RetryPolicy, RetryStatus (..), capDelay, - exponentialBackoff, recoverAll, retrying, + exponentialBackoff, retrying, rsPreviousDelay) import Data.IORef (IORef, atomicWriteIORef, newIORef, readIORef) @@ -64,7 +60,6 @@ import Data.Time.Clock (UTCTime, getCurrentTime) import PostgREST.Config (AppConfig (..), addFallbackAppName, - addTargetSessionAttrs, readAppConfig) import PostgREST.Config.Database (queryDbSettings, queryPgVersion, @@ -442,15 +437,6 @@ internalConnectionWorker appState@AppState{stateObserver=observer, stateMainThre -- retry reloading the schema cache work --- | One second in microseconds -oneSecondInUs :: Int -oneSecondInUs = 1000000 - -retryPolicy :: RetryPolicy -retryPolicy = capDelay delayMicroseconds $ exponentialBackoff oneSecondInUs - where - delayMicroseconds = 32000000 -- 32 seconds - -- | Repeatedly flush the pool, and check if a connection from the -- pool allows access to the PostgreSQL database. -- @@ -459,6 +445,8 @@ retryPolicy = capDelay delayMicroseconds $ exponentialBackoff oneSecondInUs -- Which might not happen if the server is busy with requests. No idle -- connection, no pool timeout. -- +-- It's also necessary to release the pool connections because they cache the pg catalog(see #2620) +-- -- The connection tries are capped, but if the connection times out no error is -- thrown, just 'False' is returned. establishConnection :: AppState -> IO ConnectionStatus @@ -489,6 +477,14 @@ establishConnection appState@AppState{stateObserver=observer} = when itShould $ putRetryNextIn appState delay return itShould + retryPolicy :: RetryPolicy + retryPolicy = + let + delayMicroseconds = 32000000 -- 32 seconds + in + capDelay delayMicroseconds $ exponentialBackoff oneSecondInUs + oneSecondInUs = 1000000 -- | One second in microseconds + -- | Re-reads the config plus config options from the db reReadConfig :: Bool -> AppState -> IO () reReadConfig startingUp appState@AppState{stateObserver=observer} = do @@ -526,70 +522,3 @@ reReadConfig startingUp appState@AppState{stateObserver=observer} = do pass else observer ConfigSucceededObs - --- | Starts the Listener in a thread -runListener :: AppState -> IO () -runListener appState = do - AppConfig{..} <- getConfig appState - when configDbChannelEnabled $ - void . forkIO $ retryingListen appState - --- | Starts a LISTEN connection and handles notifications. It recovers with exponential backoff if the LISTEN connection is lost. --- TODO Once the listen channel is recovered, the retry status is not reset. So if the last backoff was 4 seconds, the next time recovery kicks in the backoff will be 8 seconds. --- This is because `Hasql.Notifications.waitForNotifications` uses a forever loop that only finishes when it throws an exception. -retryingListen :: AppState -> IO () -retryingListen appState@AppState{stateObserver=observer, stateMainThreadId=mainThreadId} = do - AppConfig{..} <- getConfig appState - let - dbChannel = toS configDbChannel - -- Try, catch and rethrow the exception. This is done so we can observe the failure message and let Control.Retry.recoverAll do its work. - -- There's a `Control.Retry.recovering` we could use to avoid this rethrowing, but it's more complex to use. - -- The root cause of these workarounds is that `Hasql.Notifications.waitForNotifications` uses exceptions. - tryRethrow :: IO () -> IO () - tryRethrow action = do - act <- try action - whenLeft act (\ex -> do - putIsListenerOn appState False - observer $ DBListenFail dbChannel (Right $ Left ex) - unless configDbPoolAutomaticRecovery $ do - killThread mainThreadId - throw ex) - - recoverAll retryPolicy (\RetryStatus{rsIterNumber, rsPreviousDelay} -> do - - when (rsIterNumber > 0) $ - let delay = fromMaybe 0 rsPreviousDelay `div` oneSecondInUs in - observer $ DBListenRetry delay - - connection <- acquire $ toUtf8 (addTargetSessionAttrs $ addFallbackAppName prettyVersion configDbUri) - case connection of - Right conn -> do - - tryRethrow $ SQL.listen conn $ SQL.toPgIdentifier dbChannel - - putIsListenerOn appState True - observer $ DBListenStart dbChannel - - when (rsIterNumber > 0) $ do - -- once we can LISTEN again, we might have lost schema cache notificacions, so reload - connectionWorker appState - - tryRethrow $ SQL.waitForNotifications handleNotification conn - - Left err -> do - observer $ DBListenFail dbChannel (Left err) - -- throw an exception so recoverAll works - exitFailure - ) - - where - handleNotification channel msg = - if | BS.null msg -> observer (DBListenerGotSCacheMsg channel) >> cacheReloader - | msg == "reload schema" -> observer (DBListenerGotSCacheMsg channel) >> cacheReloader - | msg == "reload config" -> observer (DBListenerGotConfigMsg channel) >> reReadConfig False appState - | otherwise -> pure () -- Do nothing if anything else than an empty message is sent - - cacheReloader = - -- reloads the schema cache + restarts pool connections - -- it's necessary to restart the pg connections because they cache the pg catalog(see #2620) - connectionWorker appState diff --git a/src/PostgREST/Auth.hs b/src/PostgREST/Auth.hs index ec30950793..37e4fad672 100644 --- a/src/PostgREST/Auth.hs +++ b/src/PostgREST/Auth.hs @@ -1,9 +1,9 @@ {-| Module : PostgREST.Auth -Description : PostgREST authorization functions. +Description : PostgREST authentication functions. -This module provides functions to deal with the JWT authorization (http://jwt.io). -It also can be used to define other authorization functions, +This module provides functions to deal with the JWT authentication (http://jwt.io). +It also can be used to define other authentication functions, in the future Oauth, LDAP and similar integrations can be coded here. Authentication should always be implemented in an external service. diff --git a/src/PostgREST/Listener.hs b/src/PostgREST/Listener.hs new file mode 100644 index 0000000000..9a8f68fe0e --- /dev/null +++ b/src/PostgREST/Listener.hs @@ -0,0 +1,101 @@ +{-# LANGUAGE MultiWayIf #-} +{-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE RecordWildCards #-} + +module PostgREST.Listener (runListener) where + +import qualified Data.ByteString.Char8 as BS + +import Control.Exception (throw) +import Data.Either.Combinators (whenLeft) + +import qualified Hasql.Connection as SQL +import qualified Hasql.Notifications as SQL +import PostgREST.AppState (AppState, getConfig) +import PostgREST.Config (AppConfig (..)) +import PostgREST.Observation (Observation (..)) +import PostgREST.Version (prettyVersion) + +import Control.Retry (RetryPolicy, RetryStatus (..), + capDelay, exponentialBackoff, + recoverAll, rsPreviousDelay) +import qualified PostgREST.AppState as AppState +import qualified PostgREST.Config as Config + +import Protolude + +-- | Starts the Listener in a thread +runListener :: AppState -> IO () +runListener appState = do + AppConfig{..} <- getConfig appState + when configDbChannelEnabled $ + void . forkIO $ retryingListen appState + +-- | Starts a LISTEN connection and handles notifications. It recovers with exponential backoff if the LISTEN connection is lost. +-- TODO Once the listen channel is recovered, the retry status is not reset. So if the last backoff was 4 seconds, the next time recovery kicks in the backoff will be 8 seconds. +-- This is because `Hasql.Notifications.waitForNotifications` uses a forever loop that only finishes when it throws an exception. +retryingListen :: AppState -> IO () +retryingListen appState = do + AppConfig{..} <- AppState.getConfig appState + let + dbChannel = toS configDbChannel + -- Try, catch and rethrow the exception. This is done so we can observe the failure message and let Control.Retry.recoverAll do its work. + -- There's a `Control.Retry.recovering` we could use to avoid this rethrowing, but it's more complex to use. + -- The root cause of these workarounds is that `Hasql.Notifications.waitForNotifications` uses exceptions. + tryRethrow :: IO () -> IO () + tryRethrow action = do + act <- try action + whenLeft act (\ex -> do + AppState.putIsListenerOn appState False + observer $ DBListenFail dbChannel (Right $ Left ex) + unless configDbPoolAutomaticRecovery $ do + killThread mainThreadId + throw ex) + + recoverAll retryPolicy (\RetryStatus{rsIterNumber, rsPreviousDelay} -> do + + when (rsIterNumber > 0) $ + let delay = fromMaybe 0 rsPreviousDelay `div` oneSecondInUs in + observer $ DBListenRetry delay + + connection <- SQL.acquire $ toUtf8 (Config.addTargetSessionAttrs $ Config.addFallbackAppName prettyVersion configDbUri) + case connection of + Right conn -> do + + tryRethrow $ SQL.listen conn $ SQL.toPgIdentifier dbChannel + + AppState.putIsListenerOn appState True + observer $ DBListenStart dbChannel + + when (rsIterNumber > 0) $ do + -- once we can LISTEN again, we might have lost schema cache notificacions, so reload + AppState.connectionWorker appState + + tryRethrow $ SQL.waitForNotifications handleNotification conn + + Left err -> do + observer $ DBListenFail dbChannel (Left err) + -- throw an exception so recoverAll works + exitFailure + ) + + where + handleNotification channel msg = + if | BS.null msg -> observer (DBListenerGotSCacheMsg channel) >> cacheReloader + | msg == "reload schema" -> observer (DBListenerGotSCacheMsg channel) >> cacheReloader + | msg == "reload config" -> observer (DBListenerGotConfigMsg channel) >> AppState.reReadConfig False appState + | otherwise -> pure () -- Do nothing if anything else than an empty message is sent + + cacheReloader = + AppState.connectionWorker appState + + observer = AppState.getObserver appState + mainThreadId = AppState.getMainThreadId appState + + retryPolicy :: RetryPolicy + retryPolicy = + let + delayMicroseconds = 32000000 -- 32 seconds + in + capDelay delayMicroseconds $ exponentialBackoff oneSecondInUs + oneSecondInUs = 1000000 -- | One second in microseconds From da9e497ef1c8681042cd40d418812285d857326c Mon Sep 17 00:00:00 2001 From: steve-chavez Date: Thu, 23 May 2024 18:38:41 -0500 Subject: [PATCH 2/9] changelog: add architecture doc --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85cd12c841..776e0777f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,7 @@ This project adheres to [Semantic Versioning](http://semver.org/). ### Documentation - #3289, Add dark mode. Can be toggled by a button in the bottom right corner. - @laurenceisla + - #3384, Add architecture diagram and documentation - @steve-chavez ## [12.0.3] - 2024-05-09 From c8612f1df1fdf7f2598ae4c72ac25fbf5b0b3ad7 Mon Sep 17 00:00:00 2001 From: Laurence Isla Date: Fri, 24 May 2024 16:26:33 -0500 Subject: [PATCH 3/9] test: use only the first line in the logs to check the server version --- test/io/test_io.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/io/test_io.py b/test/io/test_io.py index d893440faa..80b456f41d 100644 --- a/test/io/test_io.py +++ b/test/io/test_io.py @@ -1164,9 +1164,9 @@ def test_log_postgrest_version(defaultenv): with run(env=defaultenv, no_startup_stdout=False) as postgrest: version = postgrest.session.head("/").headers["Server"].split("/")[1] - output = sorted(postgrest.read_stdout(nlines=5)) + output = postgrest.read_stdout(nlines=1) - assert "Starting PostgREST %s..." % version in output[4] + assert "Starting PostgREST %s..." % version in output[0] def test_succeed_w_role_having_superuser_settings(defaultenv): From 9c165e3edb39da7a328e49033939c8e3c1fa3270 Mon Sep 17 00:00:00 2001 From: Laurence Isla Date: Thu, 23 May 2024 19:24:26 -0500 Subject: [PATCH 4/9] fix: log connection pool events on log-level="debug" instead of "info" --- CHANGELOG.md | 2 +- src/PostgREST/Logger.hs | 2 +- test/io/test_io.py | 20 +++++++++++++++++--- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 776e0777f2..bfbe9e708e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,6 @@ This project adheres to [Semantic Versioning](http://semver.org/). - #3171, #3046, Log schema cache stats to stderr - @steve-chavez - #3210, Dump schema cache through admin API - @taimoorzaeem - #2676, Performance improvement on bulk json inserts, around 10% increase on requests per second by removing `json_typeof` from write queries - @steve-chavez - - #3214, Log connection pool events on log-level=info - @steve-chavez - #3435, Add log-level=debug, for development purposes - @steve-chavez - #1526, Add `/metrics` endpoint on admin server - @steve-chavez - Exposes connection pool metrics, schema cache metrics @@ -23,6 +22,7 @@ This project adheres to [Semantic Versioning](http://semver.org/). - #3340, Log when the LISTEN channel gets a notification - @steve-chavez - #3184, Log full pg version to stderr on connection - @steve-chavez - #3242. Add config `db-hoisted-tx-settings` to apply only hoisted function settings - @taimoorzaeem + - #3214, #3229 Log connection pool events on log-level=debug - @steve-chavez, @laurenceisla ### Fixed diff --git a/src/PostgREST/Logger.hs b/src/PostgREST/Logger.hs index 780d5c786b..8126189263 100644 --- a/src/PostgREST/Logger.hs +++ b/src/PostgREST/Logger.hs @@ -83,7 +83,7 @@ observationLogger loggerState logLevel obs = case obs of when (logLevel >= LogError) $ do logWithZTime loggerState $ observationMessage o o@(HasqlPoolObs _) -> do - when (logLevel >= LogInfo) $ do + when (logLevel >= LogDebug) $ do logWithZTime loggerState $ observationMessage o o@(SchemaCacheLoadedObs _) -> do when (logLevel >= LogDebug) $ do diff --git a/test/io/test_io.py b/test/io/test_io.py index 80b456f41d..32919544ad 100644 --- a/test/io/test_io.py +++ b/test/io/test_io.py @@ -612,7 +612,7 @@ def test_pool_acquisition_timeout(level, defaultenv, metapostgrest): if level == "crit": assert len(output) == 0 - elif level in ["info", "debug"]: + else: assert " 504 " in output[0] assert "Timed out acquiring connection from connection pool." in output[2] @@ -871,7 +871,21 @@ def test_log_level(level, defaultenv): output[1], ) assert len(output) == 2 - else: + elif level == "info": + assert re.match( + r'- - - \[.+\] "GET / HTTP/1.1" 500 - "" "python-requests/.+"', + output[0], + ) + assert re.match( + r'- - postgrest_test_anonymous \[.+\] "GET / HTTP/1.1" 200 - "" "python-requests/.+"', + output[1], + ) + assert re.match( + r'- - postgrest_test_anonymous \[.+\] "GET /unknown HTTP/1.1" 404 - "" "python-requests/.+"', + output[2], + ) + assert len(output) == 3 + elif level == "debug": assert re.match( r'- - - \[.+\] "GET / HTTP/1.1" 500 - "" "python-requests/.+"', output[0], @@ -1456,7 +1470,7 @@ def test_db_error_logging_to_stderr(level, defaultenv, metapostgrest): if level == "crit": assert len(output) == 0 - elif level in ["info", "debug"]: + elif level == "debug": assert " 500 " in output[0] assert "canceling statement due to statement timeout" in output[3] else: From 30ca64d8494852064384ac1a49b9bad4088c2630 Mon Sep 17 00:00:00 2001 From: steve-chavez Date: Fri, 24 May 2024 17:13:02 -0500 Subject: [PATCH 5/9] docs: improve observability --- docs/references/observability.rst | 2 ++ src/PostgREST/Logger.hs | 4 +++- src/PostgREST/Observation.hs | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/references/observability.rst b/docs/references/observability.rst index 687e6ac78d..de75c5d79d 100644 --- a/docs/references/observability.rst +++ b/docs/references/observability.rst @@ -3,6 +3,8 @@ Observability ############# +Observability allows measuring a system's current state based on the data it generates, such as logs, metrics, and traces. + .. contents:: :depth: 1 :local: diff --git a/src/PostgREST/Logger.hs b/src/PostgREST/Logger.hs index 8126189263..08ad5b1664 100644 --- a/src/PostgREST/Logger.hs +++ b/src/PostgREST/Logger.hs @@ -1,7 +1,8 @@ {-| Module : PostgREST.Logger -Description : Wai Middleware to log requests to stdout. +Description : Logging based on the Observation.hs module. Access logs get sent to stdout and server diagnostic get sent to stderr. -} +-- TODO log with buffering enabled to not lose throughput on logging levels higher than LogError module PostgREST.Logger ( middleware , observationLogger @@ -54,6 +55,7 @@ logWithDebounce loggerState action = do putMVar (stateLogDebouncePoolTimeout loggerState) newDebouncer newDebouncer +-- TODO stop using this middleware to reuse the same "observer" pattern for all our logs middleware :: LogLevel -> (Wai.Request -> Maybe BS.ByteString) -> Wai.Middleware middleware logLevel getAuthRole = case logLevel of LogCrit -> requestLogger (const False) diff --git a/src/PostgREST/Observation.hs b/src/PostgREST/Observation.hs index 2dfd94b223..28cdbf3ef3 100644 --- a/src/PostgREST/Observation.hs +++ b/src/PostgREST/Observation.hs @@ -1,7 +1,7 @@ {-# LANGUAGE LambdaCase #-} {-| Module : PostgREST.Observation -Description : Module for observability types +Description : Observations that can be used for Logging and Metrics -} module PostgREST.Observation ( Observation(..) From 1c371d7340db9b4a2918922ee1e09598b4160518 Mon Sep 17 00:00:00 2001 From: steve-chavez Date: Fri, 24 May 2024 17:19:00 -0500 Subject: [PATCH 6/9] docs: clarify architecture --- docs/explanations/architecture.rst | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/docs/explanations/architecture.rst b/docs/explanations/architecture.rst index 71af6a0bb8..ba5917ed58 100644 --- a/docs/explanations/architecture.rst +++ b/docs/explanations/architecture.rst @@ -13,13 +13,20 @@ Code Map This section talks briefly about various important modules. -The starting points of the program are: +Main +---- + +The starting point of the program is `Main.hs `_. + +CLI +--- + +Main then calls `CLI.hs `_, which is in charge of :ref:`cli`. -- `Main.hs `_ -- `CLI.hs `_ -- `App.hs `_ +App +--- -``App.hs`` is then in charge of composing the different modules. +`App.hs `_ is then in charge of composing the different modules. Auth ---- From a51a74b3c1a386a33db4200ecabdf8ce5588b0d3 Mon Sep 17 00:00:00 2001 From: steve-chavez Date: Fri, 24 May 2024 17:25:40 -0500 Subject: [PATCH 7/9] docs: shorten CLI --- docs/references/cli.rst | 41 +++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/docs/references/cli.rst b/docs/references/cli.rst index 823a54c1a4..8cc8c6c60e 100644 --- a/docs/references/cli.rst +++ b/docs/references/cli.rst @@ -3,44 +3,49 @@ CLI === -PostgREST provides a CLI to start and run your postgrest service. The CLI provides the commands listed below: +PostgREST provides a CLI with the commands listed below: -.. _cli_commands: +Help +---- -CLI Commands ------------- +.. code:: bash + + $ postgrest [-h|--help] -Help and Version -~~~~~~~~~~~~~~~~ +Shows all the commands available. + +Version +------- .. code:: bash - $ postgrest [-h|--help] $ postgrest [-v|--version] -Example Config -~~~~~~~~~~~~~~ +Prints the PostgREST version. + +Example +------- .. code:: bash $ postgrest [-e|--example] -These commands show the example configuration file. +Shows example configuration options. -Config -~~~~~~ +Dump Config +----------- .. code:: bash - $ postgrest [--dump-config] [FILENAME] + $ postgrest [--dump-config] -Here ``FILENAME`` is the path to configuration file. +Dumps the loaded :ref:`configuration` values, considering the configuration file, environment variables and :ref:`in_db_config`. -Schema Cache -~~~~~~~~~~~~ +Dump Schema +----------- .. code:: bash - $ postgrest [--dump-schema] [FILENAME] + $ postgrest [--dump-schema] -Here ``FILENAME`` is the path to configuration file. +Dumps the schema cache in JSON format. From 82a43c27676ea0f2c04ea5ec0d481ed1b60fac41 Mon Sep 17 00:00:00 2001 From: steve-chavez Date: Sat, 25 May 2024 15:20:49 -0500 Subject: [PATCH 8/9] nix: postgrest-gen-ctags use haskdogs --- nix/tools/devTools.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/tools/devTools.nix b/nix/tools/devTools.nix index f7f211e233..698c2931fe 100644 --- a/nix/tools/devTools.nix +++ b/nix/tools/devTools.nix @@ -325,8 +325,8 @@ let workingDir = "/"; } '' + ${haskellPackages.haskdogs}/bin/haskdogs ${ctags}/bin/ctags -a -R --fields=+l --languages=python --python-kinds=-iv -f ./tags test/io/ - ${haskellPackages.hasktags}/bin/hasktags -a -R -c -f ./tags . ''; genJwt = From 50b2d302d00e0f1442335d5738e802beb667b609 Mon Sep 17 00:00:00 2001 From: Laurence Isla Date: Fri, 31 May 2024 19:33:18 -0500 Subject: [PATCH 9/9] docs: use "curl --get" for better readability when necessary --- docs/references/api/resource_embedding.rst | 148 +++++++++++++++++---- docs/references/api/tables_views.rst | 13 +- 2 files changed, 135 insertions(+), 26 deletions(-) diff --git a/docs/references/api/resource_embedding.rst b/docs/references/api/resource_embedding.rst index 51f2d2e106..a8322a93c4 100644 --- a/docs/references/api/resource_embedding.rst +++ b/docs/references/api/resource_embedding.rst @@ -409,7 +409,10 @@ Note that the foreign keys have been named explicitly in the :ref:`SQL definitio .. code-block:: bash - curl "http://localhost:3000/orders?select=name,billing_address:addresses!billing(name),shipping_address:addresses!shipping(name)" + # curl "http://localhost:3000/orders?select=name,billing_address:addresses!billing(name),shipping_address:addresses!shipping(name)" + + curl --get "http://localhost:3000/orders" \ + -d "select=name,billing_address:addresses!billing(name),shipping_address:addresses!shipping(name)" .. code-block:: json @@ -434,7 +437,11 @@ Let's take the tables from :ref:`multiple_m2o`. To get the opposite one-to-many .. code-block:: bash - curl "http://localhost:3000/addresses?select=name,billing_orders:orders!billing(name),shipping_orders!shipping(name)&id=eq.1" + # curl "http://localhost:3000/addresses?select=name,billing_orders:orders!billing(name),shipping_orders!shipping(name)&id=eq.1" + + curl --get "http://localhost:3000/addresses" \ + -d "select=name,billing_orders:orders!billing(name),shipping_orders!shipping(name)" \ + -d "id=eq.1" .. code-block:: json @@ -492,7 +499,11 @@ Now, to query a president with their predecessor and successor: .. code-block:: bash - curl "http://localhost:3000/presidents?select=last_name,predecessor(last_name),successor(last_name)&id=eq.2" + # curl "http://localhost:3000/presidents?select=last_name,predecessor(last_name),successor(last_name)&id=eq.2" + + curl --get "http://localhost:3000/presidents" \ + -d "select=last_name,predecessor(last_name),successor(last_name)" \ + -d "id=eq.2" .. code-block:: json @@ -540,7 +551,11 @@ Now, the query would be: .. code-block:: bash - curl "http://localhost:3000/employees?select=last_name,supervisees(last_name)&id=eq.1" + # curl "http://localhost:3000/employees?select=last_name,supervisees(last_name)&id=eq.1" + + curl --get "http://localhost:3000/employees" \ + -d "select=last_name,supervisees(last_name)" \ + -d "id=eq.1" .. code-block:: json @@ -572,7 +587,11 @@ Then, the query would be: .. code-block:: bash - curl "http://localhost:3000/employees?select=last_name,supervisor(last_name)&id=eq.3" + # curl "http://localhost:3000/employees?select=last_name,supervisor(last_name)&id=eq.3" + + curl --get "http://localhost:3000/employees" \ + -d "select=last_name,supervisor(last_name)" \ + -d "id=eq.3" .. code-block:: json @@ -636,7 +655,11 @@ Then, the request would be: .. code-block:: bash - curl "http://localhost:3000/users?select=username,subscribers(username),following(username)&id=eq.4" + # curl "http://localhost:3000/users?select=username,subscribers(username),following(username)&id=eq.4" + + curl --get "http://localhost:3000/users" \ + -d "select=username,subscribers(username),following(username)" \ + -d "id=eq.4" .. code-block:: json @@ -691,7 +714,11 @@ Since it contains the ``films_id`` foreign key, it is possible to join ``box_off .. code-block:: bash - curl "http://localhost:3000/box_office?select=bo_date,gross_revenue,films(title)&gross_revenue=gte.1000000" + # curl "http://localhost:3000/box_office?select=bo_date,gross_revenue,films(title)&gross_revenue=gte.1000000" + + curl --get "http://localhost:3000/box_office" \ + -d "select=bo_date,gross_revenue,films(title)" \ + -d "gross_revenue=gte.1000000" .. note:: @@ -726,7 +753,11 @@ Since this view contains ``nominations.film_id``, which has a **foreign key** re .. code-block:: bash - curl "http://localhost:3000/nominations_view?select=film_title,films(language),roles(character),actors(last_name,first_name)&rank=eq.5" + # curl "http://localhost:3000/nominations_view?select=film_title,films(language),roles(character),actors(last_name,first_name)&rank=eq.5" + + curl --get "http://localhost:3000/nominations_view" \ + -d "select=film_title,films(language),roles(character),actors(last_name,first_name)" \ + -d "rank=eq.5" It's also possible to foreign key join `Materialized Views `_. @@ -766,7 +797,11 @@ A request with ``directors`` embedded: .. code-block:: bash - curl "http://localhost:3000/rpc/getallfilms?select=title,directors(id,last_name)&title=like.*Workers*" + # curl "http://localhost:3000/rpc/getallfilms?select=title,directors(id,last_name)&title=like.*Workers*" + + curl --get "http://localhost:3000/rpc/getallfilms" \ + -d "select=title,directors(id,last_name)" \ + -d "title=like.*Workers*" .. code-block:: json @@ -836,13 +871,21 @@ Embedded resources can be shaped similarly to their top-level counterparts. To d .. code-block:: bash - curl "http://localhost:3000/films?select=*,actors(*)&actors.order=last_name,first_name" + # curl "http://localhost:3000/films?select=*,actors(*)&actors.order=last_name,first_name" + + curl --get "http://localhost:3000/films" \ + -d "select=*,actors(*)" \ + -d "actors.order=last_name,first_name" This sorts the list of actors in each film but does *not* change the order of the films themselves. To filter the roles returned with each film: .. code-block:: bash - curl "http://localhost:3000/films?select=*,roles(*)&roles.character=in.(Chico,Harpo,Groucho)" + # curl "http://localhost:3000/films?select=*,roles(*)&roles.character=in.(Chico,Harpo,Groucho)" + + curl --get "http://localhost:3000/films" \ + -d "select=*,roles(*)" \ + -d "roles.character=in.(Chico,Harpo,Groucho)" Once again, this restricts the roles included to certain characters but does not filter the films in any way. Films without any of those characters would be included along with empty character lists. @@ -850,7 +893,11 @@ An ``or`` filter can be used for a similar operation: .. code-block:: bash - curl "http://localhost:3000/films?select=*,roles(*)&roles.or=(character.eq.Gummo,character.eq.Zeppo)" + # curl "http://localhost:3000/films?select=*,roles(*)&roles.or=(character.eq.Gummo,character.eq.Zeppo)" + + curl --get "http://localhost:3000/films" \ + -d "select=*,roles(*)" \ + -d "roles.or=(character.eq.Gummo,character.eq.Zeppo)" However, this only works for columns inside ``roles``. See :ref:`how to use "or" across multiple resources `. @@ -858,13 +905,23 @@ Limit and offset operations are possible: .. code-block:: bash - curl "http://localhost:3000/films?select=*,actors(*)&actors.limit=10&actors.offset=2" + # curl "http://localhost:3000/films?select=*,actors(*)&actors.limit=10&actors.offset=2" + + curl --get "http://localhost:3000/films" \ + -d "select=*,actors(*)" \ + -d "actors.limit=10" \ + -d "actors.offset=2" Embedded resources can be aliased and filters can be applied on these aliases: .. code-block:: bash - curl "http://localhost:3000/films?select=*,90_comps:competitions(name),91_comps:competitions(name)&90_comps.year=eq.1990&91_comps.year=eq.1991" + # curl "http://localhost:3000/films?select=*,actors(*)&actors.limit=10&actors.offset=2" + + curl --get "http://localhost:3000/films" \ + -d "select=*,90_comps:competitions(name),91_comps:competitions(name)" \ + -d "90_comps.year=eq.1990" \ + -d "91_comps.year=eq.1991" Filters can also be applied on nested embedded resources: @@ -883,7 +940,11 @@ By default, :ref:`embed_filters` don't change the top-level resource(``films``) .. code-block:: bash - curl "http://localhost:3000/films?select=title,actors(first_name,last_name)&actors.first_name=eq.Jehanne + # curl "http://localhost:3000/films?select=title,actors(first_name,last_name)&actors.first_name=eq.Jehanne + + curl --get "http://localhost:3000/films" \ + -d "select=title,actors(first_name,last_name)" \ + -d "actors.first_name=eq.Jehanne" .. code-block:: json @@ -911,7 +972,11 @@ In order to filter the top level rows you need to add ``!inner`` to the embedded .. code-block:: bash - curl "http://localhost:3000/films?select=title,actors!inner(first_name,last_name)&actors.first_name=eq.Jehanne" + # curl "http://localhost:3000/films?select=title,actors!inner(first_name,last_name)&actors.first_name=eq.Jehanne" + + curl --get "http://localhost:3000/films" \ + -d "select=title,actors!inner(first_name,last_name)" \ + -d "actors.first_name=eq.Jehanne" .. code-block:: json @@ -938,20 +1003,32 @@ For example, doing ``actors=not.is.null`` returns the same result as ``actors!in .. code-block:: bash - curl "http://localhost:3000/films?select=title,actors(*)&actors=not.is.null" + # curl "http://localhost:3000/films?select=title,actors(*)&actors=not.is.null" + + curl --get "http://localhost:3000/films" \ + -d "select=title,actors(*)" \ + -d "actors=not.is.null" The ``is.null`` filter can be used in embedded resources to perform an anti-join. To get all the films that do not have any nominations: .. code-block:: bash - curl "http://localhost:3000/films?select=title,nominations()&nominations=is.null" + # curl "http://localhost:3000/films?select=title,nominations()&nominations=is.null" + + curl --get "http://localhost:3000/films" \ + -d "select=title,nominations()" \ + -d "nominations=is.null" Both ``is.null`` and ``not.is.null`` can be included inside the `or` operator. For instance, to get the films that have no actors **or** directors registered yet: .. code-block:: bash - curl "http://localhost:3000/films?select=title,actors(*),directors(*)&or=(actors.is.null,directors.is.null)" + # curl "http://localhost:3000/films?select=title,nominations()&nominations=is.null" + + curl --get "http://localhost:3000/films" \ + -d select=title,actors(*),directors(*)" \ + -d "or=(actors.is.null,directors.is.null)" .. _or_embed_rels: @@ -963,7 +1040,13 @@ For instance, to show the films whose actors **or** directors are named John: .. code-block:: bash - curl "http://localhost:3000/films?select=title,actors(),directors()&directors.first_name=eq.John&actors.first_name=eq.John&or=(directors.not.is.null,actors.not.is.null)" + # curl "http://localhost:3000/films?select=title,actors(),directors()&directors.first_name=eq.John&actors.first_name=eq.John&or=(directors.not.is.null,actors.not.is.null)" + + curl --get "http://localhost:3000/films" \ + -d "select=title,actors(),directors()" \ + -d "directors.first_name=eq.John" \ + -d "actors.first_name=eq.John" \ + -d "or=(directors.not.is.null,actors.not.is.null)" .. _empty_embed: @@ -976,7 +1059,12 @@ To filter the films by actors but not include them: .. code-block:: bash - curl "http://localhost:3000/films?select=title,actors()&actors.first_name=eq.Jehanne&actors=not.is.null" + # curl "http://localhost:3000/films?select=title,actors()&actors.first_name=eq.Jehanne&actors=not.is.null" + + curl --get "http://localhost:3000/films" \ + -d "select=title,actors()" \ + -d "actors.first_name=eq.Jehanne" \ + -d "actors=not.is.null" .. code-block:: json @@ -997,7 +1085,11 @@ For example, to arrange the films in descending order using the director's last .. code-block:: bash - curl "http://localhost:3000/films?select=title,directors(last_name)&order=directors(last_name).desc" + # curl "http://localhost:3000/films?select=title,directors(last_name)&order=directors(last_name).desc" + + curl --get "http://localhost:3000/films" \ + -d "select=title,directors(last_name)" \ + -d "order=directors(last_name).desc" .. _spread_embed: @@ -1008,7 +1100,11 @@ On many-to-one and one-to-one relationships, you can "spread" the embedded resou .. code-block:: bash - curl "http://localhost:3000/films?select=title,...directors(director_last_name:last_name)&title=like.*Workers*" + # curl "http://localhost:3000/films?select=title,...directors(director_last_name:last_name)&title=like.*Workers*" + + curl --get "http://localhost:3000/films" \ + -d "select=title,...directors(director_last_name:last_name)" \ + -d "title=like.*Workers*" .. code-block:: json @@ -1025,7 +1121,11 @@ You can use this to get the columns of a join table in a many-to-many relationsh .. code-block:: bash - curl "http://localhost:3000/films?select=title,actors:roles(character,...actors(first_name,last_name))&title=like.*Lighthouse*" + # curl "http://localhost:3000/films?select=title,actors:roles(character,...actors(first_name,last_name))&title=like.*Lighthouse*" + + curl --get "http://localhost:3000/films" \ + -d "select=title,actors:roles(character,...actors(first_name,last_name))" \ + -d "title=like.*Lighthouse*" .. code-block:: json diff --git a/docs/references/api/tables_views.rst b/docs/references/api/tables_views.rst index 0baada72a6..3a5c477590 100644 --- a/docs/references/api/tables_views.rst +++ b/docs/references/api/tables_views.rst @@ -129,7 +129,12 @@ You can also apply complex logic to the conditions: .. code-block:: bash - curl "http://localhost:3000/people?grade=gte.90&student=is.true&or=(age.eq.14,not.and(age.gte.11,age.lte.17))" + # curl "http://localhost:3000/people?grade=gte.90&student=is.true&or=(age.eq.14,not.and(age.gte.11,age.lte.17))" + + curl --get "http://localhost:3000/people" \ + -d "grade=gte.90" \ + -d "student=is.true" \ + -d "or=(age.eq.14,not.and(age.gte.11,age.lte.17))" .. _modifiers: @@ -322,7 +327,11 @@ The arrow operators(``->``, ``->>``) can also be used for accessing composite fi .. code-block:: bash - curl "http://localhost:3000/countries?select=id,location->>lat,location->>long,primary_language:languages->0&location->lat=gte.19" + # curl "http://localhost:3000/countries?select=id,location->>lat,location->>long,primary_language:languages->0&location->lat=gte.19" + + curl --get "http://localhost:3000/countries" \ + -d "select=id,location->>lat,location->>long,primary_language:languages->0" \ + -d "location->lat=gte.19" .. code-block:: json