From b802af0027ef02282aecfe8947851969a4e94e14 Mon Sep 17 00:00:00 2001 From: DaMandal0rian Date: Mon, 22 Jul 2024 19:53:46 +0300 Subject: [PATCH 1/2] squids add db pooling and replication --- .../terraform/explorer/base/config/pgcat.toml | 115 ++++++++++++++++++ .../base/config/postgresql-replica.conf | 0 .../explorer/base/config/postgresql.conf | 7 ++ .../scripts/create_squid_node_compose_file.sh | 11 ++ ...eate_squid_node_db_replica_compose_file.sh | 46 +++++++ .../explorer/base/scripts/install_docker.sh | 8 ++ 6 files changed, 187 insertions(+) create mode 100644 templates/terraform/explorer/base/config/pgcat.toml create mode 100644 templates/terraform/explorer/base/config/postgresql-replica.conf create mode 100644 templates/terraform/explorer/base/scripts/create_squid_node_db_replica_compose_file.sh diff --git a/templates/terraform/explorer/base/config/pgcat.toml b/templates/terraform/explorer/base/config/pgcat.toml new file mode 100644 index 00000000..384edf5d --- /dev/null +++ b/templates/terraform/explorer/base/config/pgcat.toml @@ -0,0 +1,115 @@ +# +# PgCat configuration file +# + +# +# General pooler settings +[general] +# What IP to run on, 0.0.0.0 means accessible from everywhere. +host = "0.0.0.0" + +# Port to run on, same as PgBouncer used in this example. +port = 6432 + +# Whether to enable prometheus exporter or not. +enable_prometheus_exporter = true + +# Port at which prometheus exporter listens on. +prometheus_exporter_port = 9930 + +# How long to wait before aborting a server connection (ms). +connect_timeout = 5000 + +# How much time to give `SELECT 1` health check query to return with a result (ms). +healthcheck_timeout = 1000 + +# How long to keep connection available for immediate re-use, without running a healthcheck query on it +healthcheck_delay = 30000 + +# How much time to give clients during shutdown before forcibly killing client connections (ms). +shutdown_timeout = 60000 + +# For how long to ban a server if it fails a health check (seconds). +ban_time = 60 # seconds + +# If we should log client connections +log_client_connections = false + +# If we should log client disconnections +log_client_disconnections = false + +# TLS +# tls_certificate = "server.cert" +# tls_private_key = "server.key" + +# Credentials to access the virtual administrative database (pgbouncer or pgcat) +# Connecting to that database allows running commands like `SHOW POOLS`, `SHOW DATABASES`, etc.. +admin_username = "postgres" +admin_password = "postgres" + +# pool +# configs are structured as pool. +# the pool_name is what clients use as database name when connecting +# For the example below a client can connect using "postgres://sharding_user:sharding_user@pgcat_host:pgcat_port/sharded" +[pools.postgres] +# Pool mode (see PgBouncer docs for more). +# session: one server connection per connected client +# transaction: one server connection per client transaction +pool_mode = "transaction" + +# If the client doesn't specify, route traffic to +# this role by default. +# +# any: round-robin between primary and replicas, +# replica: round-robin between replicas only without touching the primary, +# primary: all queries go to the primary unless otherwise specified. +default_role = "any" + +# Query parser. If enabled, we'll attempt to parse +# every incoming query to determine if it's a read or a write. +# If it's a read query, we'll direct it to a replica. Otherwise, if it's a write, +# we'll direct it to the primary. +query_parser_enabled = true + +# If the query parser is enabled and this setting is enabled, we'll attempt to +# infer the role from the query itself. +query_parser_read_write_splitting = true + +# If the query parser is enabled and this setting is enabled, the primary will be part of the pool of databases used for +# load balancing of read queries. Otherwise, the primary will only be used for write +# queries. The primary can always be explicitly selected with our custom protocol. +primary_reads_enabled = true + +# So what if you wanted to implement a different hashing function, +# or you've already built one and you want this pooler to use it? +# +# Current options: +# +# pg_bigint_hash: PARTITION BY HASH (Postgres hashing function) +# sha1: A hashing function based on SHA1 +# +sharding_function = "pg_bigint_hash" + +# Credentials for users that may connect to this cluster +[pools.postgres.users.0] +username = "postgres" +password = "postgres" +# Maximum number of server connections that can be established for this user +# The maximum number of connection from a single Pgcat process to any database in the cluster +# is the sum of pool_size across all users. +pool_size = 9 + +# Maximum query duration. Dangerous, but protects against DBs that died in a non-obvious way. +statement_timeout = 0 + +# Shard 0 +[pools.postgres.shards.0] +# [ host, port, role ] +servers = [ + [ "postgres", 5432, "primary" ], + [ "postgres", 5432, "replica" ] +] +# Database name (e.g. "postgres") +database = "postgres" + +database = "postgres" diff --git a/templates/terraform/explorer/base/config/postgresql-replica.conf b/templates/terraform/explorer/base/config/postgresql-replica.conf new file mode 100644 index 00000000..e69de29b diff --git a/templates/terraform/explorer/base/config/postgresql.conf b/templates/terraform/explorer/base/config/postgresql.conf index fde9aa5e..488c0ba3 100644 --- a/templates/terraform/explorer/base/config/postgresql.conf +++ b/templates/terraform/explorer/base/config/postgresql.conf @@ -50,3 +50,10 @@ pg_stat_statements.track = all # Locking Settings deadlock_timeout = 1s + +# Replication Settings +wal_level = replica +max_wal_senders = 10 +max_replication_slots = 10 +hot_standby = on +max_standby_archive_delay = 30s diff --git a/templates/terraform/explorer/base/scripts/create_squid_node_compose_file.sh b/templates/terraform/explorer/base/scripts/create_squid_node_compose_file.sh index 2746d4d3..160b43b2 100755 --- a/templates/terraform/explorer/base/scripts/create_squid_node_compose_file.sh +++ b/templates/terraform/explorer/base/scripts/create_squid_node_compose_file.sh @@ -28,6 +28,17 @@ services: options: loki-url: "https://logging.subspace.network/loki/api/v1/push" + pgcat: + image: ghcr.io/postgresml/pgcat:e1e4929d439313d987c352b4517a6d99627f3e9c + command: + - "pgcat" + - "/etc/pgcat/pgcat.toml" + volumes: + - "$HOME/squid/postgresql/conf/pgcat.toml:/etc/pgcat/pgcat.toml" + ports: + - "6432:6432" + - "9930:9930" + run-migrations: image: ghcr.io/subspace/blockexplorer-processor:\${DOCKER_TAG} restart: on-failure:5 diff --git a/templates/terraform/explorer/base/scripts/create_squid_node_db_replica_compose_file.sh b/templates/terraform/explorer/base/scripts/create_squid_node_db_replica_compose_file.sh new file mode 100644 index 00000000..660d4e5c --- /dev/null +++ b/templates/terraform/explorer/base/scripts/create_squid_node_db_replica_compose_file.sh @@ -0,0 +1,46 @@ +#!/bin/sh +source $HOME/.bash_profile +cat > $HOME/squid/docker-compose.yml << EOF +version: "3.8" + +services: + db_replica: + image: postgres:16 + shm_size: 1gb + volumes: + - db_data_replica:/var/lib/postgresql/data + - ./postgresql_replica.conf:/etc/postgresql/postgresql.conf:ro + environment: + POSTGRES_DB: squid-archive + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + ports: + - "5432:5432" + command: > + bash -c " + until pg_isready -h ${PRIMARY_HOST} -p 5432 -U ${POSTGRES_USER}; do sleep 1; done; + pg_basebackup -h ${PRIMARY_HOST} -D /var/lib/postgresql/data -U ${POSTGRES_USER} -P -v -R -X stream -S replica_slot; + echo 'primary_conninfo = ''host=${PRIMARY_HOST} port=5432 user=${POSTGRES_USER} password=${POSTGRES_PASSWORD}''' >> /var/lib/postgresql/data/postgresql.auto.conf; + touch /var/lib/postgresql/data/standby.signal; + postgres -c config_file=/etc/postgresql/postgresql.conf" + + agent: + container_name: newrelic-infra + image: newrelic/infrastructure:latest + cap_add: + - SYS_PTRACE + network_mode: bridge + pid: host + privileged: true + volumes: + - "/:/host:ro" + - "/var/run/docker.sock:/var/run/docker.sock" + environment: + NRIA_LICENSE_KEY: "\${NR_API_KEY}" + NRIA_DISPLAY_NAME: "squid-\${NETWORK_NAME}-db-replica" + restart: unless-stopped + +volumes: + db_data_replica: + +EOF diff --git a/templates/terraform/explorer/base/scripts/install_docker.sh b/templates/terraform/explorer/base/scripts/install_docker.sh index 36355f3c..6efbcf98 100755 --- a/templates/terraform/explorer/base/scripts/install_docker.sh +++ b/templates/terraform/explorer/base/scripts/install_docker.sh @@ -23,3 +23,11 @@ fi sudo sysctl -p /etc/sysctl.conf sudo docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions + +# install ufw-docker +sudo wget -O /usr/local/bin/ufw-docker \ +https://github.com/chaifeng/ufw-docker/raw/master/ufw-docker +sudo chmod +x /usr/local/bin/ufw-docker +sudo ufw enable +sudo ufw-docker install +sudo systemctl restart ufw From 37ab6276079d54b58457b8be10fd6cb39a8945ca Mon Sep 17 00:00:00 2001 From: DaMandal0rian Date: Mon, 22 Jul 2024 19:57:27 +0300 Subject: [PATCH 2/2] add PORT env variable --- .../scripts/create_squid_node_compose_file.sh | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/templates/terraform/explorer/base/scripts/create_squid_node_compose_file.sh b/templates/terraform/explorer/base/scripts/create_squid_node_compose_file.sh index 160b43b2..4c2a0efb 100755 --- a/templates/terraform/explorer/base/scripts/create_squid_node_compose_file.sh +++ b/templates/terraform/explorer/base/scripts/create_squid_node_compose_file.sh @@ -28,6 +28,24 @@ services: options: loki-url: "https://logging.subspace.network/loki/api/v1/push" + db_replica: + image: postgres:16 + shm_size: 1gb + volumes: + - db_data_replica:/var/lib/postgresql/data + environment: + POSTGRES_DB: squid-archive + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + command: > + bash -c "until pg_isready -h db_primary -p 5432 -U ${POSTGRES_USER}; do sleep 1; done; + pg_basebackup -h db_primary -D /var/lib/postgresql/data -U ${POSTGRES_USER} -P -v --wal-method=stream; + echo 'hot_standby = on' >> /var/lib/postgresql/data/postgresql.conf; + echo 'primary_conninfo = ''host=db_primary port=5432 user=${POSTGRES_USER} password=${POSTGRES_PASSWORD}''' >> /var/lib/postgresql/data/postgresql.conf; + pg_ctl -D /var/lib/postgresql/data -o '-c config_file=/var/lib/postgresql/data/postgresql.conf' start" + depends_on: + - db_primary + pgcat: image: ghcr.io/postgresml/pgcat:e1e4929d439313d987c352b4517a6d99627f3e9c command: @@ -48,6 +66,8 @@ services: DB_NAME: \${DB_NAME} # provide DB password DB_PASS: \${DB_PASS} + # provide DB port + DB_PORT: \${DB_PORT} depends_on: - db command: "npm run db:migrate" @@ -65,6 +85,8 @@ services: DB_NAME: \${DB_NAME} # provide DB password DB_PASS: \${DB_PASS} + # provide DB port + DB_PORT: \${DB_PORT} depends_on: - db - run-migrations @@ -87,6 +109,8 @@ services: DB_NAME: \${DB_NAME} # provide DB password DB_PASS: \${DB_PASS} + # provide DB port + DB_PORT: \${DB_PORT} ports: - "4350:4000" logging: