diff --git a/.gitignore b/.gitignore index a6c79a88f0..abb3110f56 100644 --- a/.gitignore +++ b/.gitignore @@ -137,3 +137,5 @@ logs/ # Generated files docs/manager/rest-reference/openapi.json + +fixtures/redis diff --git a/changes/1605.feature.md b/changes/1605.feature.md new file mode 100644 index 0000000000..6253efb03b --- /dev/null +++ b/changes/1605.feature.md @@ -0,0 +1 @@ +Add SSL Support for Redis and Redis Sentinel Connections diff --git a/configs/manager/sample.etcd.redis-tls.json b/configs/manager/sample.etcd.redis-tls.json new file mode 100644 index 0000000000..cf94ba1bb7 --- /dev/null +++ b/configs/manager/sample.etcd.redis-tls.json @@ -0,0 +1,9 @@ +{ + "redis_helper_config": { + "ssl": true, + "ssl_cert_reqs": "required", + "ssl_ca_certs": "${SSL_CA_CERTS_PATH}", + "ssl_certfile": "${SSL_CERTFILE_PATH}", + "ssl_keyfile": "${SSL_KEYFILE_PATH}" + } +} diff --git a/configs/redis/sentinel.conf b/configs/redis/sentinel.conf old mode 100644 new mode 100755 index 964cf77f2a..eca701129d --- a/configs/redis/sentinel.conf +++ b/configs/redis/sentinel.conf @@ -1,12 +1,19 @@ port REDIS_SENTINEL_SELF_PORT requirepass develove +tls-cert-file /fixtures/cert.pem +tls-key-file /fixtures/key.pem +tls-ca-cert-file /fixtures/cert.pem +tls-auth-clients no +tls-port REDIS_SENTINEL_SELF_EXPOSED_PORT +tls-replication yes + sentinel resolve-hostnames yes sentinel announce-ip REDIS_SENTINEL_SELF_HOST -sentinel announce-port REDIS_SENTINEL_SELF_PORT +sentinel announce-port REDIS_SENTINEL_SELF_EXPOSED_PORT sentinel auth-pass mymaster develove sentinel down-after-milliseconds mymaster 1000 sentinel failover-timeout mymaster 5000 sentinel parallel-syncs mymaster 2 -sentinel monitor mymaster node01 9500 2 +sentinel monitor mymaster node01 REDIS_MASTER_EXPOSED_PORT 2 protected-mode no diff --git a/configs/webserver/halfstack.conf b/configs/webserver/halfstack.conf index e12ebe9429..832889eb9e 100644 --- a/configs/webserver/halfstack.conf +++ b/configs/webserver/halfstack.conf @@ -55,6 +55,12 @@ redis.addr = "localhost:6379" # redis.redis_helper_config.socket_connect_timeout = 2 # redis.redis_helper_config.reconnect_poll_timeout = 0.3 +# redis.redis_helper_config.ssl = true +# redis.redis_helper_config.ssl_cert_reqs = "required" +# redis.redis_helper_config.ssl_ca_certs = ${SSL_CA_CERTS_PATH} +# redis.redis_helper_config.ssl_certfile = ${SSL_CERTFILE_PATH} +# redis.redis_helper_config.ssl_keyfile = ${SSL_KEYFILE_PATH} + max_age = 604800 # 1 week flush_on_startup = false login_block_time = 1200 # 20 min (in sec) diff --git a/configs/webserver/sample.conf b/configs/webserver/sample.conf index b94bd05591..ed6bb38960 100644 --- a/configs/webserver/sample.conf +++ b/configs/webserver/sample.conf @@ -132,6 +132,12 @@ redis.addr = "localhost:6379" # redis.redis_helper_config.socket_connect_timeout = 2 # redis.redis_helper_config.reconnect_poll_timeout = 0.3 +# redis.redis_helper_config.ssl = true +# redis.redis_helper_config.ssl_cert_reqs = "required" +# redis.redis_helper_config.ssl_ca_certs = ${SSL_CA_CERTS_PATH} +# redis.redis_helper_config.ssl_certfile = ${SSL_CERTFILE_PATH} +# redis.redis_helper_config.ssl_keyfile = ${SSL_KEYFILE_PATH} + max_age = 604800 # 1 week flush_on_startup = false # Time to block login when an email consecutively fails to login diff --git a/docker-compose.halfstack-2303.yml b/docker-compose.halfstack-2303.yml index 2d3d214135..8cba0da7d9 100644 --- a/docker-compose.halfstack-2303.yml +++ b/docker-compose.halfstack-2303.yml @@ -30,9 +30,16 @@ services: - "8110:6379" volumes: - "./volumes/${DATADIR_PREFIX:-.}/redis-data:/data:rw" + - "./fixtures/redis:/fixtures:rw" command: > redis-server --appendonly yes + --tls-auth-clients no + --tls-port 6380 + --tls-cert-file /fixtures/cert.pem + --tls-ca-cert-file /fixtures/cert.pem + --tls-key-file /fixtures/key.pem + healthcheck: test: ["CMD-SHELL", "redis-cli ping | grep PONG"] interval: 5s diff --git a/docker-compose.halfstack-ha.yml b/docker-compose.halfstack-ha.yml index dad058c26c..0ff7df6637 100644 --- a/docker-compose.halfstack-ha.yml +++ b/docker-compose.halfstack-ha.yml @@ -22,16 +22,26 @@ services: networks: - half ports: - - 0.0.0.0:${REDIS_MASTER_PORT}:${REDIS_MASTER_PORT} + - 0.0.0.0:${REDIS_MASTER_EXPOSED_PORT}:${REDIS_MASTER_EXPOSED_PORT} + volumes: + - "./volumes/${DATADIR_PREFIX:-.}/redis-data:/data:rw" + - "./fixtures/redis:/fixtures:rw" command: > redis-server --port ${REDIS_MASTER_PORT} --requirepass ${REDIS_PASSWORD:-develove} --masterauth ${REDIS_PASSWORD:-develove} --replica-announce-ip node01 - --replica-announce-port ${REDIS_MASTER_PORT} + --replica-announce-port ${REDIS_MASTER_EXPOSED_PORT} --min-slaves-to-write 1 --min-slaves-max-lag 10 + --tls-port ${REDIS_MASTER_EXPOSED_PORT} + --tls-replication yes + --tls-auth-clients no + --tls-cert-file /fixtures/cert.pem + --tls-ca-cert-file /fixtures/cert.pem + --tls-key-file /fixtures/key.pem + # IMPORTANT: We have INTENTIONALLY OMITTED the healthchecks # because it interferes with pause/unpause of container to simulate # network partitioning. @@ -42,35 +52,53 @@ services: networks: - half ports: - - 0.0.0.0:${REDIS_SLAVE1_PORT}:${REDIS_SLAVE1_PORT} + - 0.0.0.0:${REDIS_SLAVE1_EXPOSED_PORT}:${REDIS_SLAVE1_EXPOSED_PORT} + volumes: + - "./volumes/${DATADIR_PREFIX:-.}/redis-data:/data:rw" + - "./fixtures/redis:/fixtures:rw" command: > redis-server --port ${REDIS_SLAVE1_PORT} --requirepass ${REDIS_PASSWORD:-develove} --masterauth ${REDIS_PASSWORD:-develove} - --slaveof node01 ${REDIS_MASTER_PORT} + --slaveof node01 ${REDIS_MASTER_EXPOSED_PORT} --replica-announce-ip node02 - --replica-announce-port ${REDIS_SLAVE1_PORT} + --replica-announce-port ${REDIS_SLAVE1_EXPOSED_PORT} --min-slaves-to-write 1 --min-slaves-max-lag 10 + --tls-port ${REDIS_SLAVE1_EXPOSED_PORT} + --tls-replication yes + --tls-auth-clients no + --tls-cert-file /fixtures/cert.pem + --tls-ca-cert-file /fixtures/cert.pem + --tls-key-file /fixtures/key.pem backendai-half-redis-node03: image: redis:7.0.5-alpine hostname: node03 ports: - - 0.0.0.0:${REDIS_SLAVE2_PORT}:${REDIS_SLAVE2_PORT} + - 0.0.0.0:${REDIS_SLAVE2_EXPOSED_PORT}:${REDIS_SLAVE2_EXPOSED_PORT} networks: - half + volumes: + - "./volumes/${DATADIR_PREFIX:-.}/redis-data:/data:rw" + - "./fixtures/redis:/fixtures:rw" command: > redis-server - --port ${REDIS_SLAVE2_PORT} + --port 9502 --requirepass ${REDIS_PASSWORD:-develove} --masterauth ${REDIS_PASSWORD:-develove} - --slaveof node01 ${REDIS_MASTER_PORT} + --slaveof node01 ${REDIS_MASTER_EXPOSED_PORT} --replica-announce-ip node03 - --replica-announce-port ${REDIS_SLAVE2_PORT} + --replica-announce-port ${REDIS_SLAVE2_EXPOSED_PORT} --min-slaves-to-write 1 --min-slaves-max-lag 10 + --tls-port ${REDIS_SLAVE2_EXPOSED_PORT} + --tls-replication yes + --tls-auth-clients no + --tls-cert-file /fixtures/cert.pem + --tls-ca-cert-file /fixtures/cert.pem + --tls-key-file /fixtures/key.pem backendai-half-redis-sentinel01: image: redis:7.0.5-alpine @@ -81,16 +109,16 @@ services: - type: bind source: ${COMPOSE_PATH} target: /config + - "./volumes/${DATADIR_PREFIX:-.}/redis-data:/data:rw" + - "./fixtures/redis:/fixtures:rw" ports: - - 0.0.0.0:${REDIS_SENTINEL1_PORT}:${REDIS_SENTINEL1_PORT} + - 0.0.0.0:${REDIS_SENTINEL1_EXPOSED_PORT}:${REDIS_SENTINEL1_EXPOSED_PORT} depends_on: - backendai-half-redis-node01 - backendai-half-redis-node02 - backendai-half-redis-node03 command: > redis-sentinel /config/sentinel01.conf - environment: - - REDIS_PASSWORD=${REDIS_PASSWORD:-develove} backendai-half-redis-sentinel02: image: redis:7.0.5-alpine @@ -101,16 +129,16 @@ services: - type: bind source: ${COMPOSE_PATH} target: /config + - "./volumes/${DATADIR_PREFIX:-.}/redis-data:/data:rw" + - "./fixtures/redis:/fixtures:rw" ports: - - 0.0.0.0:${REDIS_SENTINEL2_PORT}:${REDIS_SENTINEL2_PORT} + - 0.0.0.0:${REDIS_SENTINEL2_EXPOSED_PORT}:${REDIS_SENTINEL2_EXPOSED_PORT} depends_on: - backendai-half-redis-node01 - backendai-half-redis-node02 - backendai-half-redis-node03 command: > redis-sentinel /config/sentinel02.conf - environment: - - REDIS_PASSWORD=${REDIS_PASSWORD:-develove} backendai-half-redis-sentinel03: image: redis:7.0.5-alpine @@ -121,16 +149,16 @@ services: - type: bind source: ${COMPOSE_PATH} target: /config + - "./volumes/${DATADIR_PREFIX:-.}/redis-data:/data:rw" + - "./fixtures/redis:/fixtures:rw" ports: - - 0.0.0.0:${REDIS_SENTINEL3_PORT}:${REDIS_SENTINEL3_PORT} + - 0.0.0.0:${REDIS_SENTINEL3_EXPOSED_PORT}:${REDIS_SENTINEL3_EXPOSED_PORT} depends_on: - backendai-half-redis-node01 - backendai-half-redis-node02 - backendai-half-redis-node03 command: > redis-sentinel /config/sentinel03.conf - environment: - - REDIS_PASSWORD=${REDIS_PASSWORD:-develove} backendai-half-etcd-proxy: image: quay.io/coreos/etcd:v3.5.4 diff --git a/scripts/install-dev.sh b/scripts/install-dev.sh index 6f1c16776d..476112f2c6 100755 --- a/scripts/install-dev.sh +++ b/scripts/install-dev.sh @@ -129,6 +129,11 @@ usage() { echo " ${LWHITE}--configure-ha${NC}" echo " Configure HA dev environment." echo " (default: false)" + echo "" + echo " ${LWHITE}--enable-ssl${NC}" + echo " Apply SSL to Redis connection." + echo " This generate and use self-signed certificate." + echo " (default: false)" } show_error() { @@ -286,6 +291,7 @@ PYTHON_VERSION=$($bpython scripts/tomltool.py -f pants.toml get 'python.interpre CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) SHOW_GUIDE=0 +ENABLE_SSL=0 ENABLE_CUDA=0 ENABLE_CUDA_MOCK=0 CONFIGURE_HA=0 @@ -311,14 +317,31 @@ CODESPACES_POST_CREATE=0 CODESPACES=${CODESPACES:-"false"} _INSTALLED_PYENV=0 -# Only Used in HA mode REDIS_MASTER_PORT="9500" REDIS_SLAVE1_PORT="9501" REDIS_SLAVE2_PORT="9502" + +[[ "$@" =~ "enable-ssl" ]] && REDIS_MASTER_EXPOSED_PORT="9510" || REDIS_MASTER_EXPOSED_PORT="9500" +[[ "$@" =~ "enable-ssl" ]] && REDIS_SLAVE1_EXPOSED_PORT="9511" || REDIS_SLAVE1_EXPOSED_PORT="9501" +[[ "$@" =~ "enable-ssl" ]] && REDIS_SLAVE2_EXPOSED_PORT="9512" || REDIS_SLAVE2_EXPOSED_PORT="9502" + +# Only Used in HA mode + REDIS_SENTINEL1_PORT="9503" REDIS_SENTINEL2_PORT="9504" REDIS_SENTINEL3_PORT="9505" +[[ "$@" =~ "enable-ssl" ]] && REDIS_SENTINEL1_EXPOSED_PORT="9513" || REDIS_SENTINEL1_EXPOSED_PORT="9503" +[[ "$@" =~ "enable-ssl" ]] && REDIS_SENTINEL2_EXPOSED_PORT="9514" || REDIS_SENTINEL2_EXPOSED_PORT="9504" +[[ "$@" =~ "enable-ssl" ]] && REDIS_SENTINEL3_EXPOSED_PORT="9515" || REDIS_SENTINEL3_EXPOSED_PORT="9505" + +# Used in Redis SSL configuration + +REDIS_SSL_CA_CERTS_PATH="${ROOT_PATH}/fixtures/redis/cert.pem" +REDIS_SSL_CERTFILE_PATH="${ROOT_PATH}/fixtures/redis/cert.pem" +REDIS_SSL_KEYFILE_PATH="${ROOT_PATH}/fixtures/redis/key.pem" + + while [ $# -gt 0 ]; do case $1 in -h | --help) usage; exit 1 ;; @@ -351,6 +374,7 @@ while [ $# -gt 0 ]; do --agent-backend) AGENT_BACKEND=$2; shift ;; --agent-backend=*) AGENT_BACKEND="${1#*=}" ;; --configure-ha) CONFIGURE_HA=1 ;; + --enable-ssl) ENABLE_SSL=1 ;; *) echo "Unknown option: $1" echo "Run '$0 --help' for usage." @@ -765,20 +789,24 @@ setup_environment() { # Install postgresql, etcd packages via docker show_info "Creating docker compose configuration file for \"halfstack\"..." mkdir -p "$HALFSTACK_VOLUME_PATH" + if [ $CONFIGURE_HA -eq 1 ]; then - SOURCE_COMPOSE_PATH="docker-compose.halfstack-ha.yml" + show_info "Start to configure HA dev environment..." - cp "${SOURCE_COMPOSE_PATH}" "docker-compose.halfstack.current.yml" + cp "docker-compose.halfstack-ha.yml" "docker-compose.halfstack.current.yml" sed_inplace "s/8100:5432/${POSTGRES_PORT}:5432/" "docker-compose.halfstack.current.yml" - sed_inplace "s/8110:6379/${REDIS_PORT}:6379/" "docker-compose.halfstack.current.yml" - sed_inplace "s/8120:2379/${ETCD_PORT}:2379/" "docker-compose.halfstack.current.yml" - sed_inplace 's/\${REDIS_MASTER_PORT}/9500/g' "docker-compose.halfstack.current.yml" - sed_inplace 's/\${REDIS_SLAVE1_PORT}/9501/g' "docker-compose.halfstack.current.yml" - sed_inplace 's/\${REDIS_SLAVE2_PORT}/9502/g' "docker-compose.halfstack.current.yml" - sed_inplace 's/\${REDIS_SENTINEL1_PORT}/9503/g' "docker-compose.halfstack.current.yml" - sed_inplace 's/\${REDIS_SENTINEL2_PORT}/9504/g' "docker-compose.halfstack.current.yml" - sed_inplace 's/\${REDIS_SENTINEL3_PORT}/9505/g' "docker-compose.halfstack.current.yml" + sed_inplace "s/\${REDIS_MASTER_PORT}/${REDIS_MASTER_PORT}/g" "docker-compose.halfstack.current.yml" + sed_inplace "s/\${REDIS_SLAVE1_PORT}/${REDIS_SLAVE1_PORT}/g" "docker-compose.halfstack.current.yml" + sed_inplace "s/\${REDIS_SLAVE2_PORT}/${REDIS_SLAVE2_PORT}/g" "docker-compose.halfstack.current.yml" + + sed_inplace "s/\${REDIS_MASTER_EXPOSED_PORT}/${REDIS_MASTER_EXPOSED_PORT}/g" "docker-compose.halfstack.current.yml" + sed_inplace "s/\${REDIS_SLAVE1_EXPOSED_PORT}/${REDIS_SLAVE1_EXPOSED_PORT}/g" "docker-compose.halfstack.current.yml" + sed_inplace "s/\${REDIS_SLAVE2_EXPOSED_PORT}/${REDIS_SLAVE2_EXPOSED_PORT}/g" "docker-compose.halfstack.current.yml" + + sed_inplace "s/\${REDIS_SENTINEL1_EXPOSED_PORT}/${REDIS_SENTINEL1_EXPOSED_PORT}/g" "docker-compose.halfstack.current.yml" + sed_inplace "s/\${REDIS_SENTINEL2_EXPOSED_PORT}/${REDIS_SENTINEL2_EXPOSED_PORT}/g" "docker-compose.halfstack.current.yml" + sed_inplace "s/\${REDIS_SENTINEL3_EXPOSED_PORT}/${REDIS_SENTINEL3_EXPOSED_PORT}/g" "docker-compose.halfstack.current.yml" mkdir -p "./tmp/backend.ai-halfstack-ha/configs" @@ -792,23 +820,33 @@ setup_environment() { sed_inplace "s/\${COMPOSE_PATH}/${ROOT_PATH//\//\\/}\/tmp\/backend\.ai-halfstack-ha\/configs\//g" "docker-compose.halfstack.current.yml" - # Appends the given text to sentinel01.conf echo "" >> $sentinel01_cfg_path sed_inplace "s/REDIS_SENTINEL_SELF_HOST/sentinel01/g" "$sentinel01_cfg_path" + sed_inplace "s/REDIS_SENTINEL_SELF_PORT/${REDIS_SENTINEL1_PORT}/g" "$sentinel01_cfg_path" + sed_inplace "s/REDIS_SENTINEL_SELF_EXPOSED_PORT/${REDIS_SENTINEL1_EXPOSED_PORT}/g" "$sentinel01_cfg_path" + sed_inplace "s/REDIS_MASTER_EXPOSED_PORT/${REDIS_MASTER_EXPOSED_PORT}/g" "$sentinel01_cfg_path" sed_inplace "s/REDIS_PASSWORD/develove/g" "$sentinel01_cfg_path" - sed_inplace "s/REDIS_SENTINEL_SELF_PORT/9503/g" "$sentinel01_cfg_path" - # Appends the given text to sentinel02.conf echo "" >> $sentinel02_cfg_path sed_inplace "s/REDIS_SENTINEL_SELF_HOST/sentinel02/g" "$sentinel02_cfg_path" + sed_inplace "s/REDIS_SENTINEL_SELF_PORT/${REDIS_SENTINEL2_PORT}/g" "$sentinel02_cfg_path" + sed_inplace "s/REDIS_SENTINEL_SELF_EXPOSED_PORT/${REDIS_SENTINEL2_EXPOSED_PORT}/g" "$sentinel02_cfg_path" + sed_inplace "s/REDIS_MASTER_EXPOSED_PORT/${REDIS_MASTER_EXPOSED_PORT}/g" "$sentinel02_cfg_path" sed_inplace "s/REDIS_PASSWORD/develove/g" "$sentinel02_cfg_path" - sed_inplace "s/REDIS_SENTINEL_SELF_PORT/9504/g" "$sentinel02_cfg_path" - # Appends the given text to sentinel03.conf echo "" >> $sentinel03_cfg_path sed_inplace "s/REDIS_SENTINEL_SELF_HOST/sentinel03/g" "$sentinel03_cfg_path" + sed_inplace "s/REDIS_SENTINEL_SELF_PORT/${REDIS_SENTINEL3_PORT}/g" "$sentinel03_cfg_path" + sed_inplace "s/REDIS_SENTINEL_SELF_EXPOSED_PORT/${REDIS_SENTINEL3_EXPOSED_PORT}/g" "$sentinel03_cfg_path" + sed_inplace "s/REDIS_MASTER_EXPOSED_PORT/${REDIS_MASTER_EXPOSED_PORT}/g" "$sentinel03_cfg_path" sed_inplace "s/REDIS_PASSWORD/develove/g" "$sentinel03_cfg_path" - sed_inplace "s/REDIS_SENTINEL_SELF_PORT/9505/g" "$sentinel03_cfg_path" + + if [ $ENABLE_SSL -ne 1 ]; then + sed_inplace "/^tls-/s/^/# /" "$sentinel01_cfg_path" + sed_inplace "/^tls-/s/^/# /" "$sentinel02_cfg_path" + sed_inplace "/^tls-/s/^/# /" "$sentinel03_cfg_path" + fi + else SOURCE_COMPOSE_PATH="docker-compose.halfstack-${CURRENT_BRANCH//.}.yml" if [ ! -f "${SOURCE_COMPOSE_PATH}" ]; then @@ -817,14 +855,28 @@ setup_environment() { cp "${SOURCE_COMPOSE_PATH}" "docker-compose.halfstack.current.yml" fi + if [ $ENABLE_SSL -eq 1 ]; then + sed_inplace "s/8110:6379/${REDIS_PORT}:6380/" "docker-compose.halfstack.current.yml" + else + sed_inplace "s/8110:6379/${REDIS_PORT}:6379/" "docker-compose.halfstack.current.yml" + fi + sed_inplace "s/8100:5432/${POSTGRES_PORT}:5432/" "docker-compose.halfstack.current.yml" - sed_inplace "s/8110:6379/${REDIS_PORT}:6379/" "docker-compose.halfstack.current.yml" sed_inplace "s/8120:2379/${ETCD_PORT}:2379/" "docker-compose.halfstack.current.yml" mkdir -p "${HALFSTACK_VOLUME_PATH}/postgres-data" mkdir -p "${HALFSTACK_VOLUME_PATH}/etcd-data" mkdir -p "${HALFSTACK_VOLUME_PATH}/redis-data" + if [ $ENABLE_SSL -eq 1 ]; then + show_info "Generating self-signed certificate..." + mkdir -p "fixtures/redis" + openssl req -x509 -newkey rsa:4096 -keyout "${REDIS_SSL_KEYFILE_PATH}" -out "${REDIS_SSL_CA_CERTS_PATH}" -days 365 -nodes -subj "/C=US/ST=State/L=City/O=Organization/OU=Unit/CN=www.example.com" + chmod +rwx "${REDIS_SSL_KEYFILE_PATH}" "${REDIS_SSL_CA_CERTS_PATH}" + else + sed_inplace "/^ *--tls-/s/^/# /" docker-compose.halfstack.current.yml + fi + $docker_sudo docker compose -f "docker-compose.halfstack.current.yml" pull show_info "Pre-pulling frequently used kernel images..." @@ -857,7 +909,7 @@ configure_backendai() { sed_inplace "s/localhost:8100/localhost:${POSTGRES_PORT}/" ./alembic.ini if [ $CONFIGURE_HA -eq 1 ]; then - ./backend.ai mgr etcd put config/redis/sentinel "127.0.0.1:${REDIS_SENTINEL1_PORT},127.0.0.1:${REDIS_SENTINEL2_PORT},127.0.0.1:${REDIS_SENTINEL3_PORT}" + ./backend.ai mgr etcd put config/redis/sentinel "127.0.0.1:${REDIS_SENTINEL1_EXPOSED_PORT},127.0.0.1:${REDIS_SENTINEL2_EXPOSED_PORT},127.0.0.1:${REDIS_SENTINEL3_EXPOSED_PORT}" ./backend.ai mgr etcd put config/redis/service_name "mymaster" ./backend.ai mgr etcd put config/redis/password "develove" else @@ -866,6 +918,19 @@ configure_backendai() { ./backend.ai mgr etcd put-json config/redis/redis_helper_config ./configs/manager/sample.etcd.redis-helper.json + if [ $ENABLE_SSL -eq 1 ]; then + cp ./configs/manager/sample.etcd.redis-tls.json ./redis-tls.json + + sed_inplace "s|\${SSL_CA_CERTS_PATH}|${REDIS_SSL_CA_CERTS_PATH}|g" ./redis-tls.json + sed_inplace "s|\${SSL_CERTFILE_PATH}|${REDIS_SSL_CERTFILE_PATH}|g" ./redis-tls.json + sed_inplace "s|\${SSL_KEYFILE_PATH}|${REDIS_SSL_KEYFILE_PATH}|g" ./redis-tls.json + + ./backend.ai mgr etcd put-json config/redis ./redis-tls.json + rm -rf ./redis-tls.json + else + ./backend.ai mgr etcd put config/redis/redis_helper_config/ssl False + fi + cp configs/manager/sample.etcd.volumes.json ./dev.etcd.volumes.json MANAGER_AUTH_KEY=$(python -c 'import secrets; print(secrets.token_hex(32), end="")') sed_inplace "s/\"secret\": \"some-secret-shared-with-storage-proxy\"/\"secret\": \"${MANAGER_AUTH_KEY}\"/" ./dev.etcd.volumes.json @@ -881,7 +946,7 @@ configure_backendai() { sed_inplace "s@\(# \)\{0,1\}var-base-path = .*@var-base-path = "'"'"${VAR_BASE_PATH}"'"'"@" ./agent.toml # configure backend mode - if [ $AGENT_BACKEND = "k8s" ] || [ $AGENT_BACKEND = "kubernetes" ]; then + if [ $AGENT_BACKEND = "k8s" ] || [ $AGENT_BACKEND = "kubernetes" ]; then sed_inplace "s/mode = \"docker\"/mode = \"kubernetes\"/" ./agent.toml sed_inplace "s/scratch-type = \"hostdir\"/scratch-type = \"k8s-nfs\"/" ./agent.toml elif [ $AGENT_BACKEND = "docker" ]; then @@ -925,13 +990,31 @@ configure_backendai() { if [ $CONFIGURE_HA -eq 1 ]; then sed_inplace "s/redis.addr = \"localhost:6379\"/# redis.addr = \"localhost:6379\"/" ./webserver.conf - sed_inplace "s/# redis.password = \"mysecret\"/redis.password = \"develove\"/" ./webserver.conf - sed_inplace "s/# redis.service_name = \"mymaster\"/redis.service_name = \"mymaster\"/" ./webserver.conf - sed_inplace "s/# redis.sentinel = \"127.0.0.1:9503,127.0.0.1:9504,127.0.0.1:9505\"/redis.sentinel = \"127.0.0.1:9503,127.0.0.1:9504,127.0.0.1:9505\"/ " ./webserver.conf + sed_inplace "s/^# \(redis\.password =\) \"mysecret\"/\1 \"develove\"/" ./webserver.conf + sed_inplace "s/^# \(redis\.service_name =\) \"mymaster\"/\1 \"mymaster\"/" ./webserver.conf + + if [ $ENABLE_SSL -eq 1 ]; then + sed_inplace "s/# redis.sentinel = \"127.0.0.1:9503,127.0.0.1:9504,127.0.0.1:9505\"/redis.sentinel = \"127.0.0.1:9513,127.0.0.1:9514,127.0.0.1:9515\"/ " ./webserver.conf + else + sed_inplace "s/# redis.sentinel = \"127.0.0.1:9503,127.0.0.1:9504,127.0.0.1:9505\"/redis.sentinel = \"127.0.0.1:9503,127.0.0.1:9504,127.0.0.1:9505\"/ " ./webserver.conf + fi else sed_inplace "s/redis.addr = \"localhost:6379\"/redis.addr = \"localhost:${REDIS_PORT}\"/" ./webserver.conf fi + if [ $ENABLE_SSL -eq 1 ]; then + sed_inplace "s/^# \(redis\.redis_helper_config\.ssl\)/\1/" ./webserver.conf + sed_inplace "s/^# \(redis\.redis_helper_config\.ssl_cert_reqs\)/\1/" ./webserver.conf + + sed_inplace "s|^# \(redis\.redis_helper_config\.ssl_ca_certs =\) \${SSL_CA_CERTS_PATH}|\1 \${REDIS_SSL_CA_CERTS_PATH}|g" ./webserver.conf + sed_inplace "s|^# \(redis\.redis_helper_config\.ssl_certfile =\) \${SSL_CERTFILE_PATH}|\1 \${REDIS_SSL_CERTFILE_PATH}|g" ./webserver.conf + sed_inplace "s|^# \(redis\.redis_helper_config\.ssl_keyfile =\) \${SSL_KEYFILE_PATH}|\1 \${REDIS_SSL_KEYFILE_PATH}|g" ./webserver.conf + + sed_inplace "s|\${SSL_CA_CERTS_PATH}|\"$REDIS_SSL_CA_CERTS_PATH\"|g" ./webserver.conf + sed_inplace "s|\${SSL_CERTFILE_PATH}|\"$REDIS_SSL_CERTFILE_PATH\"|g" ./webserver.conf + sed_inplace "s|\${SSL_KEYFILE_PATH}|\"$REDIS_SSL_KEYFILE_PATH\"|g" ./webserver.conf + fi + # install and configure webui if [ $EDITABLE_WEBUI -eq 1 ]; then install_editable_webui diff --git a/src/ai/backend/common/config.py b/src/ai/backend/common/config.py index 942996978b..bf1bc82f2a 100644 --- a/src/ai/backend/common/config.py +++ b/src/ai/backend/common/config.py @@ -46,6 +46,11 @@ "socket_timeout": 5.0, "socket_connect_timeout": 2.0, "reconnect_poll_timeout": 0.3, + "ssl": None, + "ssl_ca_certs": None, + "ssl_cert_reqs": None, + "ssl_certfile": None, + "ssl_keyfile": None, } redis_helper_config_iv = t.Dict( @@ -53,6 +58,11 @@ t.Key("socket_timeout", default=5.0): t.Float, t.Key("socket_connect_timeout", default=2.0): t.Float, t.Key("reconnect_poll_timeout", default=0.3): t.Float, + t.Key("ssl", default=False): t.ToBool, + t.Key("ssl_keyfile", default=None): t.Null | t.String, + t.Key("ssl_ca_certs", default=None): t.Null | t.String, + t.Key("ssl_cert_reqs", default=None): t.Null | t.Enum("required", "optional", "none"), + t.Key("ssl_certfile", default=None): t.Null | t.String, } ).allow_extra("*") diff --git a/src/ai/backend/common/redis_helper.py b/src/ai/backend/common/redis_helper.py index 55a27a7aaf..05950eaf8b 100644 --- a/src/ai/backend/common/redis_helper.py +++ b/src/ai/backend/common/redis_helper.py @@ -462,6 +462,20 @@ def get_redis_object( redis_helper_config: RedisHelperConfig = cast( RedisHelperConfig, redis_config.get("redis_helper_config") ) + + use_ssl = redis_helper_config.get("ssl", False) + + ssl_configs = ( + { + "ssl_cert_reqs": redis_helper_config.get("ssl_cert_reqs", None), + "ssl_ca_certs": redis_helper_config.get("ssl_ca_certs", None), + "ssl_certfile": redis_helper_config.get("ssl_certfile", None), + "ssl_keyfile": redis_helper_config.get("ssl_keyfile", None), + } + if use_ssl + else {} + ) + conn_opts = { **_default_conn_opts, **kwargs, @@ -480,6 +494,7 @@ def get_redis_object( # for redis-py 5.0+ # if connection_ready_timeout := redis_helper_config.get("connection_ready_timeout"): # conn_pool_opts["timeout"] = float(connection_ready_timeout) + if _sentinel_addresses := redis_config.get("sentinel"): sentinel_addresses: Any = None if isinstance(_sentinel_addresses, str): @@ -499,10 +514,15 @@ def get_redis_object( [(str(host), port) for host, port in sentinel_addresses], password=password, db=str(db), + ssl=use_ssl, sentinel_kwargs={ "password": password, + "ssl": use_ssl, + **ssl_configs, **kwargs, }, + min_other_sentinels=0, + **ssl_configs, ) return RedisConnectionInfo( client=sentinel.master_for( @@ -518,15 +538,19 @@ def get_redis_object( else: redis_url = redis_config.get("addr") assert redis_url is not None - url = yarl.URL("redis://host").with_host(str(redis_url[0])).with_port( + scheme = "rediss" if use_ssl else "redis" + url = yarl.URL(f"{scheme}://host").with_host(str(redis_url[0])).with_port( redis_url[1] ).with_password(redis_config.get("password")) / str(db) + return RedisConnectionInfo( # In redis-py 5.0.1+, we should migrate to `Redis.from_pool()` API client=Redis( connection_pool=ConnectionPool.from_url( str(url), **conn_pool_opts, + **kwargs, + **ssl_configs, ), **conn_opts, auto_close_connection_pool=True, diff --git a/src/ai/backend/common/types.py b/src/ai/backend/common/types.py index feb2e1cecc..ec81791283 100644 --- a/src/ai/backend/common/types.py +++ b/src/ai/backend/common/types.py @@ -1114,6 +1114,11 @@ class RedisHelperConfig(TypedDict, total=False): reconnect_poll_timeout: float max_connections: int connection_ready_timeout: float + ssl: Optional[bool] + ssl_ca_certs: Optional[str] + ssl_cert_reqs: Optional[str] + ssl_certfile: Optional[str] + ssl_keyfile: Optional[str] @attrs.define(auto_attribs=True)