diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml index 7aa6554095b9..4cde0dfe65d0 100644 --- a/.github/workflows/cli.yml +++ b/.github/workflows/cli.yml @@ -53,6 +53,7 @@ jobs: - name: Linux launch common services run: | project_compose_ci=ci/pod/docker-compose.common.yml make ci-env-up + sudo ./ci/init-common-test-service.sh - name: Linux Before install run: sudo ./ci/${{ matrix.job_name }}_runner.sh before_install diff --git a/apisix/admin/init.lua b/apisix/admin/init.lua index 333c798e6ada..f8c47a718451 100644 --- a/apisix/admin/init.lua +++ b/apisix/admin/init.lua @@ -21,6 +21,8 @@ local route = require("apisix.utils.router") local plugin = require("apisix.plugin") local v3_adapter = require("apisix.admin.v3_adapter") local utils = require("apisix.admin.utils") +local vault = require("apisix.secret.vault") +local try_fetch_secret = require("apisix.core.config_util").try_fetch_secret local ngx = ngx local get_method = ngx.req.get_method local ngx_time = ngx.time @@ -64,6 +66,7 @@ local resources = { local _M = {version = 0.4} local router +local vault_conf local function check_token(ctx) @@ -87,7 +90,7 @@ local function check_token(ctx) local admin for i, row in ipairs(admin_key) do - if req_token == row.key then + if req_token == try_fetch_secret(vault, vault_conf, row.key) then admin = row break end @@ -452,6 +455,11 @@ function _M.init_worker() return end + vault_conf = local_conf.deployment.secret_vault + if vault_conf and not vault_conf.enable then + vault_conf = nil + end + router = route.new(uri_route) events = require("resty.worker.events") diff --git a/apisix/cli/file.lua b/apisix/cli/file.lua index 149c4e913c35..f1f86f9c0a63 100644 --- a/apisix/cli/file.lua +++ b/apisix/cli/file.lua @@ -27,11 +27,13 @@ local getmetatable = getmetatable local getenv = os.getenv local str_gmatch = string.gmatch local str_find = string.find +local str_byte = string.byte local str_sub = string.sub local print = print local _M = {} local exported_vars +local PREFIX = "$secret://" function _M.get_exported_vars() @@ -142,6 +144,31 @@ end _M.resolve_conf_var = resolve_conf_var +local function is_secret_uri(secret_uri) + if not secret_uri or type(secret_uri) ~= "string" then + return false + end + + if str_byte(secret_uri, 1, 1) ~= str_byte('$') or + not str_find(secret_uri, PREFIX, 1, true) then + return false + end + + return true +end + + +_M.is_secret_uri = is_secret_uri + + +local function resolve_secret_uri(uri) + return str_sub(uri, #PREFIX + 1) +end + + +_M.resolve_secret_uri = resolve_secret_uri + + local function replace_by_reserved_env_vars(conf) -- TODO: support more reserved environment variables local v = getenv("APISIX_DEPLOYMENT_ETCD_HOST") diff --git a/apisix/cli/ngx_tpl.lua b/apisix/cli/ngx_tpl.lua index ab8407b572ec..f774ba540646 100644 --- a/apisix/cli/ngx_tpl.lua +++ b/apisix/cli/ngx_tpl.lua @@ -559,6 +559,16 @@ http { set $upstream_host $http_host; set $upstream_uri ''; + {% if enable_config_secret then %} + ssl_certificate_by_lua_block { + apisix.http_admin_ssl_phase() + } + + access_by_lua_block { + apisix.http_admin_access_phase() + } + {% end %} + location /apisix/admin { {%if allow_admin then%} {% for _, allow_ip in ipairs(allow_admin) do %} diff --git a/apisix/cli/ops.lua b/apisix/cli/ops.lua index 8ba08c7fa974..a0b6f63ed11e 100644 --- a/apisix/cli/ops.lua +++ b/apisix/cli/ops.lua @@ -181,6 +181,14 @@ local function init(env) util.die(err, "\n") end + local enable_config_secret = false + do + local vault_conf = yaml_conf.deployment.secret_vault + if vault_conf and vault_conf.enable then + enable_config_secret = true + end + end + -- check the Admin API token local checked_admin_key = false local allow_admin = yaml_conf.deployment.admin and @@ -245,6 +253,12 @@ Please modify "admin_key" in conf/config.yaml . then util.die("missing ssl cert for https admin") end + + if enable_config_secret then + admin_api_mtls.admin_ssl_cert = "cert/ssl_PLACE_HOLDER.crt" + admin_api_mtls.admin_ssl_cert_key = "cert/ssl_PLACE_HOLDER.key" + admin_api_mtls.admin_ssl_ca_cert = "" + end end if yaml_conf.apisix.enable_admin and @@ -559,6 +573,7 @@ Please modify "admin_key" in conf/config.yaml . error_log = {level = "warn"}, enable_http = enable_http, enable_stream = enable_stream, + enable_config_secret = enable_config_secret, enabled_discoveries = enabled_discoveries, enabled_plugins = enabled_plugins, enabled_stream_plugins = enabled_stream_plugins, diff --git a/apisix/cli/schema.lua b/apisix/cli/schema.lua index 3684232f1a7f..77eb2e10353f 100644 --- a/apisix/cli/schema.lua +++ b/apisix/cli/schema.lua @@ -363,11 +363,31 @@ local admin_schema = { } } +local secret_vault_schema = { + type = "object", + properties = { + enable = { + type = "boolean", + }, + uri = { + type = "string" + }, + prefix = { + type = "string" + }, + token = { + type = "string" + }, + }, + required = {"enable", "uri", "prefix", "token"} +} + local deployment_schema = { traditional = { properties = { etcd = etcd_schema, admin = admin_schema, + secret_vault = secret_vault_schema, role_traditional = { properties = { config_provider = { @@ -383,6 +403,7 @@ local deployment_schema = { properties = { etcd = etcd_schema, admin = admin_schema, + secret_vault = secret_vault_schema, role_control_plane = { properties = { config_provider = { diff --git a/apisix/core/config_util.lua b/apisix/core/config_util.lua index 7e57ed402fd8..e64b7b035034 100644 --- a/apisix/core/config_util.lua +++ b/apisix/core/config_util.lua @@ -21,6 +21,9 @@ local core_tab = require("apisix.core.table") local log = require("apisix.core.log") +local file = require("apisix.cli.file") +local is_secret_uri = file.is_secret_uri +local resolve_secret_uri = file.resolve_secret_uri local str_byte = string.byte local str_char = string.char local ipairs = ipairs @@ -212,4 +215,19 @@ function _M.parse_time_unit(s) end +function _M.try_fetch_secret(sm, conf, uri) + if not conf or not is_secret_uri(uri) then + return uri + end + + local value, err = sm.get(conf, resolve_secret_uri(uri)) + if err then + log.error("failed to fetch secret config: ", err) + return uri + end + + return value +end + + return _M diff --git a/apisix/init.lua b/apisix/init.lua index 86b68cf62208..69c0add7d4fa 100644 --- a/apisix/init.lua +++ b/apisix/init.lua @@ -27,6 +27,7 @@ require("jit.opt").start("minstitch=2", "maxtrace=4000", require("apisix.patch").patch() local core = require("apisix.core") +local try_fetch_secret = require("apisix.core.config_util").try_fetch_secret local conf_server = require("apisix.conf_server") local plugin = require("apisix.plugin") local plugin_config = require("apisix.plugin_config") @@ -38,6 +39,7 @@ local get_var = require("resty.ngxvar").fetch local router = require("apisix.router") local apisix_upstream = require("apisix.upstream") local apisix_secret = require("apisix.secret") +local vault = require("apisix.secret.vault") local set_upstream = apisix_upstream.set_by_route local apisix_ssl = require("apisix.ssl") local apisix_global_rules = require("apisix.global_rules") @@ -47,6 +49,7 @@ local ctxdump = require("resty.ctxdump") local debug = require("apisix.debug") local pubsub_kafka = require("apisix.pubsub.kafka") local ngx = ngx +local ngx_ssl = require("ngx.ssl") local get_method = ngx.req.get_method local ngx_exit = ngx.exit local math = math @@ -76,6 +79,8 @@ end local load_balancer local local_conf +local vault_conf +local admin_api_mtls local ver_header = "APISIX/" .. core.version.VERSION local has_mod, apisix_ngx_client = pcall(require, "resty.apisix.client") @@ -166,6 +171,13 @@ function _M.http_init_worker() if local_conf.apisix and local_conf.apisix.enable_server_tokens == false then ver_header = "APISIX" end + + vault_conf = local_conf.deployment.secret_vault + if vault_conf and not vault_conf.enable then + vault_conf = nil + end + + admin_api_mtls = local_conf.deployment.admin.admin_api_mtls end @@ -931,6 +943,75 @@ local function add_content_type() core.response.set_header("Content-Type", "application/json") end + +function _M.http_admin_ssl_phase() + local cert = try_fetch_secret(vault, vault_conf, admin_api_mtls.admin_ssl_cert) + local parsed_cert, err = ngx_ssl.parse_pem_cert(cert) + if err then + core.log.error("failed to fetch ssl config: ", err) + ngx_exit(-1) + end + + local ok, err = ngx_ssl.set_cert(parsed_cert) + if not ok then + core.log.error("failed to set PEM cert: " .. err) + ngx_exit(-1) + end + + local pkey = try_fetch_secret(vault, vault_conf, admin_api_mtls.admin_ssl_cert_key) + local parsed_pkey, err = ngx_ssl.parse_pem_priv_key(pkey) + if err then + core.log.error("failed to fetch ssl config: ", err) + ngx_exit(-1) + end + + ok, err = ngx_ssl.set_priv_key(parsed_pkey) + if not ok then + core.log.error("failed to set PEM priv key: " .. err) + ngx_exit(-1) + end + + if admin_api_mtls.admin_ssl_ca_cert and apisix_ssl.support_client_verification() then + local ca_cert = try_fetch_secret(vault, vault_conf, admin_api_mtls.admin_ssl_ca_cert) + local parsed_ca_cert, err = ngx_ssl.parse_pem_cert(ca_cert) + if err then + core.log.error("failed to fetch ssl config: ", err) + ngx_exit(-1) + end + + ok, err = ngx_ssl.verify_client(parsed_ca_cert, nil, true) + if not ok then + core.log.error("failed to verify client cert: ", err) + ngx_exit(-1) + end + end +end + + +function _M.http_admin_access_phase() + if not admin_api_mtls.admin_ssl_ca_cert or not apisix_ssl.support_client_verification() then + return + end + + local api_ctx = core.tablepool.fetch("api_ctx", 0, 32) + + core.ctx.set_vars_meta(api_ctx) + local res = api_ctx.var.ssl_client_verify + + core.tablepool.release("api_ctx", api_ctx) + + if res ~= "SUCCESS" then + if res == "NONE" then + core.log.error("client certificate was not present") + else + core.log.error("client certificate verification is not passed: ", res) + end + + return core.response.exit(400) + end +end + + do local router diff --git a/ci/linux_apisix_current_luarocks_runner.sh b/ci/linux_apisix_current_luarocks_runner.sh index a8836f43b691..152af8280758 100755 --- a/ci/linux_apisix_current_luarocks_runner.sh +++ b/ci/linux_apisix_current_luarocks_runner.sh @@ -26,6 +26,9 @@ do_install() { ./ci/linux-install-openresty.sh ./utils/linux-install-luarocks.sh ./ci/linux-install-etcd-client.sh + + # install vault cli capabilities + install_vault_cli } script() { diff --git a/ci/linux_apisix_master_luarocks_runner.sh b/ci/linux_apisix_master_luarocks_runner.sh index 3e99baf34116..1f00b3509f40 100755 --- a/ci/linux_apisix_master_luarocks_runner.sh +++ b/ci/linux_apisix_master_luarocks_runner.sh @@ -54,7 +54,7 @@ script() { sudo PATH=$PATH apisix quit for i in {1..10} do - if [ ! -f /usr/local/apisix/logs/nginx.pid ];then + if [ ! -f /usr/local/apisix/logs/nginx.pid ]; then break fi sleep 0.3 diff --git a/ci/pod/docker-compose.common.yml b/ci/pod/docker-compose.common.yml index 222dc1e1eed4..05bf5761e05b 100644 --- a/ci/pod/docker-compose.common.yml +++ b/ci/pod/docker-compose.common.yml @@ -101,4 +101,6 @@ services: environment: VAULT_DEV_ROOT_TOKEN_ID: root VAULT_DEV_LISTEN_ADDRESS: 0.0.0.0:8200 + volumes: + - ./t/certs:/certs command: [ "vault", "server", "-dev" ] diff --git a/t/admin/api.t b/t/admin/api.t index 982becc79e7a..c12adeb486f6 100644 --- a/t/admin/api.t +++ b/t/admin/api.t @@ -236,3 +236,34 @@ GET /apisix/admin/routes --- error_code: 200 --- error_log Admin key is bypassed! + + + +=== TEST 16: store api key into vault +--- exec +VAULT_TOKEN='root' VAULT_ADDR='http://0.0.0.0:8200' vault kv put kv/apisix/apisix_config admin_key=value +--- response_body +Success! Data written to: kv/apisix/apisix_config + + + +=== TEST 17: Access with api key from vault +--- main_config +env VAULT_TOKEN=root; +--- yaml_config +deployment: + admin: + admin_key: + - key: "$secret://apisix_config/admin_key" + name: a + role: admin + secret_vault: + enable: true + uri: "http://127.0.0.1:8200" + prefix: "kv/apisix" + token: "${{VAULT_TOKEN}}" +--- request +GET /apisix/admin/routes +--- more_headers +X-API-KEY: value +--- error_code: 200 diff --git a/t/cli/test_admin_mtls.sh b/t/cli/test_admin_mtls.sh index 7bbad286e416..ad4fa4525d2c 100755 --- a/t/cli/test_admin_mtls.sh +++ b/t/cli/test_admin_mtls.sh @@ -38,6 +38,13 @@ make run sleep 1 +if [ -e ./logs/nginx.pid ] || [ -e /home/runner/work/apisix/apisix/logs/nginx.pid ]; then + echo "run APISIX success" +else + echo "failed: failed to run APISIX" + exit 1 +fi + # correct certs code=$(curl -i -o /dev/null -s -w %{http_code} --cacert ./t/certs/mtls_ca.crt --key ./t/certs/mtls_client.key --cert ./t/certs/mtls_client.crt -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' https://admin.apisix.dev:9180/apisix/admin/routes) if [ ! "$code" -eq 200 ]; then @@ -53,3 +60,45 @@ if [ ! "$code" -eq 400 ]; then fi echo "passed: enabled mTLS for admin" + +# re-try with vault + +make stop + +export VAULT_TOKEN="root" +docker exec -i vault sh -c "VAULT_TOKEN='root' VAULT_ADDR='http://0.0.0.0:8200' vault kv put kv/apisix/apisix_config admin_ssl_cert=@/certs/mtls_server.crt admin_ssl_cert_key=@/certs/mtls_server.key admin_ssl_ca_cert=@/certs/mtls_ca.crt" + +echo ' +deployment: + admin: + admin_listen: + port: 9180 + https_admin: true + admin_api_mtls: + admin_ssl_cert: "$secret://apisix_config/admin_ssl_cert" + admin_ssl_cert_key: "$secret://apisix_config/admin_ssl_cert_key" + admin_ssl_ca_cert: "$secret://apisix_config/admin_ssl_ca_cert" + secret_vault: + enable: true + uri: "http://127.0.0.1:8200" + prefix: "kv/apisix" + token: "${{VAULT_TOKEN}}" +' > conf/config.yaml + +make run + +sleep 1 + +if [ -e ./logs/nginx.pid ] || [ -e /home/runner/work/apisix/apisix/logs/nginx.pid ]; then + echo "run APISIX with vault success" +else + echo "failed: failed to run APISIX with vault" + exit 1 +fi + +# correct certs +code=$(curl -i -o /dev/null -s -w %{http_code} --cacert ./t/certs/mtls_ca.crt --key ./t/certs/mtls_client.key --cert ./t/certs/mtls_client.crt -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' https://admin.apisix.dev:9180/apisix/admin/routes) +if [ ! "$code" -eq 200 ]; then + echo "failed: failed to enabled mTLS for admin with vault" + exit 1 +fi diff --git a/t/core/config_util.t b/t/core/config_util.t index 6d9e1e2f8fce..3188052d02b1 100644 --- a/t/core/config_util.t +++ b/t/core/config_util.t @@ -117,3 +117,38 @@ __DATA__ qr/fire \w+/ --- grep_error_log_out eval "fire one\nfire two\n" x 3 + + + +=== TEST 3: store api key into vault +--- exec +VAULT_TOKEN='root' VAULT_ADDR='http://0.0.0.0:8200' vault kv put kv/apisix/apisix_config admin_key=value +--- response_body +Success! Data written to: kv/apisix/apisix_config + + + +=== TEST 4: try_fetch_secret +--- config + location /t { + content_by_lua_block { + local try_fetch_secret = require("apisix.core.config_util").try_fetch_secret + local vault = require("apisix.secret.vault") + + local raw_uri = "$secret://apisix_config/admin_key" + local invalid_raw_uri = "$invalid://foo/bar" + local res = try_fetch_secret(vault, nil, raw_uri) + assert(raw_uri == res) + + local vault_conf = { + uri = "http://127.0.0.1:8200", + prefix = "kv/apisix", + token = "root" + } + res = try_fetch_secret(vault, vault_conf, invalid_raw_uri) + assert(invalid_raw_uri == res) + + res = try_fetch_secret(vault, vault_conf, raw_uri) + assert("value" == res) + } + }