feat: allow admin key and admin api cert to be stored in vault

apisix/admin/init.lua

@@ -21,6 +21,8 @@ local route = require("apisix.utils.router")
 local plugin = require("apisix.plugin")
 local v3_adapter = require("apisix.admin.v3_adapter")
 local utils = require("apisix.admin.utils")
+local vault = require("apisix.secret.vault")
+local try_fetch_secret = require("apisix.core.config_util").try_fetch_secret
 local ngx = ngx
 local get_method = ngx.req.get_method
 local ngx_time = ngx.time
@@ -64,6 +66,7 @@ local resources = {
 
 local _M = {version = 0.4}
 local router
+local vault_conf
 
 
 local function check_token(ctx)
@@ -87,7 +90,7 @@ local function check_token(ctx)
 
     local admin
     for i, row in ipairs(admin_key) do
-        if req_token == row.key then
+        if req_token == try_fetch_secret(vault, vault_conf, row.key) then
             admin = row
             break
         end
@@ -412,6 +415,11 @@ function _M.init_worker()
         return
     end
 
+    vault_conf = local_conf.deployment.secret_vault
+    if vault_conf and not vault_conf.enable then
+        vault_conf = nil
+    end
+
     router = route.new(uri_route)
     events = require("resty.worker.events")
 

apisix/cli/file.lua

@@ -27,11 +27,13 @@ local getmetatable = getmetatable
 local getenv = os.getenv
 local str_gmatch = string.gmatch
 local str_find = string.find
+local str_byte = string.byte
 local str_sub = string.sub
 local print = print
 
 local _M = {}
 local exported_vars
+local PREFIX = "$secret://"
 
 
 function _M.get_exported_vars()
@@ -142,6 +144,31 @@ end
 _M.resolve_conf_var = resolve_conf_var
 
 
+local function is_secret_uri(secret_uri)
+    if not secret_uri or type(secret_uri) ~= "string" then
+        return false
+    end
+
+    if str_byte(secret_uri, 1, 1) ~= str_byte('$') or
+        not str_find(secret_uri, PREFIX, 1, true) then
+        return false
+    end
+
+    return true
+end
+
+
+_M.is_secret_uri = is_secret_uri
+
+
+local function resolve_secret_uri(uri)
+    return str_sub(uri, #PREFIX + 1)
+end
+
+
+_M.resolve_secret_uri = resolve_secret_uri
+
+
 local function replace_by_reserved_env_vars(conf)
     -- TODO: support more reserved environment variables
     local v = getenv("APISIX_DEPLOYMENT_ETCD_HOST")

apisix/cli/ngx_tpl.lua

@@ -559,6 +559,16 @@ http {
         set $upstream_host               $http_host;
         set $upstream_uri                '';
 
+        {% if enable_config_secret then %}
+        ssl_certificate_by_lua_block {
+            apisix.http_admin_ssl_phase()
+        }
+
+        access_by_lua_block {
+            apisix.http_admin_access_phase()
+        }
+        {% end %}
+
         location /apisix/admin {
             {%if allow_admin then%}
                 {% for _, allow_ip in ipairs(allow_admin) do %}

apisix/cli/ops.lua

@@ -181,6 +181,14 @@ local function init(env)
         util.die(err, "\n")
     end
 
+    local enable_config_secret = false
+    do
+        local vault_conf = yaml_conf.deployment.secret_vault
+        if vault_conf and vault_conf.enable then
+            enable_config_secret = true
+        end
+    end
+
     -- check the Admin API token
     local checked_admin_key = false
     local allow_admin = yaml_conf.deployment.admin and
@@ -245,6 +253,12 @@ Please modify "admin_key" in conf/config.yaml .
         then
             util.die("missing ssl cert for https admin")
         end
+
+        if enable_config_secret then
+            admin_api_mtls.admin_ssl_cert = "cert/ssl_PLACE_HOLDER.crt"
+            admin_api_mtls.admin_ssl_cert_key = "cert/ssl_PLACE_HOLDER.key"
+            admin_api_mtls.admin_ssl_ca_cert = ""
+        end
     end
 
     if yaml_conf.apisix.enable_admin and
@@ -559,6 +573,7 @@ Please modify "admin_key" in conf/config.yaml .
         error_log = {level = "warn"},
         enable_http = enable_http,
         enable_stream = enable_stream,
+        enable_config_secret = enable_config_secret,
         enabled_discoveries = enabled_discoveries,
         enabled_plugins = enabled_plugins,
         enabled_stream_plugins = enabled_stream_plugins,

apisix/cli/schema.lua

@@ -363,11 +363,31 @@ local admin_schema = {
     }
 }
 
+local secret_vault_schema = {
+    type = "object",
+    properties = {
+        enable = {
+            type = "boolean",
+        },
+        uri = {
+            type = "string"
+        },
+        prefix = {
+            type = "string"
+        },
+        token = {
+            type = "string"
+        },
+    },
+    required = {"enable", "uri", "prefix", "token"}
+}
+
 local deployment_schema = {
     traditional = {
         properties = {
             etcd = etcd_schema,
             admin = admin_schema,
+            secret_vault = secret_vault_schema,
             role_traditional = {
                 properties = {
                     config_provider = {
@@ -383,6 +403,7 @@ local deployment_schema = {
         properties = {
             etcd = etcd_schema,
             admin = admin_schema,
+            secret_vault = secret_vault_schema,
             role_control_plane = {
                 properties = {
                     config_provider = {

apisix/core/config_util.lua

@@ -21,6 +21,9 @@
 
 local core_tab = require("apisix.core.table")
 local log = require("apisix.core.log")
+local file = require("apisix.cli.file")
+local is_secret_uri = file.is_secret_uri
+local resolve_secret_uri = file.resolve_secret_uri
 local str_byte = string.byte
 local str_char = string.char
 local ipairs = ipairs
@@ -212,4 +215,19 @@ function _M.parse_time_unit(s)
 end
 
 
+function _M.try_fetch_secret(sm, conf, uri)
+    if not conf or not is_secret_uri(uri) then
+        return uri
+    end
+
+    local value, err = sm.get(conf, resolve_secret_uri(uri))
+    if err then
+        log.error("failed to fetch secret config: ", err)
+        return uri
+    end
+
+    return value
+end
+
+
 return _M

apisix/init.lua

@@ -27,6 +27,7 @@ require("jit.opt").start("minstitch=2", "maxtrace=4000",
 
 require("apisix.patch").patch()
 local core            = require("apisix.core")
+local try_fetch_secret    = require("apisix.core.config_util").try_fetch_secret
 local conf_server     = require("apisix.conf_server")
 local plugin          = require("apisix.plugin")
 local plugin_config   = require("apisix.plugin_config")
@@ -38,6 +39,7 @@ local get_var         = require("resty.ngxvar").fetch
 local router          = require("apisix.router")
 local apisix_upstream = require("apisix.upstream")
 local apisix_secret   = require("apisix.secret")
+local vault           = require("apisix.secret.vault")
 local set_upstream    = apisix_upstream.set_by_route
 local apisix_ssl      = require("apisix.ssl")
 local apisix_global_rules    = require("apisix.global_rules")
@@ -47,6 +49,7 @@ local ctxdump         = require("resty.ctxdump")
 local debug           = require("apisix.debug")
 local pubsub_kafka    = require("apisix.pubsub.kafka")
 local ngx             = ngx
+local ngx_ssl         = require("ngx.ssl")
 local get_method      = ngx.req.get_method
 local ngx_exit        = ngx.exit
 local math            = math
@@ -76,6 +79,8 @@ end
 
 local load_balancer
 local local_conf
+local vault_conf
+local admin_api_mtls
 local ver_header = "APISIX/" .. core.version.VERSION
 
 local has_mod, apisix_ngx_client = pcall(require, "resty.apisix.client")
@@ -166,6 +171,13 @@ function _M.http_init_worker()
     if local_conf.apisix and local_conf.apisix.enable_server_tokens == false then
         ver_header = "APISIX"
     end
+
+    vault_conf = local_conf.deployment.secret_vault
+    if vault_conf and not vault_conf.enable then
+        vault_conf = nil
+    end
+
+    admin_api_mtls = local_conf.deployment.admin.admin_api_mtls
 end
 
 
@@ -931,6 +943,75 @@ local function add_content_type()
     core.response.set_header("Content-Type", "application/json")
 end
 
+
+function _M.http_admin_ssl_phase()
+    local cert = try_fetch_secret(vault, vault_conf, admin_api_mtls.admin_ssl_cert)
+    local parsed_cert, err = ngx_ssl.parse_pem_cert(cert)
+    if err then
+        core.log.error("failed to fetch ssl config: ", err)
+        ngx_exit(-1)
+    end
+
+    local ok, err = ngx_ssl.set_cert(parsed_cert)
+    if not ok then
+        core.log.error("failed to set PEM cert: " .. err)
+        ngx_exit(-1)
+    end
+
+    local pkey = try_fetch_secret(vault, vault_conf, admin_api_mtls.admin_ssl_cert_key)
+    local parsed_pkey, err = ngx_ssl.parse_pem_priv_key(pkey)
+    if err then
+        core.log.error("failed to fetch ssl config: ", err)
+        ngx_exit(-1)
+    end
+
+    ok, err = ngx_ssl.set_priv_key(parsed_pkey)
+    if not ok then
+        core.log.error("failed to set PEM priv key: " .. err)
+        ngx_exit(-1)
+    end
+
+    if admin_api_mtls.admin_ssl_ca_cert and apisix_ssl.support_client_verification() then
+        local ca_cert = try_fetch_secret(vault, vault_conf, admin_api_mtls.admin_ssl_ca_cert)
+        local parsed_ca_cert, err = ngx_ssl.parse_pem_cert(ca_cert)
+        if err then
+            core.log.error("failed to fetch ssl config: ", err)
+            ngx_exit(-1)
+        end
+
+        ok, err = ngx_ssl.verify_client(parsed_ca_cert, nil, true)
+        if not ok then
+            core.log.error("failed to verify client cert: ", err)
+            ngx_exit(-1)
+        end
+    end
+end
+
+
+function _M.http_admin_access_phase()
+    if not admin_api_mtls.admin_ssl_ca_cert or not apisix_ssl.support_client_verification() then
+        return
+    end
+
+    local api_ctx = core.tablepool.fetch("api_ctx", 0, 32)
+
+    core.ctx.set_vars_meta(api_ctx)
+    local res = api_ctx.var.ssl_client_verify
+
+    core.tablepool.release("api_ctx", api_ctx)
+
+    if res ~= "SUCCESS" then
+        if res == "NONE" then
+            core.log.error("client certificate was not present")
+        else
+            core.log.error("client certificate verification is not passed: ", res)
+        end
+
+        return core.response.exit(400)
+    end
+end
+
+
 do
     local router
 

ci/linux_apisix_current_luarocks_runner.sh

@@ -26,6 +26,9 @@ do_install() {
     ./ci/linux-install-openresty.sh
     ./utils/linux-install-luarocks.sh
     ./ci/linux-install-etcd-client.sh
+
+    # install vault cli capabilities
+    install_vault_cli
 }
 
 script() {

t/admin/api.t

@@ -236,3 +236,34 @@ GET /apisix/admin/routes
 --- error_code: 200
 --- error_log
 Admin key is bypassed!
+
+
+
+=== TEST 16: store api key into vault
+--- exec
+VAULT_TOKEN='root' VAULT_ADDR='http://0.0.0.0:8200' vault kv put kv/apisix/apisix_config admin_key=value
+--- response_body
+Success! Data written to: kv/apisix/apisix_config
+
+
+
+=== TEST 17: Access with api key from vault
+--- main_config
+env VAULT_TOKEN=root;
+--- yaml_config
+deployment:
+  admin:
+    admin_key:
+      - key: "$secret://apisix_config/admin_key"
+        name: a
+        role: admin
+  secret_vault:
+    enable: true
+    uri: "http://127.0.0.1:8200"
+    prefix: "kv/apisix"
+    token: "${{VAULT_TOKEN}}"
+--- request
+GET /apisix/admin/routes
+--- more_headers
+X-API-KEY: value
+--- error_code: 200