feat(secret): support the aws string value

HuanXin-Chen · Aug 2, 2024 · 4eba7a3 · 4eba7a3
1 parent 09e2dc6
commit 4eba7a3
Show file tree

Hide file tree

Showing 5 changed files with 85 additions and 108 deletions.
diff --git a/apisix/secret/aws.lua b/apisix/secret/aws.lua
@@ -21,7 +21,7 @@ local http = require("resty.http")
 local aws = require("resty.aws")
 
 local sub = core.string.sub
-local rfind_char = core.string.rfind_char
+local find = core.string.find
 local env = core.env
 local type = type
 local unpack = unpack
@@ -121,28 +121,29 @@ end
 local function get(conf, key)
     core.log.info("fetching data from aws for key: ", key)
 
-    local idx = rfind_char(key, '/')
-    if not idx then
-        return nil, "error key format, key: " .. key
-    end
+    local idx = find(key, '/')
 
-    local main_key = sub(key, 1, idx - 1)
+    local main_key = idx and sub(key, 1, idx - 1) or key
     if main_key == "" then
         return nil, "can't find main key, key: " .. key
     end
 
-    local sub_key = sub(key, idx + 1)
-    if sub_key == "" then
-        return nil, "can't find sub key, key: " .. key
+    local sub_key = idx and sub(key, idx + 1) or nil
+    if not sub_key then
+        core.log.info("main: ", main_key)
+    else
+        core.log.info("main: ", main_key, " sub: ", sub_key)
     end
 
-    core.log.info("main: ", main_key, " sub: ", sub_key)
-
     local res, err = make_request_to_aws(conf, main_key)
     if not res then
         return nil, "failed to retrtive data from aws secret manager: " .. err
     end
 
+    if not sub_key then
+        return res
+    end
+
     local ret = core.json.decode(res)
     if not ret then
         return nil, "failed to decode result, res: " .. res

diff --git a/ci/init-common-test-service.sh b/ci/init-common-test-service.sh
@@ -22,4 +22,6 @@ docker exec -i vault sh -c "VAULT_TOKEN='root' VAULT_ADDR='http://0.0.0.0:8200'
 
 # prepare localstack
 sleep 3s
-docker exec -i localstack sh -c "awslocal secretsmanager create-secret --name /apisix-key/jack --description 'APISIX Secret' --secret-string '{\"key\":\"value\"}'"
+docker exec -i localstack sh -c "awslocal secretsmanager create-secret --name apisix-key --description 'APISIX Secret' --secret-string '{\"jack\":\"value\"}'"
+sleep 3s
+docker exec -i localstack sh -c "awslocal secretsmanager create-secret --name apisix-mysql --description 'APISIX Secret' --secret-string 'secret'"
diff --git a/docs/en/latest/terminology/secret.md b/docs/en/latest/terminology/secret.md
@@ -38,7 +38,7 @@ Its working principle is shown in the figure:
 APISIX currently supports storing secrets in the following ways:
 
 - [Environment Variables](#use-environment-variables-to-manage-secrets)
-- [HashiCorp Vault](#use-vault-to-manage-secrets)
+- [HashiCorp Vault](#use-hashicorp-vault-to-manage-secrets)
 - [AWS Secrets Manager](#use-aws-secrets-manager-to-manage-secrets)
 
 You can use APISIX Secret functions by specifying format variables in the consumer configuration of the following plugins, such as `key-auth`.
@@ -207,7 +207,7 @@ $secret://$manager/$id/$secret_name/$key
 - manager: secrets management service, could be the HashiCorp Vault, AWS, etc.
 - id: APISIX Secrets resource ID, which needs to be consistent with the one specified when adding the APISIX Secrets resource
 - secret_name: the secret name in the secrets management service
-- key: the key corresponding to the secret in the secrets management service
+- key: get the value of a property when the value of the secret is a JSON string
 
 ### Required Parameters
 

diff --git a/docs/zh/latest/terminology/secret.md b/docs/zh/latest/terminology/secret.md
@@ -210,7 +210,7 @@ $secret://$manager/$id/$secret_name/$key
 - manager: 密钥管理服务，可以是 Vault、AWS 等
 - APISIX Secret 资源 ID，需要与添加 APISIX Secret 资源时指定的 ID 保持一致
 - secret_name: 密钥管理服务中的密钥名称
-- key：密钥管理服务中密钥对应的 key
+- key：当密钥的值是 JSON 字符串时，获取某个属性的值
 
 ### 相关参数
 

diff --git a/t/secret/aws.t b/t/secret/aws.t
@@ -33,30 +33,53 @@ run_tests;
 
 __DATA__
 
-=== TEST 1: check key: error format
+=== TEST 1: sanity
+--- request
+GET /t
 --- config
     location /t {
         content_by_lua_block {
-            local aws = require("apisix.secret.aws")
-            local conf = {
-                endpoint_url = "http://127.0.0.1:4566",
-                region = "us-east-1",
-                access_key_id = "access",
-                secret_access_key = "secret",
-                session_token = "token",
+            local test_case = {
+                {},
+                {access_key_id = "access"},
+                {secret_access_key = "secret"},
+                {access_key_id = "access", secret_access_key = 1234},
+                {access_key_id = 1234, secret_access_key = "secret"},
+                {access_key_id = "access", secret_access_key = "secret"},
+                {access_key_id = "access", secret_access_key = "secret", session_token = "token"},
+                {access_key_id = "access", secret_access_key = "secret", session_token = 1234},
+                {access_key_id = "access", secret_access_key = "secret", region = "us-east-1"},
+                {access_key_id = "access", secret_access_key = "secret", region = 1234},
+                {access_key_id = "access", secret_access_key = "secret", endpoint_url = "http://127.0.0.1:4566"},
+                {access_key_id = "access", secret_access_key = "secret", endpoint_url = 1234},
+                {access_key_id = "access", secret_access_key = "secret", session_token = "token", endpoint_url = "http://127.0.0.1:4566", region = "us-east-1"},
             }
-            local data, err = aws.get(conf, "apisix")
-            if err then
-                return ngx.say(err)
+            local aws = require("apisix.secret.aws")
+            local core = require("apisix.core")
+            local metadata_schema = aws.schema
+
+            for _, conf in ipairs(test_case) do
+                local ok, err = core.schema.check(metadata_schema, conf)
+                ngx.say(ok and "done" or err)
             end
-
-            ngx.say("done")
         }
     }
---- request
-GET /t
+--- timeout
+6
 --- response_body
-error key format, key: apisix
+property "access_key_id" is required
+property "secret_access_key" is required
+property "access_key_id" is required
+property "secret_access_key" validation failed: wrong type: expected string, got number
+property "access_key_id" validation failed: wrong type: expected string, got number
+done
+done
+property "session_token" validation failed: wrong type: expected string, got number
+done
+property "region" validation failed: wrong type: expected string, got number
+done
+property "endpoint_url" validation failed: wrong type: expected string, got number
+done
 
 
 
@@ -87,77 +110,76 @@ can't find main key, key: /apisix
 
 
 
-=== TEST 3: check key: no sub key
+=== TEST 3: error aws endpoint_url
 --- config
     location /t {
         content_by_lua_block {
             local aws = require("apisix.secret.aws")
             local conf = {
-                endpoint_url = "http://127.0.0.1:4566",
+                endpoint_url = "http://127.0.0.1:8080",
                 region = "us-east-1",
                 access_key_id = "access",
                 secret_access_key = "secret",
                 session_token = "token",
             }
-            local data, err = aws.get(conf, "apisix/")
+            local data, err = aws.get(conf, "apisix-key/jack")
             if err then
                 return ngx.say(err)
             end
-
             ngx.say("done")
         }
     }
 --- request
 GET /t
 --- response_body
-can't find sub key, key: apisix/
+failed to retrtive data from aws secret manager: SecretsManager:getSecretValue() failed to connect to 'http://127.0.0.1:8080': connection refused
+--- timeout: 6
 
 
 
-=== TEST 4: error aws endpoint_url
+=== TEST 4: get value from aws (err region, status ~= 200)
 --- config
     location /t {
         content_by_lua_block {
             local aws = require("apisix.secret.aws")
             local conf = {
-                endpoint_url = "http://127.0.0.1:8080",
-                region = "us-east-1",
+                endpoint_url = "http://127.0.0.1:4566",
+                region = "error-region",
                 access_key_id = "access",
                 secret_access_key = "secret",
                 session_token = "token",
             }
-            local data, err = aws.get(conf, "/apisix-key/jack/key")
+            local data, err = aws.get(conf, "apisix-key/jack")
             if err then
-                return ngx.say(err)
+                return ngx.say("err")
             end
             ngx.say("done")
         }
     }
 --- request
 GET /t
 --- response_body
-failed to retrtive data from aws secret manager: SecretsManager:getSecretValue() failed to connect to 'http://127.0.0.1:8080': connection refused
---- timeout: 6
+err
 
 
 
-=== TEST 5: get value from aws (err region, status ~= 200)
+=== TEST 5: get value from aws (err key, status ~= 200)
 --- config
     location /t {
         content_by_lua_block {
             local aws = require("apisix.secret.aws")
             local conf = {
                 endpoint_url = "http://127.0.0.1:4566",
-                region = "error-region",
+                region = "us-east-1",
                 access_key_id = "access",
                 secret_access_key = "secret",
                 session_token = "token",
             }
-            local data, err = aws.get(conf, "/apisix-key/jack/key")
+            local data, err = aws.get(conf, "apisix-error-key/jack")
             if err then
                 return ngx.say("err")
             end
-            ngx.say("done")
+            ngx.say("value")
         }
     }
 --- request
@@ -167,7 +189,7 @@ err
 
 
 
-=== TEST 6: get value from aws (err key, status ~= 200)
+=== TEST 6: get json value from aws
 --- config
     location /t {
         content_by_lua_block {
@@ -179,33 +201,33 @@ err
                 secret_access_key = "secret",
                 session_token = "token",
             }
-            local data, err = aws.get(conf, "/apisix-key/jack-error/key")
+            local data, err = aws.get(conf, "apisix-key/jack")
             if err then
-                return ngx.say("err")
+                return ngx.say(err)
             end
             ngx.say("value")
         }
     }
 --- request
 GET /t
 --- response_body
-err
+value
 
 
 
-=== TEST 7: get value from aws
+=== TEST 7: get json value from aws using env var
 --- config
     location /t {
         content_by_lua_block {
             local aws = require("apisix.secret.aws")
             local conf = {
                 endpoint_url = "http://127.0.0.1:4566",
                 region = "us-east-1",
-                access_key_id = "access",
-                secret_access_key = "secret",
-                session_token = "token",
+                access_key_id = "$ENV://AWS_ACCESS_KEY_ID",
+                secret_access_key = "$ENV://AWS_SECRET_ACCESS_KEY",
+                session_token = "$ENV://AWS_SESSION_TOKEN",
             }
-            local data, err = aws.get(conf, "/apisix-key/jack/key")
+            local data, err = aws.get(conf, "apisix-key/jack")
             if err then
                 return ngx.say(err)
             end
@@ -219,7 +241,7 @@ value
 
 
 
-=== TEST 8: get value from aws using env var
+=== TEST 8: get string value from aws 
 --- config
     location /t {
         content_by_lua_block {
@@ -231,17 +253,17 @@ value
                 secret_access_key = "$ENV://AWS_SECRET_ACCESS_KEY",
                 session_token = "$ENV://AWS_SESSION_TOKEN",
             }
-            local data, err = aws.get(conf, "/apisix-key/jack/key")
+            local data, err = aws.get(conf, "apisix-mysql")
             if err then
                 return ngx.say(err)
             end
-            ngx.say("value")
+            ngx.say(data)
         }
     }
 --- request
 GET /t
 --- response_body
-value
+secret
 
 
 
@@ -319,51 +341,3 @@ GET /t
     }
 --- response_body
 nil
-
-
-
-=== TEST 10: sanity
---- request
-GET /t
---- config
-    location /t {
-        content_by_lua_block {
-            local test_case = {
-                {},
-                {access_key_id = "access"},
-                {secret_access_key = "secret"},
-                {access_key_id = "access", secret_access_key = 1234},
-                {access_key_id = 1234, secret_access_key = "secret"},
-                {access_key_id = "access", secret_access_key = "secret"},
-                {access_key_id = "access", secret_access_key = "secret", session_token = "token"},
-                {access_key_id = "access", secret_access_key = "secret", session_token = 1234},
-                {access_key_id = "access", secret_access_key = "secret", region = "us-east-1"},
-                {access_key_id = "access", secret_access_key = "secret", region = 1234},
-                {access_key_id = "access", secret_access_key = "secret", endpoint_url = "http://127.0.0.1:4566"},
-                {access_key_id = "access", secret_access_key = "secret", endpoint_url = 1234},
-                {access_key_id = "access", secret_access_key = "secret", session_token = "token", endpoint_url = "http://127.0.0.1:4566", region = "us-east-1"},
-            }
-            local aws = require("apisix.secret.aws")
-            local core = require("apisix.core")
-            local metadata_schema = aws.schema
-
-            for _, conf in ipairs(test_case) do
-                local ok, err = core.schema.check(metadata_schema, conf)
-                ngx.say(ok and "done" or err)
-            end
-        }
-    }
---- response_body
-property "access_key_id" is required
-property "secret_access_key" is required
-property "access_key_id" is required
-property "secret_access_key" validation failed: wrong type: expected string, got number
-property "access_key_id" validation failed: wrong type: expected string, got number
-done
-done
-property "session_token" validation failed: wrong type: expected string, got number
-done
-property "region" validation failed: wrong type: expected string, got number
-done
-property "endpoint_url" validation failed: wrong type: expected string, got number
-done