From 541017dfa9ce92ecd5a86f713f38065a34d75510 Mon Sep 17 00:00:00 2001 From: peng Date: Sun, 22 Mar 2020 16:04:45 +0800 Subject: [PATCH 01/16] support set multipart/form file --- .gitignore | 3 + lib/resty/requests/multipart.lua | 317 +++++++++++++++++++++++++++++++ lib/resty/requests/request.lua | 9 + lib/resty/requests/util.lua | 21 ++ 4 files changed, 350 insertions(+) create mode 100644 lib/resty/requests/multipart.lua diff --git a/.gitignore b/.gitignore index 1d659ca..985830b 100644 --- a/.gitignore +++ b/.gitignore @@ -39,4 +39,7 @@ luac.out *.x86_64 *.hex +#develop env +.idea + t/servroot/* diff --git a/lib/resty/requests/multipart.lua b/lib/resty/requests/multipart.lua new file mode 100644 index 0000000..41c9f5f --- /dev/null +++ b/lib/resty/requests/multipart.lua @@ -0,0 +1,317 @@ +--https://github.com/pytpeng/lua-multipart +local setmetatable = setmetatable +local tostring = tostring +local insert = table.insert +local remove = table.remove +local concat = table.concat +local ipairs = ipairs +local pairs = pairs +local match = string.match +local find = string.find +local sub = string.sub + + +local RANDOM_BOUNDARY = sub(tostring({}), 10) + + +local MultipartData = { RANDOM_BOUNDARY = RANDOM_BOUNDARY} + + +MultipartData.__index = MultipartData + + +setmetatable(MultipartData, { + __call = function (cls, ...) + return cls.new(...) + end, +}) + + +local function is_header(value) + return match(value, "(%S+):%s*(%S+)") +end + + +local function table_size(t) + local res = 0 + + if t then + for _,_ in pairs(t) do + res = res + 1 + end + end + + return res +end + +-- Create a table representation of multipart/data body +-- +-- @param {string} body The multipart/data string body +-- @param {string} boundary The multipart/data boundary +-- @return {table} Lua representation of the body +local function decode(body, boundary) + local result = { + data = {}, + indexes = {}, + } + + if not boundary then + return result + end + + local part_name + local part_index = 1 + local part_headers = {} + local part_value = {} + local part_value_ct = 0 + + local end_boundary_length = boundary and #boundary + 2 + local processing_part_value = false + + local position = 1 + local done = false + + repeat + local s = find(body, "[\r\n]", position) + + local line + + if s then + line = sub(body, position, s - 1) + position = s + 1 + + else + if position == 1 then + line = body + + else + line = sub(body, position) + end + + done = true + end + + if line == "" then + if s and processing_part_value then + part_value_ct = part_value_ct + 1 + part_value[part_value_ct] = sub(body, s, s) + end + + else + if sub(line, 1, 2) == "--" and sub(line, 3, end_boundary_length) == boundary then + processing_part_value = false + + if part_name ~= nil then + if part_value[part_value_ct] == "\n" then + part_value[part_value_ct] = nil + end + + if part_value[part_value_ct - 1] == "\r" then + part_value[part_value_ct - 1] = nil + end + + result.data[part_index] = { + name = part_name, + headers = part_headers, + value = concat(part_value) + } + + result.indexes[part_name] = part_index + + -- Reset fields for the next part + part_headers = {} + part_value = {} + part_value_ct = 0 + part_name = nil + part_index = part_index + 1 + end + + else + --Beginning of part + if not processing_part_value and line:sub(1, 19):lower() == "content-disposition" then + -- Extract part_name + for v in line:gmatch("[^;]+") do + if not is_header(v) then -- If it's not content disposition part + local pos = v:match("^%s*[Nn][Aa][Mm][Ee]=()") + if pos then + local current_value = v:match("^%s*([^=]*)", pos):gsub("%s*$", "") + part_name = sub(current_value, 2, #current_value - 1) + end + end + end + + insert(part_headers, line) + + if s and sub(body, s, s + 3) == "\r\n\r\n" then + processing_part_value = true + position = s + 4 + end + + elseif not processing_part_value and is_header(line) then + insert(part_headers, line) + + if s and sub(body, s, s + 3) == "\r\n\r\n" then + processing_part_value = true + position = s + 4 + end + + else + processing_part_value = true + + -- The value part begins + part_value_ct = part_value_ct + 1 + part_value[part_value_ct] = line + + if s then + part_value_ct = part_value_ct + 1 + part_value[part_value_ct] = sub(body, s, s) + end + end + end + end + + until done + + if part_name ~= nil then + result.data[part_index] = { + name = part_name, + headers = part_headers, + value = concat(part_value) + } + + result.indexes[part_name] = part_index + end + + return result +end + +-- Creates a multipart/data body from a table +-- +-- @param {table} t The table that contains the multipart/data body properties +-- @param {boundary} boundary The multipart/data boundary to use +-- @return {string} The multipart/data string body +local function encode(t, boundary) + if not boundary then + boundary = RANDOM_BOUNDARY + end + + local result = {} + local i = 0 + + for _, v in ipairs(t.data) do + if v.value then + result[i + 1] = "--" + result[i + 2] = boundary + result[i + 3] = "\r\n" + + i = i + 3 + + for _, header in ipairs(v.headers) do + result[i + 1] = header + result[i + 2] = "\r\n" + + i = i + 2 + end + + result[i + 1] = "\r\n" + result[i + 2] = v.value + result[i + 3] = "\r\n" + + i = i + 3 + end + end + + if i == 0 then + return "" + end + + result[i + 1] = "--" + result[i + 2] = boundary + result[i + 3] = "--\r\n" + + return concat(result) +end + + +function MultipartData.new(data, content_type) + local instance = setmetatable({}, MultipartData) + + if content_type then + local boundary = match(content_type, ";%s*boundary=(%S+)") + if boundary then + if (sub(boundary, 1, 1) == '"' and sub(boundary, -1) == '"') or + (sub(boundary, 1, 1) == "'" and sub(boundary, -1) == "'") then + boundary = sub(boundary, 2, -2) + end + + if boundary ~= "" then + instance._boundary = boundary + end + end + end + + instance._data = decode(data or "", instance._boundary) + + return instance +end + + +function MultipartData:get(name) + return self._data.data[self._data.indexes[name]] +end + + +function MultipartData:get_all() + local result = {} + + for k, v in pairs(self._data.indexes) do + result[k] = self._data.data[v].value + end + + return result +end + + +function MultipartData:set_simple(name, value) + if self._data.indexes[name] then + self._data.data[self._data.indexes[name]] = { + name = name, + value = value, + headers = { 'Content-Disposition: form-data; name="' .. name .. '"' } + } + + else + local part_index = table_size(self._data.indexes) + 1 + self._data.indexes[name] = part_index + self._data.data[part_index] = { + name = name, + value = value, + headers = { 'Content-Disposition: form-data; name="' .. name .. '"' } + } + end +end + + +function MultipartData:delete(name) + local index = self._data.indexes[name] + + if index then + remove(self._data.data, index) + self._data.indexes[name] = nil + + -- need to recount index + for key, value in pairs(self._data.indexes) do + if value > index then + self._data.indexes[key] = value - 1 + end + end + end +end + + +function MultipartData:tostring() + return encode(self._data, self._boundary) +end + + +return MultipartData diff --git a/lib/resty/requests/request.lua b/lib/resty/requests/request.lua index ba5447d..5d03820 100644 --- a/lib/resty/requests/request.lua +++ b/lib/resty/requests/request.lua @@ -69,11 +69,20 @@ local function prepare(url_parts, session, config) local content local json = config.json local body = config.body + local files = config.files if json then content = cjson.encode(json) headers["content-length"] = #content headers["content-type"] = "application/json" + elseif files and is_tab(files) then + local content_type = headers["content-type"] + if not content_type then + content_type = "multipart/form-data; boundary="..util.choose_boundary() + end + local multipart_body= util.make_multipart_body(files, content_type) + headers["content-type"] = content_type + content = multipart_body else content = body if is_func(body) then diff --git a/lib/resty/requests/util.lua b/lib/resty/requests/util.lua index 33a02fa..82addd7 100644 --- a/lib/resty/requests/util.lua +++ b/lib/resty/requests/util.lua @@ -5,10 +5,13 @@ local pcall = pcall local pairs = pairs local error = error local rawget = rawget +local ipairs = ipairs +local require = require local setmetatable = setmetatable local lower = string.lower local ngx_gsub = ngx.re.gsub local base64 = ngx.encode_base64 +local Multipart = require("resty.requests.multipart") local _M = { _VERSION = '0.1' } @@ -131,6 +134,9 @@ local function set_config(opts) -- 4) body config.body = opts.body + -- 4.1) files + config.files = opts.files + -- 5) ssl verify config.ssl = opts.ssl @@ -192,6 +198,19 @@ local function set_config(opts) end +local function make_multipart_body(files, content_type) + local m = Multipart("", content_type) + for _, v in ipairs(files) do + m:set_simple(v[1], v[2], v[3], v[4]) + end + return m:tostring() +end + + +local function choose_boundary() + return string.sub(tostring({}), 10) +end + _M.new_tab = new_tab _M.is_str = is_str _M.is_num = is_num @@ -206,5 +225,7 @@ _M.STATE = STATE _M.STATE_NAME = STATE_NAME _M.HTTP10 = HTTP10 _M.HTTP11 = HTTP11 +_M.make_multipart_body = make_multipart_body +_M.choose_boundary = choose_boundary return _M From 0ee96a5a58fbbdc6097f1adf61c9eb23068f52db Mon Sep 17 00:00:00 2001 From: peng Date: Sun, 22 Mar 2020 16:50:10 +0800 Subject: [PATCH 02/16] fix add content-length --- lib/resty/requests/multipart.lua | 21 ++++++++++++++++----- lib/resty/requests/request.lua | 1 + 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/lib/resty/requests/multipart.lua b/lib/resty/requests/multipart.lua index 41c9f5f..4d94fee 100644 --- a/lib/resty/requests/multipart.lua +++ b/lib/resty/requests/multipart.lua @@ -272,12 +272,23 @@ function MultipartData:get_all() end -function MultipartData:set_simple(name, value) +function MultipartData:set_simple(name, value, filename, content_type) + local headers = {'Content-Disposition: form-data; name="' , name , '"'} + if filename then + headers[4] = '; filename="' + headers[5] = filename + headers[6] = '"' + end + if content_type then + headers[7] = "\r\ncontent-type: " + headers[8] = content_type + end + headers = concat(headers) if self._data.indexes[name] then self._data.data[self._data.indexes[name]] = { - name = name, - value = value, - headers = { 'Content-Disposition: form-data; name="' .. name .. '"' } + name = name, + value = value, + headers = {headers} } else @@ -286,7 +297,7 @@ function MultipartData:set_simple(name, value) self._data.data[part_index] = { name = name, value = value, - headers = { 'Content-Disposition: form-data; name="' .. name .. '"' } + headers = {headers} } end end diff --git a/lib/resty/requests/request.lua b/lib/resty/requests/request.lua index 5d03820..48fa94a 100644 --- a/lib/resty/requests/request.lua +++ b/lib/resty/requests/request.lua @@ -82,6 +82,7 @@ local function prepare(url_parts, session, config) end local multipart_body= util.make_multipart_body(files, content_type) headers["content-type"] = content_type + headers["content-length"] = #multipart_body content = multipart_body else content = body From 513cb461b8838a6fc7ecc8512ec1b1dfdaa2cd3a Mon Sep 17 00:00:00 2001 From: peng Date: Sun, 22 Mar 2020 16:57:23 +0800 Subject: [PATCH 03/16] update README --- README.markdown | 1 + lib/resty/requests/multipart.lua | 2 ++ 2 files changed, 3 insertions(+) diff --git a/README.markdown b/README.markdown index 26b1540..bff77fe 100644 --- a/README.markdown +++ b/README.markdown @@ -166,6 +166,7 @@ The third param, an optional Lua table, which contains a number of options: * a Lua string, or * a Lua function, without parameter and returns a piece of data (string) or an empty Lua string to represent EOF, or * a Lua table, each key-value pair will be concatenated with the "&", and Content-Type header will `"application/x-www-form-urlencoded"` +* `files`, multipart/form upload file body, should be table contains more tables, like that: {{"name", file_body, "file_name", "file_type"},...} * `error_filter`, holds a Lua function which takes two parameters, `state` and `err`. the parameter `err` describes the error and `state` is always one of these values(represents the current stage): diff --git a/lib/resty/requests/multipart.lua b/lib/resty/requests/multipart.lua index 4d94fee..5a6153d 100644 --- a/lib/resty/requests/multipart.lua +++ b/lib/resty/requests/multipart.lua @@ -1,4 +1,6 @@ --https://github.com/pytpeng/lua-multipart +--fork from https://github.com/Kong/lua-multipart + local setmetatable = setmetatable local tostring = tostring local insert = table.insert From 8633569f234823c7680e6a85baad00d475a27f3c Mon Sep 17 00:00:00 2001 From: peng Date: Sun, 22 Mar 2020 19:43:20 +0800 Subject: [PATCH 04/16] local var --- lib/resty/requests/util.lua | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/resty/requests/util.lua b/lib/resty/requests/util.lua index 82addd7..7c28516 100644 --- a/lib/resty/requests/util.lua +++ b/lib/resty/requests/util.lua @@ -9,6 +9,8 @@ local ipairs = ipairs local require = require local setmetatable = setmetatable local lower = string.lower +local str_sub = string.sub +local tostring = tostring local ngx_gsub = ngx.re.gsub local base64 = ngx.encode_base64 local Multipart = require("resty.requests.multipart") @@ -208,7 +210,7 @@ end local function choose_boundary() - return string.sub(tostring({}), 10) + return str_sub(tostring({}), 10) end _M.new_tab = new_tab @@ -219,13 +221,13 @@ _M.is_func = is_func _M.set_config = set_config _M.dict = dict _M.basic_auth = basic_auth +_M.make_multipart_body = make_multipart_body +_M.choose_boundary = choose_boundary _M.DEFAULT_TIMEOUTS = DEFAULT_TIMEOUTS _M.BUILTIN_HEADERS = BUILTIN_HEADERS _M.STATE = STATE _M.STATE_NAME = STATE_NAME _M.HTTP10 = HTTP10 _M.HTTP11 = HTTP11 -_M.make_multipart_body = make_multipart_body -_M.choose_boundary = choose_boundary return _M From 51bd31ad729de1e215d52accee5334ce1da70a47 Mon Sep 17 00:00:00 2001 From: peng Date: Mon, 23 Mar 2020 10:31:11 +0800 Subject: [PATCH 05/16] fix code style --- lib/resty/requests/request.lua | 7 +++++-- lib/resty/requests/util.lua | 19 ++++++++++--------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/lib/resty/requests/request.lua b/lib/resty/requests/request.lua index 48fa94a..d4399b5 100644 --- a/lib/resty/requests/request.lua +++ b/lib/resty/requests/request.lua @@ -69,21 +69,24 @@ local function prepare(url_parts, session, config) local content local json = config.json local body = config.body + local files = config.files if json then content = cjson.encode(json) headers["content-length"] = #content headers["content-type"] = "application/json" + elseif files and is_tab(files) then local content_type = headers["content-type"] if not content_type then - content_type = "multipart/form-data; boundary="..util.choose_boundary() + content_type = "multipart/form-data; boundary=" .. util.choose_boundary() end - local multipart_body= util.make_multipart_body(files, content_type) + local multipart_body = util.make_multipart_body(files, content_type) headers["content-type"] = content_type headers["content-length"] = #multipart_body content = multipart_body + else content = body if is_func(body) then diff --git a/lib/resty/requests/util.lua b/lib/resty/requests/util.lua index 7c28516..60d7ec5 100644 --- a/lib/resty/requests/util.lua +++ b/lib/resty/requests/util.lua @@ -1,19 +1,19 @@ -- Copyright (C) Alex Zhang +local multipart = require("resty.requests.multipart") + local type = type local pcall = pcall local pairs = pairs local error = error local rawget = rawget local ipairs = ipairs -local require = require local setmetatable = setmetatable local lower = string.lower local str_sub = string.sub local tostring = tostring local ngx_gsub = ngx.re.gsub local base64 = ngx.encode_base64 -local Multipart = require("resty.requests.multipart") local _M = { _VERSION = '0.1' } @@ -94,7 +94,7 @@ end local function set_config(opts) opts = opts or {} - local config = new_tab(0, 14) + local config = new_tab(0, 15) -- 1) timeouts local timeouts = opts.timeouts @@ -136,9 +136,6 @@ local function set_config(opts) -- 4) body config.body = opts.body - -- 4.1) files - config.files = opts.files - -- 5) ssl verify config.ssl = opts.ssl @@ -196,14 +193,17 @@ local function set_config(opts) -- 14) use_default_type config.use_default_type = opts.use_default_type ~= false + -- 15) files + config.files = opts.files + return config end local function make_multipart_body(files, content_type) - local m = Multipart("", content_type) - for _, v in ipairs(files) do - m:set_simple(v[1], v[2], v[3], v[4]) + local m = multipart("", content_type) + for i=1,#files do + m:set_simple(files[i][1], files[i][2], files[i][3], files[i][4]) end return m:tostring() end @@ -213,6 +213,7 @@ local function choose_boundary() return str_sub(tostring({}), 10) end + _M.new_tab = new_tab _M.is_str = is_str _M.is_num = is_num From 8a23a0831326af59440752115bb7ad840ec89247 Mon Sep 17 00:00:00 2001 From: peng Date: Mon, 23 Mar 2020 15:57:18 +0800 Subject: [PATCH 06/16] remove git address,once upstream merage my pr,so could install lua-multipart by LuaRocks/OPM --- lib/resty/requests/multipart.lua | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/resty/requests/multipart.lua b/lib/resty/requests/multipart.lua index 5a6153d..da10b68 100644 --- a/lib/resty/requests/multipart.lua +++ b/lib/resty/requests/multipart.lua @@ -1,5 +1,3 @@ ---https://github.com/pytpeng/lua-multipart ---fork from https://github.com/Kong/lua-multipart local setmetatable = setmetatable local tostring = tostring From 3e46a90988429dfe2afcc99f0cbb89b05c97ecc7 Mon Sep 17 00:00:00 2001 From: peng Date: Tue, 24 Mar 2020 17:58:58 +0800 Subject: [PATCH 07/16] add test example and fix code style --- lib/resty/requests/util.lua | 4 +- t/09-mutipart.t | 104 ++++++++++++++++++++++++++++++++++++ t/multipart/t1.txt | 1 + 3 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 t/09-mutipart.t create mode 100644 t/multipart/t1.txt diff --git a/lib/resty/requests/util.lua b/lib/resty/requests/util.lua index 60d7ec5..c23f690 100644 --- a/lib/resty/requests/util.lua +++ b/lib/resty/requests/util.lua @@ -202,9 +202,11 @@ end local function make_multipart_body(files, content_type) local m = multipart("", content_type) - for i=1,#files do + + for i=1, #files do m:set_simple(files[i][1], files[i][2], files[i][3], files[i][4]) end + return m:tostring() end diff --git a/t/09-mutipart.t b/t/09-mutipart.t new file mode 100644 index 0000000..441b102 --- /dev/null +++ b/t/09-mutipart.t @@ -0,0 +1,104 @@ +use Test::Nginx::Socket::Lua; + +repeat_each(1); +plan tests => repeat_each() * (blocks() * 3); + +our $http_config = << 'EOC'; + lua_package_path "lib/?.lua;;"; + + server { + listen 10086; + location = /t1 { + content_by_lua_block { + ngx.req.read_body() + local multipart = require("resty.requests.multipart") + local content_type = ngx.req.get_headers()["content-type"] + local body = ngx.req.get_body_data() + local m = multipart(body, content_type) + local parameter = m:get("name") + ngx.print(parameter.headers) + ngx.print("\r\n") + local file_body = parameter.value + ngx.print(file_body) + } + } + location = /t2 { + content_by_lua_block { + ngx.say(ngx.req.get_headers()["content-type"]) + } + } + } +EOC + +no_long_string(); +run_tests(); + +__DATA__ + +=== TEST 1: the normal multipart upload file POST request. + +--- http_config eval: $::http_config + +--- config + +location /t { + content_by_lua_block { + local requests = require "resty.requests" + local url = "http://127.0.0.1:10086/t1" + local f = io.open("t/multipart/t1.txt") + local file_body = f:read("*all") + f:close() + local r, err = requests.post(url,{files={{"name", file_body, "t1.txt", "text/txt"}}}) + if not r then + ngx.log(ngx.ERR, err) + end + + local data, err = r:body() + ngx.print(data) + } +} + + +--- request +GET /t + +--- status_code +200 +--- response_body eval +qq{Content-Disposition: form-data; name="name"; filename="t1.txt"content-type: text/txt\r +hello world}; +--- no_error_log +[error] + + + +=== TEST 2: check the normal multipart POST request headers. + +--- http_config eval: $::http_config + +--- config + +location /t { + content_by_lua_block { + local requests = require "resty.requests" + local url = "http://127.0.0.1:10086/t2" + local r, err = requests.post(url,{files={{"name", "123", "t2.txt", "text/txt"}}}) + if not r then + ngx.log(ngx.ERR, err) + end + + local data, err = r:body() + ngx.print(data) + } +} + + +--- request +GET /t + +--- status_code +200 +--- response_body_like +^multipart/form-data; boundary=\w+$ +--- no_error_log +[error] diff --git a/t/multipart/t1.txt b/t/multipart/t1.txt new file mode 100644 index 0000000..95d09f2 --- /dev/null +++ b/t/multipart/t1.txt @@ -0,0 +1 @@ +hello world \ No newline at end of file From 117b6690f0b2911a868e496064cc4c2d1732ef79 Mon Sep 17 00:00:00 2001 From: peng Date: Tue, 24 Mar 2020 18:39:17 +0800 Subject: [PATCH 08/16] del unnecessary local var --- lib/resty/requests/util.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/resty/requests/util.lua b/lib/resty/requests/util.lua index c23f690..2cd1690 100644 --- a/lib/resty/requests/util.lua +++ b/lib/resty/requests/util.lua @@ -7,7 +7,6 @@ local pcall = pcall local pairs = pairs local error = error local rawget = rawget -local ipairs = ipairs local setmetatable = setmetatable local lower = string.lower local str_sub = string.sub From 3a47469fabd59ebb3e677cd1e72be880534a6090 Mon Sep 17 00:00:00 2001 From: peng Date: Wed, 8 Apr 2020 23:29:28 +0800 Subject: [PATCH 09/16] fix code style --- lib/resty/requests/request.lua | 2 ++ lib/resty/requests/util.lua | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/resty/requests/request.lua b/lib/resty/requests/request.lua index d4399b5..0cf87b0 100644 --- a/lib/resty/requests/request.lua +++ b/lib/resty/requests/request.lua @@ -79,9 +79,11 @@ local function prepare(url_parts, session, config) elseif files and is_tab(files) then local content_type = headers["content-type"] + if not content_type then content_type = "multipart/form-data; boundary=" .. util.choose_boundary() end + local multipart_body = util.make_multipart_body(files, content_type) headers["content-type"] = content_type headers["content-length"] = #multipart_body diff --git a/lib/resty/requests/util.lua b/lib/resty/requests/util.lua index 2cd1690..afb1b33 100644 --- a/lib/resty/requests/util.lua +++ b/lib/resty/requests/util.lua @@ -1,6 +1,6 @@ -- Copyright (C) Alex Zhang -local multipart = require("resty.requests.multipart") +local multipart = require "resty.requests.multipart" local type = type local pcall = pcall From 4c445cfcb81eb0c7c79ad1b609bcff2e390403ff Mon Sep 17 00:00:00 2001 From: peng Date: Fri, 1 May 2020 22:32:04 +0800 Subject: [PATCH 10/16] update multipart code --- .gitignore | 1 + lib/resty/requests/fields.lua | 109 ++++++++++++++++++++++++++++++++ lib/resty/requests/filepost.lua | 57 +++++++++++++++++ lib/resty/requests/models.lua | 86 +++++++++++++++++++++++++ lib/resty/requests/request.lua | 9 +-- lib/resty/requests/util.lua | 74 ++++++++++++++++++---- 6 files changed, 317 insertions(+), 19 deletions(-) create mode 100644 lib/resty/requests/fields.lua create mode 100644 lib/resty/requests/filepost.lua create mode 100644 lib/resty/requests/models.lua diff --git a/.gitignore b/.gitignore index 985830b..263df5f 100644 --- a/.gitignore +++ b/.gitignore @@ -41,5 +41,6 @@ luac.out #develop env .idea +.vscode t/servroot/* diff --git a/lib/resty/requests/fields.lua b/lib/resty/requests/fields.lua new file mode 100644 index 0000000..28bee92 --- /dev/null +++ b/lib/resty/requests/fields.lua @@ -0,0 +1,109 @@ +local util = require "resty.requests.util" + +local pairs = pairs +local concat = table.concat +local setmetatable = setmetatable +local strformat = string.format + +local _M = { _VERSION = "0.0.1"} +local mt = { __index = _M , _ID = "FIELDS"} + + +local function format_header_param_html5(name, value) + -- todo _replace_multiple + return strformat('%s="%s"', name, value) +end + + +local function new(name, data, filename, headers, header_formatter) + local self = { + _name = name, + _filename = filename, + data = data, + headers = headers or {}, + header_formatter = header_formatter or format_header_param_html5 + } + + return setmetatable(self, mt) +end + + +local function from_table(fieldname, value, header_formatter) + local filename, data, content_type + if util.is_tab(value) and util.is_array(value) then + filename, data, content_type = value[1], value[2], value[3] or "application/octet-stream" + + else + data = value + end + + local request_param = new(fieldname, data, filename, header_formatter) + request_param:make_multipart({content_type=content_type}) + return request_param +end + + +local function _render_parts(self, headers_parts) + local parts = util.new_tab(2, 0) + local parts_index = 1 + + if util.is_func(headers_parts) and not util.is_array(headers_parts) then + headers_parts = util.to_key_value_list(headers_parts) + end + + for i=1, util.len(headers_parts) do + local name = headers_parts[i][1] + local value = headers_parts[i][2] + + if value then + parts[parts_index] = self.header_formatter(name, value) + end + + end + + return concat(parts, "; ") +end + + +local function make_multipart(self, opts) + self.headers["Content-Disposition"] = opts.content_disposition or "form-data" + self.headers["Content-Disposition"] = concat({self.headers["Content-Disposition"], self:_render_parts({{"name", self._name}, {"filename", self._filename}})}, "; ") + self.headers["Content-Type"] = opts.content_type + self.headers["Content-Location"] = opts.content_location +end + + +local function render_headers(self) + local lines = util.new_tab(10, 0) + local lines_index = 1 + local sort_keys = {"Content-Disposition", "Content-Type", "Content-Location"} + + for i=1, 3 do + local tmp_value = self.headers[sort_keys[i]] + if tmp_value then + lines[lines_index] = strformat("%s: %s", sort_keys[i], tmp_value) + lines_index = lines_index + 1 + end + end + + for k, v in pairs(self.headers) do + if not util.is_inarray(k, sort_keys) and v then + lines[lines_index] = strformat("%s: %s", k, v) + lines_index = lines_index + 1 + end + + end + + lines[lines_index] = "\r\n" + return concat(lines, "\r\n") +end + + +_M.new = new +_M.from_table = from_table +_M.make_multipart = make_multipart +_M.render_headers = render_headers +_M._render_parts = _render_parts + + +return _M \ No newline at end of file diff --git a/lib/resty/requests/filepost.lua b/lib/resty/requests/filepost.lua new file mode 100644 index 0000000..423df9a --- /dev/null +++ b/lib/resty/requests/filepost.lua @@ -0,0 +1,57 @@ +local util = require "resty.requests.util" +local request_fields = require "resty.requests.fields" + +local tostring = tostring +local str_sub = string.sub +local concat = table.concat + + +local _M = { _VERSION = "0.0.1"} + + +local function choose_boundary() + return str_sub(tostring({}), 10) +end + + +local function iter_base_func(fields, i) + i = i + 1 + local field = fields[i] + + if field == nil then + return + end + + local is_array = util.is_array(field) + if is_array or (is_array == "table" and not field._ID) then + field = request_fields.from_table(field[1], field[2]) + end + + return i, field +end + + +local function iter_field_objects(fields) + return iter_base_func, fields, 0 +end + + +local function encode_multipart_formdata(fields, boundary) + boundary = boundary or choose_boundary() + local body = "" + for i, field in iter_field_objects(fields) do + body = concat({body, "--", boundary, "\r\n", field:render_headers(), field.data, "\r\n"}, "") + end + + body = body .. "--" .. boundary .. "--\r\n" + local content_type = "multipart/form-data; boundary=" .. boundary + + return body, content_type +end + + +_M.encode_multipart_formdata = encode_multipart_formdata +_M.choose_boundary = choose_boundary + + +return _M \ No newline at end of file diff --git a/lib/resty/requests/models.lua b/lib/resty/requests/models.lua new file mode 100644 index 0000000..f22bf48 --- /dev/null +++ b/lib/resty/requests/models.lua @@ -0,0 +1,86 @@ +local util = require "resty.requests.util" +local filepost = require "resty.requests.filepost" +local request_fields = require "resty.requests.fields" + + +local _M = { _VERSION = "0.0.1" } + + +local function encode_files(files, data) + if not files then + error("Files must be provided.") + end + + local new_fields = util.new_tab(30, 0) + local new_fields_index = 1 + local fields = util.to_key_value_list(data or {}) + files = util.to_key_value_list(files) + + for i=1, util.len(fields) do + local field = fields[i][1] + local val = fields[i][2] + if util.is_str(val) then + val = {val} + end + + for i=1, util.len(val) do + if not val[i] then + goto CONTINUE + end + + new_fields[new_fields_index] = {field, val[i]} + new_fields_index = new_fields_index + 1 + + ::CONTINUE:: + end + end + + for i=1, util.len(files) do + local fn, ft, fh, fp, fdata + local k = files[i][1] + local v = files[i][2] + if util.is_array(v) then + local length = util.len(v) + if length == 2 then + fn, fp = v[1], v[2] + + elseif length == 3 then + fn, fp, ft = v[1], v[2], v[3] + + else + fn, fp, ft, fh = v[1], v[2], v[3], v[4] + end + + else + fn = k + fp = v + end + + if util.is_func(fp) then + fdata = fp:read("*all") + fp:close() + + elseif fp == nil then + goto CONTINUE + + else + fdata = fp + end + + local rf = request_fields.new(k, fdata, fn, fh) + rf:make_multipart({content_type=ft}) + new_fields[new_fields_index] = rf + new_fields_index = new_fields_index + 1 + + ::CONTINUE:: + end + + local body, content_type = filepost.encode_multipart_formdata(new_fields) + return body, content_type +end + + +_M.encode_files = encode_files + + +return _M \ No newline at end of file diff --git a/lib/resty/requests/request.lua b/lib/resty/requests/request.lua index 0cf87b0..2bf77b2 100644 --- a/lib/resty/requests/request.lua +++ b/lib/resty/requests/request.lua @@ -2,6 +2,7 @@ local cjson = require "cjson.safe" local util = require "resty.requests.util" +local models = require "resty.requests.models" local setmetatable = setmetatable local pairs = pairs @@ -78,13 +79,7 @@ local function prepare(url_parts, session, config) headers["content-type"] = "application/json" elseif files and is_tab(files) then - local content_type = headers["content-type"] - - if not content_type then - content_type = "multipart/form-data; boundary=" .. util.choose_boundary() - end - - local multipart_body = util.make_multipart_body(files, content_type) + local multipart_body, content_type = models.encode_files(files, body) headers["content-type"] = content_type headers["content-length"] = #multipart_body content = multipart_body diff --git a/lib/resty/requests/util.lua b/lib/resty/requests/util.lua index afb1b33..fbc40d6 100644 --- a/lib/resty/requests/util.lua +++ b/lib/resty/requests/util.lua @@ -1,12 +1,11 @@ -- Copyright (C) Alex Zhang - -local multipart = require "resty.requests.multipart" - local type = type local pcall = pcall local pairs = pairs local error = error local rawget = rawget +local ipairs = ipairs +local require = require local setmetatable = setmetatable local lower = string.lower local str_sub = string.sub @@ -34,6 +33,7 @@ if not ok then end end + local BUILTIN_HEADERS = { ["accept"] = "*/*", ["user-agent"] = "resty-requests", @@ -76,6 +76,30 @@ local function is_tab(obj) return type(obj) == "table" end local function is_func(obj) return type(obj) == "function" end +local is_array_ok, tisarray = pcall(require, "table.isarray") +local function is_array(obj) + if not is_tab(obj) then + return false + end + + if not is_array_ok then + return "table" + end + + return tisarray(obj) +end + + +local nkeys_ok, nkeys = pcall(require, "table.nkeys") +local function len(table) + if not nkeys_ok then + return #table + end + + return nkeys(table) +end + + local function dict(d, narr, nrec) if not d then d = new_tab(narr, nrec) @@ -195,23 +219,46 @@ local function set_config(opts) -- 15) files config.files = opts.files + return config end -local function make_multipart_body(files, content_type) - local m = multipart("", content_type) +local function to_key_value_list(value) + if not value then + return nil or {} + end + + if not is_tab(value) then + error("cannot encode objects that are not 2-tables") + end + + if is_array(value) == true then + return value + end - for i=1, #files do - m:set_simple(files[i][1], files[i][2], files[i][3], files[i][4]) + local new_value = {} + local new_value_index = 1 + for k, v in pairs(value) do + new_value[new_value_index] = {k, v} + new_value_index = new_value_index + 1 end - return m:tostring() + return new_value end -local function choose_boundary() - return str_sub(tostring({}), 10) +local function is_inarray(str, array) + local ret = false + for i=1, len(array) do + if str == array[i] then + ret = true + break + end + + end + + return ret end @@ -220,11 +267,13 @@ _M.is_str = is_str _M.is_num = is_num _M.is_tab = is_tab _M.is_func = is_func +_M.is_array = is_array +_M.is_inarray = is_inarray _M.set_config = set_config +_M.len = len _M.dict = dict _M.basic_auth = basic_auth -_M.make_multipart_body = make_multipart_body -_M.choose_boundary = choose_boundary +_M.to_key_value_list = to_key_value_list _M.DEFAULT_TIMEOUTS = DEFAULT_TIMEOUTS _M.BUILTIN_HEADERS = BUILTIN_HEADERS _M.STATE = STATE @@ -232,4 +281,5 @@ _M.STATE_NAME = STATE_NAME _M.HTTP10 = HTTP10 _M.HTTP11 = HTTP11 + return _M From 158f7a326b9b2bec274ac6b37bbf9754cdd6a06f Mon Sep 17 00:00:00 2001 From: peng Date: Fri, 1 May 2020 23:14:29 +0800 Subject: [PATCH 11/16] fix code style --- lib/resty/requests/fields.lua | 10 +- lib/resty/requests/filepost.lua | 2 - lib/resty/requests/models.lua | 5 - lib/resty/requests/multipart.lua | 328 ------------------------------- lib/resty/requests/request.lua | 2 +- lib/resty/requests/util.lua | 17 +- 6 files changed, 9 insertions(+), 355 deletions(-) delete mode 100644 lib/resty/requests/multipart.lua diff --git a/lib/resty/requests/fields.lua b/lib/resty/requests/fields.lua index 28bee92..cb74822 100644 --- a/lib/resty/requests/fields.lua +++ b/lib/resty/requests/fields.lua @@ -23,7 +23,7 @@ local function new(name, data, filename, headers, header_formatter) headers = headers or {}, header_formatter = header_formatter or format_header_param_html5 } - + return setmetatable(self, mt) end @@ -44,21 +44,18 @@ end local function _render_parts(self, headers_parts) - local parts = util.new_tab(2, 0) - local parts_index = 1 - if util.is_func(headers_parts) and not util.is_array(headers_parts) then headers_parts = util.to_key_value_list(headers_parts) end + local parts = util.new_tab(15, 0) + local parts_index = 1 for i=1, util.len(headers_parts) do local name = headers_parts[i][1] local value = headers_parts[i][2] - if value then parts[parts_index] = self.header_formatter(name, value) end - end return concat(parts, "; ") @@ -91,7 +88,6 @@ local function render_headers(self) lines[lines_index] = strformat("%s: %s", k, v) lines_index = lines_index + 1 end - end lines[lines_index] = "\r\n" diff --git a/lib/resty/requests/filepost.lua b/lib/resty/requests/filepost.lua index 423df9a..a96d8aa 100644 --- a/lib/resty/requests/filepost.lua +++ b/lib/resty/requests/filepost.lua @@ -17,7 +17,6 @@ end local function iter_base_func(fields, i) i = i + 1 local field = fields[i] - if field == nil then return end @@ -45,7 +44,6 @@ local function encode_multipart_formdata(fields, boundary) body = body .. "--" .. boundary .. "--\r\n" local content_type = "multipart/form-data; boundary=" .. boundary - return body, content_type end diff --git a/lib/resty/requests/models.lua b/lib/resty/requests/models.lua index f22bf48..f762ca3 100644 --- a/lib/resty/requests/models.lua +++ b/lib/resty/requests/models.lua @@ -15,7 +15,6 @@ local function encode_files(files, data) local new_fields_index = 1 local fields = util.to_key_value_list(data or {}) files = util.to_key_value_list(files) - for i=1, util.len(fields) do local field = fields[i][1] local val = fields[i][2] @@ -27,10 +26,8 @@ local function encode_files(files, data) if not val[i] then goto CONTINUE end - new_fields[new_fields_index] = {field, val[i]} new_fields_index = new_fields_index + 1 - ::CONTINUE:: end end @@ -71,7 +68,6 @@ local function encode_files(files, data) rf:make_multipart({content_type=ft}) new_fields[new_fields_index] = rf new_fields_index = new_fields_index + 1 - ::CONTINUE:: end @@ -82,5 +78,4 @@ end _M.encode_files = encode_files - return _M \ No newline at end of file diff --git a/lib/resty/requests/multipart.lua b/lib/resty/requests/multipart.lua deleted file mode 100644 index da10b68..0000000 --- a/lib/resty/requests/multipart.lua +++ /dev/null @@ -1,328 +0,0 @@ - -local setmetatable = setmetatable -local tostring = tostring -local insert = table.insert -local remove = table.remove -local concat = table.concat -local ipairs = ipairs -local pairs = pairs -local match = string.match -local find = string.find -local sub = string.sub - - -local RANDOM_BOUNDARY = sub(tostring({}), 10) - - -local MultipartData = { RANDOM_BOUNDARY = RANDOM_BOUNDARY} - - -MultipartData.__index = MultipartData - - -setmetatable(MultipartData, { - __call = function (cls, ...) - return cls.new(...) - end, -}) - - -local function is_header(value) - return match(value, "(%S+):%s*(%S+)") -end - - -local function table_size(t) - local res = 0 - - if t then - for _,_ in pairs(t) do - res = res + 1 - end - end - - return res -end - --- Create a table representation of multipart/data body --- --- @param {string} body The multipart/data string body --- @param {string} boundary The multipart/data boundary --- @return {table} Lua representation of the body -local function decode(body, boundary) - local result = { - data = {}, - indexes = {}, - } - - if not boundary then - return result - end - - local part_name - local part_index = 1 - local part_headers = {} - local part_value = {} - local part_value_ct = 0 - - local end_boundary_length = boundary and #boundary + 2 - local processing_part_value = false - - local position = 1 - local done = false - - repeat - local s = find(body, "[\r\n]", position) - - local line - - if s then - line = sub(body, position, s - 1) - position = s + 1 - - else - if position == 1 then - line = body - - else - line = sub(body, position) - end - - done = true - end - - if line == "" then - if s and processing_part_value then - part_value_ct = part_value_ct + 1 - part_value[part_value_ct] = sub(body, s, s) - end - - else - if sub(line, 1, 2) == "--" and sub(line, 3, end_boundary_length) == boundary then - processing_part_value = false - - if part_name ~= nil then - if part_value[part_value_ct] == "\n" then - part_value[part_value_ct] = nil - end - - if part_value[part_value_ct - 1] == "\r" then - part_value[part_value_ct - 1] = nil - end - - result.data[part_index] = { - name = part_name, - headers = part_headers, - value = concat(part_value) - } - - result.indexes[part_name] = part_index - - -- Reset fields for the next part - part_headers = {} - part_value = {} - part_value_ct = 0 - part_name = nil - part_index = part_index + 1 - end - - else - --Beginning of part - if not processing_part_value and line:sub(1, 19):lower() == "content-disposition" then - -- Extract part_name - for v in line:gmatch("[^;]+") do - if not is_header(v) then -- If it's not content disposition part - local pos = v:match("^%s*[Nn][Aa][Mm][Ee]=()") - if pos then - local current_value = v:match("^%s*([^=]*)", pos):gsub("%s*$", "") - part_name = sub(current_value, 2, #current_value - 1) - end - end - end - - insert(part_headers, line) - - if s and sub(body, s, s + 3) == "\r\n\r\n" then - processing_part_value = true - position = s + 4 - end - - elseif not processing_part_value and is_header(line) then - insert(part_headers, line) - - if s and sub(body, s, s + 3) == "\r\n\r\n" then - processing_part_value = true - position = s + 4 - end - - else - processing_part_value = true - - -- The value part begins - part_value_ct = part_value_ct + 1 - part_value[part_value_ct] = line - - if s then - part_value_ct = part_value_ct + 1 - part_value[part_value_ct] = sub(body, s, s) - end - end - end - end - - until done - - if part_name ~= nil then - result.data[part_index] = { - name = part_name, - headers = part_headers, - value = concat(part_value) - } - - result.indexes[part_name] = part_index - end - - return result -end - --- Creates a multipart/data body from a table --- --- @param {table} t The table that contains the multipart/data body properties --- @param {boundary} boundary The multipart/data boundary to use --- @return {string} The multipart/data string body -local function encode(t, boundary) - if not boundary then - boundary = RANDOM_BOUNDARY - end - - local result = {} - local i = 0 - - for _, v in ipairs(t.data) do - if v.value then - result[i + 1] = "--" - result[i + 2] = boundary - result[i + 3] = "\r\n" - - i = i + 3 - - for _, header in ipairs(v.headers) do - result[i + 1] = header - result[i + 2] = "\r\n" - - i = i + 2 - end - - result[i + 1] = "\r\n" - result[i + 2] = v.value - result[i + 3] = "\r\n" - - i = i + 3 - end - end - - if i == 0 then - return "" - end - - result[i + 1] = "--" - result[i + 2] = boundary - result[i + 3] = "--\r\n" - - return concat(result) -end - - -function MultipartData.new(data, content_type) - local instance = setmetatable({}, MultipartData) - - if content_type then - local boundary = match(content_type, ";%s*boundary=(%S+)") - if boundary then - if (sub(boundary, 1, 1) == '"' and sub(boundary, -1) == '"') or - (sub(boundary, 1, 1) == "'" and sub(boundary, -1) == "'") then - boundary = sub(boundary, 2, -2) - end - - if boundary ~= "" then - instance._boundary = boundary - end - end - end - - instance._data = decode(data or "", instance._boundary) - - return instance -end - - -function MultipartData:get(name) - return self._data.data[self._data.indexes[name]] -end - - -function MultipartData:get_all() - local result = {} - - for k, v in pairs(self._data.indexes) do - result[k] = self._data.data[v].value - end - - return result -end - - -function MultipartData:set_simple(name, value, filename, content_type) - local headers = {'Content-Disposition: form-data; name="' , name , '"'} - if filename then - headers[4] = '; filename="' - headers[5] = filename - headers[6] = '"' - end - if content_type then - headers[7] = "\r\ncontent-type: " - headers[8] = content_type - end - headers = concat(headers) - if self._data.indexes[name] then - self._data.data[self._data.indexes[name]] = { - name = name, - value = value, - headers = {headers} - } - - else - local part_index = table_size(self._data.indexes) + 1 - self._data.indexes[name] = part_index - self._data.data[part_index] = { - name = name, - value = value, - headers = {headers} - } - end -end - - -function MultipartData:delete(name) - local index = self._data.indexes[name] - - if index then - remove(self._data.data, index) - self._data.indexes[name] = nil - - -- need to recount index - for key, value in pairs(self._data.indexes) do - if value > index then - self._data.indexes[key] = value - 1 - end - end - end -end - - -function MultipartData:tostring() - return encode(self._data, self._boundary) -end - - -return MultipartData diff --git a/lib/resty/requests/request.lua b/lib/resty/requests/request.lua index 2bf77b2..e0c3919 100644 --- a/lib/resty/requests/request.lua +++ b/lib/resty/requests/request.lua @@ -78,7 +78,7 @@ local function prepare(url_parts, session, config) headers["content-length"] = #content headers["content-type"] = "application/json" - elseif files and is_tab(files) then + elseif files and util.is_array(files) then local multipart_body, content_type = models.encode_files(files, body) headers["content-type"] = content_type headers["content-length"] = #multipart_body diff --git a/lib/resty/requests/util.lua b/lib/resty/requests/util.lua index fbc40d6..acfafad 100644 --- a/lib/resty/requests/util.lua +++ b/lib/resty/requests/util.lua @@ -4,12 +4,9 @@ local pcall = pcall local pairs = pairs local error = error local rawget = rawget -local ipairs = ipairs local require = require local setmetatable = setmetatable local lower = string.lower -local str_sub = string.sub -local tostring = tostring local ngx_gsub = ngx.re.gsub local base64 = ngx.encode_base64 @@ -219,16 +216,15 @@ local function set_config(opts) -- 15) files config.files = opts.files - return config end local function to_key_value_list(value) if not value then - return nil or {} + return {} end - + if not is_tab(value) then error("cannot encode objects that are not 2-tables") end @@ -249,16 +245,13 @@ end local function is_inarray(str, array) - local ret = false for i=1, len(array) do if str == array[i] then - ret = true - break + return true end - end - - return ret + + return false end From 0a8ebf4deab6f4971a4839311699b4487652cad9 Mon Sep 17 00:00:00 2001 From: peng Date: Sat, 2 May 2020 09:14:27 +0800 Subject: [PATCH 12/16] fix code style && update test case --- lib/resty/requests/fields.lua | 4 +--- lib/resty/requests/filepost.lua | 3 --- lib/resty/requests/models.lua | 3 +-- lib/resty/requests/util.lua | 3 +-- t/09-mutipart.t | 27 +++++++++++---------------- 5 files changed, 14 insertions(+), 26 deletions(-) diff --git a/lib/resty/requests/fields.lua b/lib/resty/requests/fields.lua index cb74822..a3d1a22 100644 --- a/lib/resty/requests/fields.lua +++ b/lib/resty/requests/fields.lua @@ -8,7 +8,6 @@ local strformat = string.format local _M = { _VERSION = "0.0.1"} local mt = { __index = _M , _ID = "FIELDS"} - local function format_header_param_html5(name, value) -- todo _replace_multiple return strformat('%s="%s"', name, value) @@ -23,7 +22,7 @@ local function new(name, data, filename, headers, header_formatter) headers = headers or {}, header_formatter = header_formatter or format_header_param_html5 } - + return setmetatable(self, mt) end @@ -101,5 +100,4 @@ _M.make_multipart = make_multipart _M.render_headers = render_headers _M._render_parts = _render_parts - return _M \ No newline at end of file diff --git a/lib/resty/requests/filepost.lua b/lib/resty/requests/filepost.lua index a96d8aa..3eaa077 100644 --- a/lib/resty/requests/filepost.lua +++ b/lib/resty/requests/filepost.lua @@ -5,10 +5,8 @@ local tostring = tostring local str_sub = string.sub local concat = table.concat - local _M = { _VERSION = "0.0.1"} - local function choose_boundary() return str_sub(tostring({}), 10) end @@ -51,5 +49,4 @@ end _M.encode_multipart_formdata = encode_multipart_formdata _M.choose_boundary = choose_boundary - return _M \ No newline at end of file diff --git a/lib/resty/requests/models.lua b/lib/resty/requests/models.lua index f762ca3..2e31779 100644 --- a/lib/resty/requests/models.lua +++ b/lib/resty/requests/models.lua @@ -2,7 +2,6 @@ local util = require "resty.requests.util" local filepost = require "resty.requests.filepost" local request_fields = require "resty.requests.fields" - local _M = { _VERSION = "0.0.1" } @@ -10,7 +9,7 @@ local function encode_files(files, data) if not files then error("Files must be provided.") end - + local new_fields = util.new_tab(30, 0) local new_fields_index = 1 local fields = util.to_key_value_list(data or {}) diff --git a/lib/resty/requests/util.lua b/lib/resty/requests/util.lua index acfafad..fdeeabe 100644 --- a/lib/resty/requests/util.lua +++ b/lib/resty/requests/util.lua @@ -250,7 +250,7 @@ local function is_inarray(str, array) return true end end - + return false end @@ -274,5 +274,4 @@ _M.STATE_NAME = STATE_NAME _M.HTTP10 = HTTP10 _M.HTTP11 = HTTP11 - return _M diff --git a/t/09-mutipart.t b/t/09-mutipart.t index 441b102..849c554 100644 --- a/t/09-mutipart.t +++ b/t/09-mutipart.t @@ -11,15 +11,9 @@ our $http_config = << 'EOC'; location = /t1 { content_by_lua_block { ngx.req.read_body() - local multipart = require("resty.requests.multipart") - local content_type = ngx.req.get_headers()["content-type"] - local body = ngx.req.get_body_data() - local m = multipart(body, content_type) - local parameter = m:get("name") - ngx.print(parameter.headers) - ngx.print("\r\n") - local file_body = parameter.value - ngx.print(file_body) + local cjson = require "cjson" + local body = ngx.req.get_post_args() + ngx.say(cjson.encode(body)) } } location = /t2 { @@ -44,17 +38,17 @@ __DATA__ location /t { content_by_lua_block { local requests = require "resty.requests" + local cjson = require "cjson" local url = "http://127.0.0.1:10086/t1" local f = io.open("t/multipart/t1.txt") local file_body = f:read("*all") f:close() - local r, err = requests.post(url,{files={{"name", file_body, "t1.txt", "text/txt"}}}) + local r, err = requests.post(url,{files={{"name", {"t1.txt", file_body,"text/txt", {testheader="i_am_test_header"}}}}, body={testbody1={pp=1}, testbody2={1,2,3}}}) if not r then ngx.log(ngx.ERR, err) end - local data, err = r:body() - ngx.print(data) + ngx.say(data) } } @@ -64,9 +58,10 @@ GET /t --- status_code 200 ---- response_body eval -qq{Content-Disposition: form-data; name="name"; filename="t1.txt"content-type: text/txt\r -hello world}; + +--- response_body_like +{\"--[a-z0-9]{8}[\s\S]*--[a-z0-9]{8}--[\s\S]+\"} + --- no_error_log [error] @@ -82,7 +77,7 @@ location /t { content_by_lua_block { local requests = require "resty.requests" local url = "http://127.0.0.1:10086/t2" - local r, err = requests.post(url,{files={{"name", "123", "t2.txt", "text/txt"}}}) + local r, err = requests.post(url,{files={{"name", {"t2.txt", "hello world", "text/txt"}}}}) if not r then ngx.log(ngx.ERR, err) end From 1cc478ddc292224a8113ac1bdcf24a69464491c8 Mon Sep 17 00:00:00 2001 From: peng Date: Sat, 2 May 2020 09:24:24 +0800 Subject: [PATCH 13/16] remove dependencies by Kong\lua-multipart && update readme --- README.markdown | 46 ++++++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/README.markdown b/README.markdown index bff77fe..5da78f0 100644 --- a/README.markdown +++ b/README.markdown @@ -12,27 +12,28 @@ resty -e 'print(require "resty.requests".get{ url = "https://github.com", stream Table of Contents ================= -* [Name](#name) -* [Status](#status) -* [Features](#features) -* [Synopsis](#synopsis) -* [Installation](#installation) -* [Methods](#methods) - * [request](#request) - * [state](#state) - * [get](#get) - * [head](#head) - * [post](#post) - * [put](#put) - * [delete](#delete) - * [options](#options) - * [patch](#patch) -* [Response Object](#response-object) -* [Session](#session) -* [TODO](#todo) -* [Author](#author) -* [Copyright and License](#copyright-and-license) -* [See Also](#see-also) +- [Name](#name) +- [Table of Contents](#table-of-contents) +- [Status](#status) +- [Features](#features) +- [Synopsis](#synopsis) +- [Installation](#installation) +- [Methods](#methods) + - [request](#request) + - [state](#state) + - [get](#get) + - [head](#head) + - [post](#post) + - [put](#put) + - [delete](#delete) + - [options](#options) + - [patch](#patch) +- [Response Object](#response-object) +- [Session](#session) +- [TODO](#todo) +- [Author](#author) +- [Copyright and License](#copyright-and-license) +- [See Also](#see-also) Status ====== @@ -166,7 +167,8 @@ The third param, an optional Lua table, which contains a number of options: * a Lua string, or * a Lua function, without parameter and returns a piece of data (string) or an empty Lua string to represent EOF, or * a Lua table, each key-value pair will be concatenated with the "&", and Content-Type header will `"application/x-www-form-urlencoded"` -* `files`, multipart/form upload file body, should be table contains more tables, like that: {{"name", file_body, "file_name", "file_type"},...} +* `files`, multipart/form upload file body, should be table contains more multi-tables, like that: {{"name", {"file_name",fp, "file_type"}},...} + * `fp` is binary file body or file func with method `read` * `error_filter`, holds a Lua function which takes two parameters, `state` and `err`. the parameter `err` describes the error and `state` is always one of these values(represents the current stage): From aae1b4ff8da5b5b88a5c17726b97b43db4f6cb8a Mon Sep 17 00:00:00 2001 From: peng Date: Sat, 2 May 2020 09:49:31 +0800 Subject: [PATCH 14/16] fix files body with userdata fp --- README.markdown | 1 + lib/resty/requests/models.lua | 8 ++++---- lib/resty/requests/util.lua | 2 ++ t/09-mutipart.t | 38 ++++++++++++++++++++++++++++++++++- 4 files changed, 44 insertions(+), 5 deletions(-) diff --git a/README.markdown b/README.markdown index 5da78f0..65ffb6b 100644 --- a/README.markdown +++ b/README.markdown @@ -420,3 +420,4 @@ See Also * upyun-resty: https://github.com/upyun/upyun-resty * httpipe: https://github.com/timebug/lua-resty-httpipe +* python-requests: https://github.com/psf/requests diff --git a/lib/resty/requests/models.lua b/lib/resty/requests/models.lua index 2e31779..a3cff70 100644 --- a/lib/resty/requests/models.lua +++ b/lib/resty/requests/models.lua @@ -52,13 +52,13 @@ local function encode_files(files, data) fp = v end - if util.is_func(fp) then + if fp == nil then + goto CONTINUE + + elseif util.is_userdata(fp) and util.is_func(fp.read) then fdata = fp:read("*all") fp:close() - elseif fp == nil then - goto CONTINUE - else fdata = fp end diff --git a/lib/resty/requests/util.lua b/lib/resty/requests/util.lua index fdeeabe..d6ceac0 100644 --- a/lib/resty/requests/util.lua +++ b/lib/resty/requests/util.lua @@ -71,6 +71,7 @@ local function is_str(obj) return type(obj) == "string" end local function is_num(obj) return type(obj) == "number" end local function is_tab(obj) return type(obj) == "table" end local function is_func(obj) return type(obj) == "function" end +local function is_userdata(obj) return type(obj) == "userdata" end local is_array_ok, tisarray = pcall(require, "table.isarray") @@ -262,6 +263,7 @@ _M.is_tab = is_tab _M.is_func = is_func _M.is_array = is_array _M.is_inarray = is_inarray +_M.is_userdata = is_userdata _M.set_config = set_config _M.len = len _M.dict = dict diff --git a/t/09-mutipart.t b/t/09-mutipart.t index 849c554..ac50fc3 100644 --- a/t/09-mutipart.t +++ b/t/09-mutipart.t @@ -67,7 +67,43 @@ GET /t -=== TEST 2: check the normal multipart POST request headers. +=== TEST 2: test multipart upload file POST request with userdata fp. + +--- http_config eval: $::http_config + +--- config + +location /t { + content_by_lua_block { + local requests = require "resty.requests" + local cjson = require "cjson" + local url = "http://127.0.0.1:10086/t1" + local fp = io.open("t/multipart/t1.txt") + local r, err = requests.post(url,{files={{"name", {"t1.txt", fp,"text/txt", {testheader="i_am_test_header"}}}}, body={testbody1={pp=1}, testbody2={1,2,3}}}) + if not r then + ngx.log(ngx.ERR, err) + end + local data, err = r:body() + ngx.say(data) + } +} + + +--- request +GET /t + +--- status_code +200 + +--- response_body_like +{\"--[a-z0-9]{8}[\s\S]*--[a-z0-9]{8}--[\s\S]+\"} + +--- no_error_log +[error] + + + +=== TEST 3: check the normal multipart POST request headers. --- http_config eval: $::http_config From 868162ee1b0d20a18c0b5132d4b801fc687db5de Mon Sep 17 00:00:00 2001 From: peng Date: Sat, 2 May 2020 10:20:59 +0800 Subject: [PATCH 15/16] fix iter_base_func is_array --- lib/resty/requests/fields.lua | 4 ++-- lib/resty/requests/filepost.lua | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/resty/requests/fields.lua b/lib/resty/requests/fields.lua index a3d1a22..852cb5a 100644 --- a/lib/resty/requests/fields.lua +++ b/lib/resty/requests/fields.lua @@ -5,8 +5,8 @@ local concat = table.concat local setmetatable = setmetatable local strformat = string.format -local _M = { _VERSION = "0.0.1"} -local mt = { __index = _M , _ID = "FIELDS"} +local _M = { _VERSION = "0.0.1" } +local mt = { __index = _M , _ID = "FIELDS" } local function format_header_param_html5(name, value) -- todo _replace_multiple diff --git a/lib/resty/requests/filepost.lua b/lib/resty/requests/filepost.lua index 3eaa077..26d323b 100644 --- a/lib/resty/requests/filepost.lua +++ b/lib/resty/requests/filepost.lua @@ -5,7 +5,7 @@ local tostring = tostring local str_sub = string.sub local concat = table.concat -local _M = { _VERSION = "0.0.1"} +local _M = { _VERSION = "0.0.1" } local function choose_boundary() return str_sub(tostring({}), 10) @@ -20,7 +20,7 @@ local function iter_base_func(fields, i) end local is_array = util.is_array(field) - if is_array or (is_array == "table" and not field._ID) then + if is_array == true or (is_array == "table" and not field._ID) then field = request_fields.from_table(field[1], field[2]) end From 2a9c60428a8884c65a73635d179cd4c1e1614e8f Mon Sep 17 00:00:00 2001 From: peng Date: Wed, 6 May 2020 18:28:50 +0800 Subject: [PATCH 16/16] local var error --- lib/resty/requests/fields.lua | 4 ++-- lib/resty/requests/models.lua | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/resty/requests/fields.lua b/lib/resty/requests/fields.lua index 852cb5a..74e9f1d 100644 --- a/lib/resty/requests/fields.lua +++ b/lib/resty/requests/fields.lua @@ -47,7 +47,7 @@ local function _render_parts(self, headers_parts) headers_parts = util.to_key_value_list(headers_parts) end - local parts = util.new_tab(15, 0) + local parts = {} local parts_index = 1 for i=1, util.len(headers_parts) do local name = headers_parts[i][1] @@ -70,7 +70,7 @@ end local function render_headers(self) - local lines = util.new_tab(10, 0) + local lines = {} local lines_index = 1 local sort_keys = {"Content-Disposition", "Content-Type", "Content-Location"} diff --git a/lib/resty/requests/models.lua b/lib/resty/requests/models.lua index a3cff70..61faaa1 100644 --- a/lib/resty/requests/models.lua +++ b/lib/resty/requests/models.lua @@ -2,6 +2,8 @@ local util = require "resty.requests.util" local filepost = require "resty.requests.filepost" local request_fields = require "resty.requests.fields" +local error = error + local _M = { _VERSION = "0.0.1" } @@ -10,7 +12,7 @@ local function encode_files(files, data) error("Files must be provided.") end - local new_fields = util.new_tab(30, 0) + local new_fields = {} local new_fields_index = 1 local fields = util.to_key_value_list(data or {}) files = util.to_key_value_list(files)