diff --git a/.gitignore b/.gitignore index 1d659ca..263df5f 100644 --- a/.gitignore +++ b/.gitignore @@ -39,4 +39,8 @@ luac.out *.x86_64 *.hex +#develop env +.idea +.vscode + t/servroot/* diff --git a/README.markdown b/README.markdown index 26b1540..65ffb6b 100644 --- a/README.markdown +++ b/README.markdown @@ -12,27 +12,28 @@ resty -e 'print(require "resty.requests".get{ url = "https://github.com", stream Table of Contents ================= -* [Name](#name) -* [Status](#status) -* [Features](#features) -* [Synopsis](#synopsis) -* [Installation](#installation) -* [Methods](#methods) - * [request](#request) - * [state](#state) - * [get](#get) - * [head](#head) - * [post](#post) - * [put](#put) - * [delete](#delete) - * [options](#options) - * [patch](#patch) -* [Response Object](#response-object) -* [Session](#session) -* [TODO](#todo) -* [Author](#author) -* [Copyright and License](#copyright-and-license) -* [See Also](#see-also) +- [Name](#name) +- [Table of Contents](#table-of-contents) +- [Status](#status) +- [Features](#features) +- [Synopsis](#synopsis) +- [Installation](#installation) +- [Methods](#methods) + - [request](#request) + - [state](#state) + - [get](#get) + - [head](#head) + - [post](#post) + - [put](#put) + - [delete](#delete) + - [options](#options) + - [patch](#patch) +- [Response Object](#response-object) +- [Session](#session) +- [TODO](#todo) +- [Author](#author) +- [Copyright and License](#copyright-and-license) +- [See Also](#see-also) Status ====== @@ -166,6 +167,8 @@ The third param, an optional Lua table, which contains a number of options: * a Lua string, or * a Lua function, without parameter and returns a piece of data (string) or an empty Lua string to represent EOF, or * a Lua table, each key-value pair will be concatenated with the "&", and Content-Type header will `"application/x-www-form-urlencoded"` +* `files`, multipart/form upload file body, should be table contains more multi-tables, like that: {{"name", {"file_name",fp, "file_type"}},...} + * `fp` is binary file body or file func with method `read` * `error_filter`, holds a Lua function which takes two parameters, `state` and `err`. the parameter `err` describes the error and `state` is always one of these values(represents the current stage): @@ -417,3 +420,4 @@ See Also * upyun-resty: https://github.com/upyun/upyun-resty * httpipe: https://github.com/timebug/lua-resty-httpipe +* python-requests: https://github.com/psf/requests diff --git a/lib/resty/requests/fields.lua b/lib/resty/requests/fields.lua new file mode 100644 index 0000000..74e9f1d --- /dev/null +++ b/lib/resty/requests/fields.lua @@ -0,0 +1,103 @@ +local util = require "resty.requests.util" + +local pairs = pairs +local concat = table.concat +local setmetatable = setmetatable +local strformat = string.format + +local _M = { _VERSION = "0.0.1" } +local mt = { __index = _M , _ID = "FIELDS" } + +local function format_header_param_html5(name, value) + -- todo _replace_multiple + return strformat('%s="%s"', name, value) +end + + +local function new(name, data, filename, headers, header_formatter) + local self = { + _name = name, + _filename = filename, + data = data, + headers = headers or {}, + header_formatter = header_formatter or format_header_param_html5 + } + + return setmetatable(self, mt) +end + + +local function from_table(fieldname, value, header_formatter) + local filename, data, content_type + if util.is_tab(value) and util.is_array(value) then + filename, data, content_type = value[1], value[2], value[3] or "application/octet-stream" + + else + data = value + end + + local request_param = new(fieldname, data, filename, header_formatter) + request_param:make_multipart({content_type=content_type}) + return request_param +end + + +local function _render_parts(self, headers_parts) + if util.is_func(headers_parts) and not util.is_array(headers_parts) then + headers_parts = util.to_key_value_list(headers_parts) + end + + local parts = {} + local parts_index = 1 + for i=1, util.len(headers_parts) do + local name = headers_parts[i][1] + local value = headers_parts[i][2] + if value then + parts[parts_index] = self.header_formatter(name, value) + end + end + + return concat(parts, "; ") +end + + +local function make_multipart(self, opts) + self.headers["Content-Disposition"] = opts.content_disposition or "form-data" + self.headers["Content-Disposition"] = concat({self.headers["Content-Disposition"], self:_render_parts({{"name", self._name}, {"filename", self._filename}})}, "; ") + self.headers["Content-Type"] = opts.content_type + self.headers["Content-Location"] = opts.content_location +end + + +local function render_headers(self) + local lines = {} + local lines_index = 1 + local sort_keys = {"Content-Disposition", "Content-Type", "Content-Location"} + + for i=1, 3 do + local tmp_value = self.headers[sort_keys[i]] + if tmp_value then + lines[lines_index] = strformat("%s: %s", sort_keys[i], tmp_value) + lines_index = lines_index + 1 + end + end + + for k, v in pairs(self.headers) do + if not util.is_inarray(k, sort_keys) and v then + lines[lines_index] = strformat("%s: %s", k, v) + lines_index = lines_index + 1 + end + end + + lines[lines_index] = "\r\n" + return concat(lines, "\r\n") +end + + +_M.new = new +_M.from_table = from_table +_M.make_multipart = make_multipart +_M.render_headers = render_headers +_M._render_parts = _render_parts + +return _M \ No newline at end of file diff --git a/lib/resty/requests/filepost.lua b/lib/resty/requests/filepost.lua new file mode 100644 index 0000000..26d323b --- /dev/null +++ b/lib/resty/requests/filepost.lua @@ -0,0 +1,52 @@ +local util = require "resty.requests.util" +local request_fields = require "resty.requests.fields" + +local tostring = tostring +local str_sub = string.sub +local concat = table.concat + +local _M = { _VERSION = "0.0.1" } + +local function choose_boundary() + return str_sub(tostring({}), 10) +end + + +local function iter_base_func(fields, i) + i = i + 1 + local field = fields[i] + if field == nil then + return + end + + local is_array = util.is_array(field) + if is_array == true or (is_array == "table" and not field._ID) then + field = request_fields.from_table(field[1], field[2]) + end + + return i, field +end + + +local function iter_field_objects(fields) + return iter_base_func, fields, 0 +end + + +local function encode_multipart_formdata(fields, boundary) + boundary = boundary or choose_boundary() + local body = "" + for i, field in iter_field_objects(fields) do + body = concat({body, "--", boundary, "\r\n", field:render_headers(), field.data, "\r\n"}, "") + end + + body = body .. "--" .. boundary .. "--\r\n" + local content_type = "multipart/form-data; boundary=" .. boundary + return body, content_type +end + + +_M.encode_multipart_formdata = encode_multipart_formdata +_M.choose_boundary = choose_boundary + +return _M \ No newline at end of file diff --git a/lib/resty/requests/models.lua b/lib/resty/requests/models.lua new file mode 100644 index 0000000..61faaa1 --- /dev/null +++ b/lib/resty/requests/models.lua @@ -0,0 +1,82 @@ +local util = require "resty.requests.util" +local filepost = require "resty.requests.filepost" +local request_fields = require "resty.requests.fields" + +local error = error + +local _M = { _VERSION = "0.0.1" } + + +local function encode_files(files, data) + if not files then + error("Files must be provided.") + end + + local new_fields = {} + local new_fields_index = 1 + local fields = util.to_key_value_list(data or {}) + files = util.to_key_value_list(files) + for i=1, util.len(fields) do + local field = fields[i][1] + local val = fields[i][2] + if util.is_str(val) then + val = {val} + end + + for i=1, util.len(val) do + if not val[i] then + goto CONTINUE + end + new_fields[new_fields_index] = {field, val[i]} + new_fields_index = new_fields_index + 1 + ::CONTINUE:: + end + end + + for i=1, util.len(files) do + local fn, ft, fh, fp, fdata + local k = files[i][1] + local v = files[i][2] + if util.is_array(v) then + local length = util.len(v) + if length == 2 then + fn, fp = v[1], v[2] + + elseif length == 3 then + fn, fp, ft = v[1], v[2], v[3] + + else + fn, fp, ft, fh = v[1], v[2], v[3], v[4] + end + + else + fn = k + fp = v + end + + if fp == nil then + goto CONTINUE + + elseif util.is_userdata(fp) and util.is_func(fp.read) then + fdata = fp:read("*all") + fp:close() + + else + fdata = fp + end + + local rf = request_fields.new(k, fdata, fn, fh) + rf:make_multipart({content_type=ft}) + new_fields[new_fields_index] = rf + new_fields_index = new_fields_index + 1 + ::CONTINUE:: + end + + local body, content_type = filepost.encode_multipart_formdata(new_fields) + return body, content_type +end + + +_M.encode_files = encode_files + +return _M \ No newline at end of file diff --git a/lib/resty/requests/request.lua b/lib/resty/requests/request.lua index ba5447d..e0c3919 100644 --- a/lib/resty/requests/request.lua +++ b/lib/resty/requests/request.lua @@ -2,6 +2,7 @@ local cjson = require "cjson.safe" local util = require "resty.requests.util" +local models = require "resty.requests.models" local setmetatable = setmetatable local pairs = pairs @@ -70,10 +71,19 @@ local function prepare(url_parts, session, config) local json = config.json local body = config.body + local files = config.files + if json then content = cjson.encode(json) headers["content-length"] = #content headers["content-type"] = "application/json" + + elseif files and util.is_array(files) then + local multipart_body, content_type = models.encode_files(files, body) + headers["content-type"] = content_type + headers["content-length"] = #multipart_body + content = multipart_body + else content = body if is_func(body) then diff --git a/lib/resty/requests/util.lua b/lib/resty/requests/util.lua index 33a02fa..d6ceac0 100644 --- a/lib/resty/requests/util.lua +++ b/lib/resty/requests/util.lua @@ -1,10 +1,10 @@ -- Copyright (C) Alex Zhang - local type = type local pcall = pcall local pairs = pairs local error = error local rawget = rawget +local require = require local setmetatable = setmetatable local lower = string.lower local ngx_gsub = ngx.re.gsub @@ -30,6 +30,7 @@ if not ok then end end + local BUILTIN_HEADERS = { ["accept"] = "*/*", ["user-agent"] = "resty-requests", @@ -70,6 +71,31 @@ local function is_str(obj) return type(obj) == "string" end local function is_num(obj) return type(obj) == "number" end local function is_tab(obj) return type(obj) == "table" end local function is_func(obj) return type(obj) == "function" end +local function is_userdata(obj) return type(obj) == "userdata" end + + +local is_array_ok, tisarray = pcall(require, "table.isarray") +local function is_array(obj) + if not is_tab(obj) then + return false + end + + if not is_array_ok then + return "table" + end + + return tisarray(obj) +end + + +local nkeys_ok, nkeys = pcall(require, "table.nkeys") +local function len(table) + if not nkeys_ok then + return #table + end + + return nkeys(table) +end local function dict(d, narr, nrec) @@ -89,7 +115,7 @@ end local function set_config(opts) opts = opts or {} - local config = new_tab(0, 14) + local config = new_tab(0, 15) -- 1) timeouts local timeouts = opts.timeouts @@ -188,18 +214,61 @@ local function set_config(opts) -- 14) use_default_type config.use_default_type = opts.use_default_type ~= false + -- 15) files + config.files = opts.files + return config end +local function to_key_value_list(value) + if not value then + return {} + end + + if not is_tab(value) then + error("cannot encode objects that are not 2-tables") + end + + if is_array(value) == true then + return value + end + + local new_value = {} + local new_value_index = 1 + for k, v in pairs(value) do + new_value[new_value_index] = {k, v} + new_value_index = new_value_index + 1 + end + + return new_value +end + + +local function is_inarray(str, array) + for i=1, len(array) do + if str == array[i] then + return true + end + end + + return false +end + + _M.new_tab = new_tab _M.is_str = is_str _M.is_num = is_num _M.is_tab = is_tab _M.is_func = is_func +_M.is_array = is_array +_M.is_inarray = is_inarray +_M.is_userdata = is_userdata _M.set_config = set_config +_M.len = len _M.dict = dict _M.basic_auth = basic_auth +_M.to_key_value_list = to_key_value_list _M.DEFAULT_TIMEOUTS = DEFAULT_TIMEOUTS _M.BUILTIN_HEADERS = BUILTIN_HEADERS _M.STATE = STATE diff --git a/t/09-mutipart.t b/t/09-mutipart.t new file mode 100644 index 0000000..ac50fc3 --- /dev/null +++ b/t/09-mutipart.t @@ -0,0 +1,135 @@ +use Test::Nginx::Socket::Lua; + +repeat_each(1); +plan tests => repeat_each() * (blocks() * 3); + +our $http_config = << 'EOC'; + lua_package_path "lib/?.lua;;"; + + server { + listen 10086; + location = /t1 { + content_by_lua_block { + ngx.req.read_body() + local cjson = require "cjson" + local body = ngx.req.get_post_args() + ngx.say(cjson.encode(body)) + } + } + location = /t2 { + content_by_lua_block { + ngx.say(ngx.req.get_headers()["content-type"]) + } + } + } +EOC + +no_long_string(); +run_tests(); + +__DATA__ + +=== TEST 1: the normal multipart upload file POST request. + +--- http_config eval: $::http_config + +--- config + +location /t { + content_by_lua_block { + local requests = require "resty.requests" + local cjson = require "cjson" + local url = "http://127.0.0.1:10086/t1" + local f = io.open("t/multipart/t1.txt") + local file_body = f:read("*all") + f:close() + local r, err = requests.post(url,{files={{"name", {"t1.txt", file_body,"text/txt", {testheader="i_am_test_header"}}}}, body={testbody1={pp=1}, testbody2={1,2,3}}}) + if not r then + ngx.log(ngx.ERR, err) + end + local data, err = r:body() + ngx.say(data) + } +} + + +--- request +GET /t + +--- status_code +200 + +--- response_body_like +{\"--[a-z0-9]{8}[\s\S]*--[a-z0-9]{8}--[\s\S]+\"} + +--- no_error_log +[error] + + + +=== TEST 2: test multipart upload file POST request with userdata fp. + +--- http_config eval: $::http_config + +--- config + +location /t { + content_by_lua_block { + local requests = require "resty.requests" + local cjson = require "cjson" + local url = "http://127.0.0.1:10086/t1" + local fp = io.open("t/multipart/t1.txt") + local r, err = requests.post(url,{files={{"name", {"t1.txt", fp,"text/txt", {testheader="i_am_test_header"}}}}, body={testbody1={pp=1}, testbody2={1,2,3}}}) + if not r then + ngx.log(ngx.ERR, err) + end + local data, err = r:body() + ngx.say(data) + } +} + + +--- request +GET /t + +--- status_code +200 + +--- response_body_like +{\"--[a-z0-9]{8}[\s\S]*--[a-z0-9]{8}--[\s\S]+\"} + +--- no_error_log +[error] + + + +=== TEST 3: check the normal multipart POST request headers. + +--- http_config eval: $::http_config + +--- config + +location /t { + content_by_lua_block { + local requests = require "resty.requests" + local url = "http://127.0.0.1:10086/t2" + local r, err = requests.post(url,{files={{"name", {"t2.txt", "hello world", "text/txt"}}}}) + if not r then + ngx.log(ngx.ERR, err) + end + + local data, err = r:body() + ngx.print(data) + } +} + + +--- request +GET /t + +--- status_code +200 +--- response_body_like +^multipart/form-data; boundary=\w+$ +--- no_error_log +[error] diff --git a/t/multipart/t1.txt b/t/multipart/t1.txt new file mode 100644 index 0000000..95d09f2 --- /dev/null +++ b/t/multipart/t1.txt @@ -0,0 +1 @@ +hello world \ No newline at end of file