From d77dc45e7eee74a7c39e850070103fcbbc8f38b0 Mon Sep 17 00:00:00 2001 From: HuabingZhao Date: Wed, 28 Feb 2018 11:10:50 +0800 Subject: Support IP Hash LB policy Issue-ID: MSB-154 Change-Id: I11b8e3a314c6045183971bf2207b9ccee7df10c2 Signed-off-by: HuabingZhao --- .../resources/openresty/nginx/conf/nginx.conf | 8 +- .../openresty/nginx/luaext/conf/msbinit.lua | 10 +- .../openresty/nginx/luaext/core/error_handler.lua | 61 ++ .../openresty/nginx/luaext/core/peerwatcher.lua | 107 +++ .../openresty/nginx/luaext/core/router.lua | 293 +++++++ .../resources/openresty/nginx/luaext/dao/dao.lua | 49 +- .../openresty/nginx/luaext/lib/utils/dns_util.lua | 99 +++ .../openresty/nginx/luaext/lib/utils/str_util.lua | 18 +- .../openresty/nginx/luaext/lib/utils/svc_util.lua | 93 ++- .../nginx/luaext/loadbalance/balancer.lua | 24 +- .../nginx/luaext/loadbalance/baseupstream.lua | 55 +- .../luaext/loadbalance/policy/consistent_hash.lua | 135 +++ .../openresty/nginx/luaext/monitor/stats.lua | 177 ++++ .../resources/openresty/nginx/luaext/msb.lua | 54 +- .../nginx/luaext/plugins/config_default.lua | 9 +- .../plugins/redirect-transformer/handler.lua | 42 +- .../plugins/redirect-transformer/url_matcher.lua | 164 ++++ .../openresty/nginx/luaext/vendor/pl/List.lua | 566 +++++++++++++ .../openresty/nginx/luaext/vendor/pl/class.lua | 261 ++++++ .../openresty/nginx/luaext/vendor/pl/compat.lua | 143 ++++ .../openresty/nginx/luaext/vendor/pl/stringx.lua | 548 ++++++++++++ .../openresty/nginx/luaext/vendor/pl/tablex.lua | 927 +++++++++++++++++++++ .../openresty/nginx/luaext/vendor/pl/types.lua | 145 ++++ .../openresty/nginx/luaext/vendor/pl/utils.lua | 516 ++++++++++++ .../msb-enabled/location-default/msblocations.conf | 116 +-- .../nginx/msb-enabled/location-ext/monitor.conf | 56 ++ .../resources/openresty/nginx/msb-enabled/msb.conf | 12 +- .../openresty/nginx/msb-enabled/msbhttps.conf | 5 +- .../openresty/nginx/sites-enabled/cosSample | 88 ++ .../src/assembly/resources/openresty/reload.sh | 7 +- 30 files changed, 4662 insertions(+), 126 deletions(-) create mode 100644 openresty-ext/src/assembly/resources/openresty/nginx/luaext/core/error_handler.lua create mode 100644 openresty-ext/src/assembly/resources/openresty/nginx/luaext/core/peerwatcher.lua create mode 100644 openresty-ext/src/assembly/resources/openresty/nginx/luaext/core/router.lua create mode 100644 openresty-ext/src/assembly/resources/openresty/nginx/luaext/lib/utils/dns_util.lua create mode 100644 openresty-ext/src/assembly/resources/openresty/nginx/luaext/loadbalance/policy/consistent_hash.lua create mode 100644 openresty-ext/src/assembly/resources/openresty/nginx/luaext/monitor/stats.lua create mode 100644 openresty-ext/src/assembly/resources/openresty/nginx/luaext/plugins/redirect-transformer/url_matcher.lua create mode 100644 openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/pl/List.lua create mode 100644 openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/pl/class.lua create mode 100644 openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/pl/compat.lua create mode 100644 openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/pl/stringx.lua create mode 100644 openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/pl/tablex.lua create mode 100644 openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/pl/types.lua create mode 100644 openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/pl/utils.lua create mode 100644 openresty-ext/src/assembly/resources/openresty/nginx/msb-enabled/location-ext/monitor.conf create mode 100644 openresty-ext/src/assembly/resources/openresty/nginx/sites-enabled/cosSample (limited to 'openresty-ext') diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/conf/nginx.conf b/openresty-ext/src/assembly/resources/openresty/nginx/conf/nginx.conf index c51d10a..743eb31 100644 --- a/openresty-ext/src/assembly/resources/openresty/nginx/conf/nginx.conf +++ b/openresty-ext/src/assembly/resources/openresty/nginx/conf/nginx.conf @@ -1,5 +1,5 @@ # -# Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE) +# Copyright (C) 2017-2018 ZTE, Inc. and others. All rights reserved. (ZTE) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -59,8 +59,10 @@ http { #open_file_cache_errors on; gzip on; - gzip_min_length 1000; - gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml; + gzip_min_length 1k; + gzip_buffers 4 16k; + gzip_comp_level 2; + gzip_types text/plain text/css application/json application/javascript application/x-javascript text/xml application/xml; include ../msb-enabled/*.conf; include ../sites-enabled/*.conf; diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/conf/msbinit.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/conf/msbinit.lua index bc8a13f..ec7d7b0 100644 --- a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/conf/msbinit.lua +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/conf/msbinit.lua @@ -1,6 +1,6 @@ --[[ - Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE) + Copyright (C) 2018 ZTE, Inc. and others. All rights reserved. (ZTE) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -55,4 +55,10 @@ _M.server = { ["fail_timeout"] = 10, ["max_fails"] = 1 } -return _M \ No newline at end of file +_M.dns = { + ["servers"] = mark_empty_as_nil(os.getenv("UPSTREAM_DNS_SERVERS")), + ["cache_positive_ttl"] = 180, --shcache use,in seconds + ["cache_negative_ttl"] = 2, --shcache use,in seconds + ["cache_actualize_ttl"] = 120, --shcache use,in seconds +} +return _M diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/core/error_handler.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/core/error_handler.lua new file mode 100644 index 0000000..f21f10a --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/core/error_handler.lua @@ -0,0 +1,61 @@ +--[[ + + Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE) + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--]] + +local _M = {} +_M._VERSION = '1.0.0' +local msbConf = require('conf.msbinit') +local enablefullsearch = msbConf.systemConf.enablefullsearch +local ngx_var = ngx.var +local error_page_head = '502 Bad Gateway

502 Bad Gateway

error message:' +local error_page_foot = '
nginx
' +local upstream_not_found_err = "service info is incorrect:using own upstream flag is on but upstream name is empty" + +function _M.svc_not_found(err_info,detail_info) + ngx.log(ngx.WARN, ngx.var.request_id.." "..(err_info or "").." detail_info:"..(detail_info or "")) + if enablefullsearch and ngx_var.svc_type ~= "custom" then + -- test against the custom services after the commonrewrite phase + --ngx.status = ngx.HTTP_GONE + return ngx.exec("@commonnotfound"); + else + ngx.status = ngx.HTTP_BAD_GATEWAY + ngx.print(error_page_head..err_info..error_page_foot) + end + return ngx.exit(ngx.status) +end + +function _M.svc_not_allow_access(err_info,detail_info) + ngx.log(ngx.WARN, ngx.var.request_id.." "..(err_info or "").." detail_info:"..(detail_info or "")) + ngx.status = ngx.HTTP_FORBIDDEN + return ngx.exit(ngx.status) +end + +function _M.upstream_not_found() + ngx.log(ngx.WARN, ngx.var.request_id.." "..upstream_not_found_err) + ngx.status = ngx.HTTP_BAD_GATEWAY + ngx.print(error_page_head..upstream_not_found_err..error_page_foot) + return ngx.exit(ngx.status) +end + +function _M.no_server_available(err_info,detail_info) + ngx.log(ngx.WARN, ngx.var.request_id.." "..(err_info or "").." detail_info:"..(detail_info or "")) + ngx.status = ngx.HTTP_BAD_GATEWAY + ngx.print(error_page_head..err_info..error_page_foot) + return ngx.exit(ngx.status) +end + +return _M \ No newline at end of file diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/core/peerwatcher.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/core/peerwatcher.lua new file mode 100644 index 0000000..6b7e522 --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/core/peerwatcher.lua @@ -0,0 +1,107 @@ +--[[ + + Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE) + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--]] + +local _M = { + _VERSION = '1.0.0', + STATUS_OK = 0, STATUS_UNSTABLE = 1, STATUS_ERR = 2 +} +local msbConf = require('conf.msbinit') +local str_format = string.format +local now = ngx.now +local fail_timeout = msbConf.server.fail_timeout or 10 +local max_fails = msbConf.server.max_fails or 1 + +local cluster_status = {} +_M.cluster_status = cluster_status + +function _M.is_server_ok(skey, srv) + return _M.get_srv_status(skey, srv)==_M.STATUS_OK +end + +function _M.get_srv_status(skey, srv) + local server_status = cluster_status[skey] + if not server_status then + return _M.STATUS_OK + end + + local srv_key = str_format("%s:%d", srv.ip, srv.port) + local srv_status = server_status[srv_key] + + if srv_status and srv_status.lastmodify + fail_timeout > now() then + return srv_status.status + end + + return _M.STATUS_OK +end + +function _M.set_srv_status(skey, srv, failed) + local server_status = cluster_status[skey] + if not server_status then + server_status = {} + cluster_status[skey] = server_status + end + + local time_now = now() + local srv_key = str_format("%s:%d", srv.ip, srv.port) + local srv_status = server_status[srv_key] + if not srv_status then -- first set + srv_status = { + status = _M.STATUS_OK, + failed_count = 0, + lastmodify = time_now + } + server_status[srv_key] = srv_status + elseif srv_status.lastmodify + fail_timeout < time_now then -- srv_status expired + srv_status.status = _M.STATUS_OK + srv_status.failed_count = 0 + srv_status.lastmodify = time_now + end + + if failed then + srv_status.failed_count = srv_status.failed_count + 1 + if srv_status.failed_count >= max_fails then + srv_status.status = _M.STATUS_ERR + end + end +end + +function _M.check_and_reset_srv_status_ifneed(skey,servers) + local server_status = cluster_status[skey] + --if disabled servers of the service is empty,do nothing + if not server_status then + ngx.log(ngx.DEBUG, "service:",skey," server_status is nil") + return + end + local need_reset = true + for _, srv in ipairs(servers) do + local srv_key = str_format("%s:%d", srv.ip, srv.port) + local srv_status = server_status[srv_key] + if not (srv_status and srv_status.status == _M.STATUS_ERR and srv_status.lastmodify + fail_timeout > now()) then + --once find the server is not disabled now, no need to reset the status table. break the loop + ngx.log(ngx.DEBUG, "service:",skey," donot need reset,break the loop") + need_reset = false + break + end + end + if need_reset then + ngx.log(ngx.DEBUG, "service:",skey," need reset") + cluster_status[skey] = {} + end +end + +return _M \ No newline at end of file diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/core/router.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/core/router.lua new file mode 100644 index 0000000..8915697 --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/core/router.lua @@ -0,0 +1,293 @@ +--[[ + + Copyright (C) 2017 ZTE, Inc. and others. All rights reserved. (ZTE) + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--]] + +-- unified layer to access back DB, using two levels of cache mechanism(LRUCache and shcache) +local _M = {} +_M._VERSION = '1.0.0' +local msbConf = require('conf.msbinit') +local dbclient = require('dao.db_access') +local tbl_util = require('lib.utils.table_util') +local svc_util = require('lib.utils.svc_util') +local log_util = require('lib.utils.log_util') +local stats = require ('monitor.stats') +local error_handler = require('core.error_handler') +local dns_util = require('lib.utils.dns_util') + +local defaultport = msbConf.systemConf.defaultport +local defaulthttpsport = msbConf.systemConf.defaulthttpsport +local defaultprefix = msbConf.systemConf.defaultprefix +local router_subdomain = msbConf.routerConf.subdomain +local router_defaultprefix = msbConf.routerConf.defaultprefix +local str_sub = string.sub +local str_len = string.len +local str_low = string.lower +local tbl_concat = table.concat +local tbl_isempty = tbl_util.isempty +local svc_is_api_related_types = svc_util.is_api_related_types +local svc_isactive = svc_util.isactive +local svc_get_url = svc_util.get_url +local svc_get_backend_protocol = svc_util.get_backend_protocol +local svc_use_own_upstream = svc_util.use_own_upstream +local svc_enable_refer_match = svc_util.enable_refer_match +local svc_is_allow_access = svc_util.is_allow_access +local ngx_var = ngx.var +local log = log_util.log +local error_svc_not_found = error_handler.svc_not_found +--local error_upstream_not_found = error_handler.upstream_not_found +local error_no_server_available = error_handler.no_server_available +local error_svc_not_allow_access = error_handler.svc_not_allow_access +local dns_query = dns_util.query +local tbl_insert = table.insert + +local enablerefercheck = msbConf.systemConf.enablerefercheck +local useconsultemplate = msbConf.systemConf.useconsultemplate + +local function _get_key_prefix(server_port) + if(not server_port) then + server_port = ngx_var.server_port + end + local svc_name = ngx_var.svc_name + if ("microservices" == svc_name or "msdiscover" == svc_name) then + return defaultprefix + elseif (server_port == defaultport or server_port == defaulthttpsport) then + local m, err = ngx.re.match(ngx_var.host, "(?.+)\\."..router_subdomain,"o") + if m then + return router_defaultprefix..":"..m["hostname"] + else + return defaultprefix + end + else + return "msb:"..server_port + end +end + +local function _load_common_svc_info(svc_type,server_port) + local key_prefix = _get_key_prefix(server_port) + local req_res = ngx_var.req_res + local svc_name = ngx_var.svc_name + local svc_key = "" + local svc_pub_url = "" + if(svc_is_api_related_types(svc_type)) then + -- process version info first + local version1 = ngx_var.svc_version1 + local version2 = ngx_var.svc_version2 + local version = "" + -- check version info appearing befor or after + if(not version2) then version2 = "" end --convert nil to empty sting avoiding throw error + if(not version1 or version1 == "") then + version = version2 + else + version = version1 + ngx_var.req_res = version2..req_res + end + -- remove the slash in front of the version (e.g. /V1.0) + local svc_version=str_sub(version,2,str_len(version)) + svc_key = tbl_concat({key_prefix,"api",svc_name,svc_version},":") + svc_pub_url = "/"..svc_type.."/"..svc_name + if(svc_version ~= "") then svc_pub_url = svc_pub_url.."/"..svc_version end + else + svc_key = tbl_concat({key_prefix,"iui",svc_name},":") + svc_pub_url = "/iui/"..svc_name + end + + local svcinfo = dbclient.load_serviceinfo(svc_key) + if tbl_isempty(svcinfo) then + return nil,"","","Common not match. key--"..svc_key + end + + if not svc_isactive(svcinfo) then + return nil,"","","Common matched but service is disabled! key--"..svc_key + end + + ngx.ctx.svc_pub_url = svc_pub_url + return svcinfo,svc_key,svc_name,"" +end + +local function _load_custom_svc_info(svc_type,server_port) + local key_prefix = _get_key_prefix(server_port) + local get_svckey_custom = function(svcname) + return tbl_concat({key_prefix,"custom",svcname},":") + end + local custom_svc_keypattern = tbl_concat({key_prefix,"custom","*"},":") + local svcnames,err = dbclient.load_customsvcnames(custom_svc_keypattern) + if not svcnames then + error_svc_not_found("Failed to load the route table!","keypattern--"..custom_svc_keypattern) + end + ngx.ctx.svcnames = svcnames + local matchedsvcname + local svcinfo + local svc_key = "" + for _, svcname in ipairs(svcnames) do + if (svcname == "/") then + svc_key = get_svckey_custom(svcname) + local svc_info,err = dbclient.load_serviceinfo(svc_key) + if svc_info and svc_isactive(svc_info)then + matchedsvcname = svcname + svcinfo = svc_info + break + end + end + local from, to, err = ngx.re.find(ngx_var.uri, "^"..svcname.."(/(.*))?$", "jo") + --check whether svcname is the prefix of the req uri + if from then + svc_key = get_svckey_custom(svcname) + local svc_info,err = dbclient.load_serviceinfo(svc_key) + if svc_info and svc_isactive(svc_info) then + matchedsvcname = svcname + svcinfo = svc_info + break + end + else + --do nothing + end + end + --add by wangyg:20160418 special handler for refer + if not matchedsvcname and enablerefercheck then + local refer = ngx_var.http_referer + if(refer and refer~="") then + for _, svcname in ipairs(svcnames) do + local urlreg ="^(https://|http://|)(([1-9]|([1-9]\\d)|(1\\d\\d)|(2([0-4]\\d|5[0-5])))\\.)(([0-9]|([1-9]\\d)|(1\\d\\d)|(2([0-4]\\d|5[0-5])))\\.){2}([1-9]|([1-9]\\d)|(1\\d\\d)|(2([0-4]\\d|5[0-5])))(:\\d{1,5})?"..svcname.."(/(.*))?$"; + local from, to, err = ngx.re.find(refer, urlreg, "jo") + ----check whether svcname is the prefix of the req refer + if from then + svc_key = get_svckey_custom(svcname) + local svc_info,err = dbclient.load_serviceinfo(svc_key) + if svc_info and svc_isactive(svc_info) and svc_enable_refer_match(svc_info) then + matchedsvcname = svcname + svcinfo = svc_info + ngx.ctx.matched_usingrefer = true + log("matched_usingrefer",true) + break + end + end + end + end + end + --end of special handler for refer + if not matchedsvcname or tbl_isempty(svcinfo) then + return nil,"","","Custom not match" + end + ngx.ctx.svc_pub_url = matchedsvcname + return svcinfo,svc_key,matchedsvcname,"" +end + +-- syntax: svc_info,svc_key,matched_svcname,err = _load_service_info(svc_type,server_port) +local function _load_service_info(svc_type,server_port) + if(svc_type ~= "custom") then + return _load_common_svc_info(svc_type,server_port) + else + return _load_custom_svc_info(svc_type,server_port) + end +end + +--------------------------------------------------------------- +--Main Entry of the rewrite module +--------------------------------------------------------------- + +function _M.execute(server_port,system_tag) + --------------------------------------------------------------- + --step1:query the service info from share memory or backend db + -- svc_info: the requested service information + -- svc_key: the redis key + -- matched_svcname: the matched service name in the route table + -- err: the detail error info while load failed + --------------------------------------------------------------- + local svc_type = ngx_var.svc_type + local svc_info,svc_key,matched_svcname,err = _load_service_info(svc_type,server_port) + if(not svc_info) then + error_svc_not_found("No route found for this request!",err) + end + ngx.ctx.svc_key = svc_key + ngx.ctx.svc_info = svc_info + --log the route info + log("matched",matched_svcname) + + if not svc_is_allow_access(system_tag,svc_info) then + error_svc_not_allow_access("Route is not allowed to access!","system_tag:"..system_tag.." svc_key:"..svc_key) + end + --------------------------------------------------------------- + --step2:rewrite the request uri using the svc_info + --------------------------------------------------------------- + local svc_url = svc_get_url(svc_info,svc_type) + local rewrited_uri ="" + if(svc_type ~= "custom") then + rewrited_uri = svc_url..ngx_var.req_res + elseif (matched_svcname == "/") then + --special handling: if "/" matched, contact directly + rewrited_uri = svc_url..ngx_var.uri + else + local newuri,n,err = ngx.re.sub(ngx_var.uri, "^"..matched_svcname.."(/.*)?", svc_url.."$1", "o") + --add by wangyg:20160418 special handler for refer + if(n==0 and ngx.ctx.matched_usingrefer) then newuri = svc_url..ngx_var.uri end --special handling if matched using refer + --end of add by wangyg:20160418 special handler for refer + rewrited_uri = newuri + end + if (rewrited_uri == "") then return ngx.redirect(ngx.var.uri.."/") end + ngx.req.set_uri(rewrited_uri) + ngx.ctx.svc_url = svc_url + + --------------------------------------------------------------- + --step3:process the proxy upstream part + -- con1-using consul template:set the upstream name + -- con2-using msb balancer:query the server list and store in the ctx + --------------------------------------------------------------- + --set the http_protocol used by proxy_pass directive + ngx_var.http_protocol = svc_get_backend_protocol(svc_info) + + --[[ + if svc_use_own_upstream(svc_info) then + ngx.ctx.use_ownupstream = true + end + local consul_servicename = svc_info.spec["consulServiceName"] + if not consul_servicename or consul_servicename == "" then + error_upstream_not_found() + end + ngx_var.backend = consul_servicename + ngx.ctx.use_ownupstream = true + else + ]]-- + local backservers = svc_info.spec.nodes + if tbl_isempty(backservers) then + error_no_server_available("No active backend server found!"," key--"..svc_key) + end + local new_backservers = {} + for i, server in ipairs(backservers) do + local m, err = ngx.re.match(server["ip"], "^([0-9a-zA-Z]([0-9a-zA-Z-]+[\\.]{1})+[a-zA-Z-]+)$","o") + if m then + local ipaddr = dns_query(server["ip"]) + if ipaddr then + local new_server = {} + new_server["ip"] = ipaddr + new_server["port"] = server["port"] + tbl_insert(new_backservers,new_server) + end + else + local new_server = {} + new_server["ip"] = server["ip"] + new_server["port"] = server["port"] + tbl_insert(new_backservers,new_server) + end + end + if tbl_isempty(new_backservers) then + error_no_server_available("The domain name of backendserver was not resolved!"," key--"..svc_key) + end + ngx.ctx.backservers = new_backservers + --end +end + +return _M \ No newline at end of file diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/dao/dao.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/dao/dao.lua index 58d058c..8a69499 100644 --- a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/dao/dao.lua +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/dao/dao.lua @@ -1,6 +1,6 @@ --[[ - Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE) + Copyright (C) 2018 ZTE, Inc. and others. All rights reserved. (ZTE) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -31,6 +31,7 @@ local cacheConf = msbConf.cacheConf local positive_ttl = cacheConf.positive_ttl or 60 local negative_ttl = cacheConf.negative_ttl or 2 local actualize_ttl = cacheConf.actualize_ttl or 120 +local stats_prefix = "monitor:stats:" local svc_shcache = ngx.shared.svc_cache @@ -120,4 +121,48 @@ local function load_customsvcnames(keypattern) end _M.load_customsvcnames = load_customsvcnames -return _M \ No newline at end of file + +function _M.save_reqnum_stats(date,info) + local _db = DB:new(options) + local c, err = _db:connectdb() + if not c then + return nil, err + end + + local red = _db.redis + local resp,err = red:rpush(stats_prefix..date,info) + _db:keepalivedb() + if not resp then + return nil, "save_reqnum_stats failed! key:"..date.." value:"..info + else + return resp,nil + end +end + +function _M.delete_reqnum_stats(date) + local _db = DB:new(options) + local c, err = _db:connectdb() + if not c then + return nil, err + end + local red = _db.redis + red:del(stats_prefix..date) + _db:keepalivedb() +end + +function _M.get_reqnum_stats(date,latest_num) + if(type(latest_num)~="number") then + return nil,"the input num is illegal!" + end + local _db = DB:new(options) + local c, err = _db:connectdb() + if not c then + return nil, err + end + local red = _db.redis + local resp,err = red:lrange(stats_prefix..date,0-latest_num,-1) + _db:keepalivedb() + return resp,nil +end + +return _M diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/lib/utils/dns_util.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/lib/utils/dns_util.lua new file mode 100644 index 0000000..c2ac7b1 --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/lib/utils/dns_util.lua @@ -0,0 +1,99 @@ +--[[ + + Copyright (C) 2018 ZTE, Inc. and others. All rights reserved. (ZTE) + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--]] + +local _M = {} +_M._VERSION = '1.0.0' + +local shcache = require("vendor.shcache") +local msbConf = require('conf.msbinit') +local resolver = require "resty.dns.resolver" +local str_util = require('lib.utils.str_util') + +local dns_cache = ngx.shared.dns_cache +local dns_servers = msbConf.dns.servers +local dns_cache_positive_ttl = msbConf.dns.cache_positive_ttl or 60 +local dns_cache_negative_ttl = msbConf.dns.cache_negative_ttl or 2 +local dns_cache_actualize_ttl = msbConf.dns.cache_actualize_ttl or 120 +local str_split = str_util.split + +local nameservers = nil +ngx.log(ngx.WARN, "environment variable UPSTREAM_DNS_SERVERS:",dns_servers) +local ok,res = pcall(function() return str_split(dns_servers,",") end) +if not ok then + ngx.log(ngx.WARN, "failed to parse the DNS Servers from the environment variable UPSTREAM_DNS_SERVERS"," Error:"..res) +else + nameservers = res +end + +local function query(domain) + -- closure to perform external lookup to redis + local dns_query_from_server = function () + local r, err = resolver:new{ + nameservers = nameservers, + retrans = 5, -- 5 retransmissions on receive timeout + timeout = 2000, -- 2 sec + } + + if not r then + ngx.log(ngx.ERR, "failed to instantiate the resolver:",err) + return nil,"failed to instantiate the resolver:"..err + end + + --local answers, err = r:query("wygtest.service.openpalette") + local answers, err = r:query(domain) + if not answers then + ngx.log(ngx.ERR, "failed to query the DNS server:",err) + return nil,"failed to query the DNS server:"..err + end + + if answers.errcode then + ngx.log(ngx.ERR, "server returned error code: ", answers.errcode, + ": ", answers.errstr) + return nil,"server returned error code: "..answers.errcode.. + ": ".. answers.errstr + end + + for i, ans in ipairs(answers) do + if r.TYPE_A==ans.type and r.CLASS_IN==ans.class then + return ans.address + end + end + return nil,"dns servers return answers,but no server is TYPE_A and CLASS_IN" + end + + local dns_cache_table = shcache:new( + dns_cache, + { external_lookup = dns_query_from_server, + --encode = cmsgpack.pack, + --encode = cjson_safe.encode, + --decode = cmsgpack.unpack + --decode = cjson_safe.decode + }, + { positive_ttl = dns_cache_positive_ttl, -- default cache good data for 60s + negative_ttl = dns_cache_negative_ttl, -- default cache failed lookup for 1s + actualize_ttl = dns_cache_actualize_ttl, + name = 'dns_cache' -- "named" cache, useful for debug / report + } + ) + local server, from_cache = dns_cache_table:load(domain) + + return server +end +_M.query = query + +return _M diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/lib/utils/str_util.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/lib/utils/str_util.lua index 6fa8d39..00996c4 100644 --- a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/lib/utils/str_util.lua +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/lib/utils/str_util.lua @@ -1,6 +1,6 @@ --[[ - Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE) + Copyright (C) 2017-2018 ZTE, Inc. and others. All rights reserved. (ZTE) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -19,6 +19,10 @@ local _M = {} _M._VERSION = '1.0.0' +local pl_stringx = require "pl.stringx" +local split = pl_stringx.split +local strip = pl_stringx.strip + function _M.mark_empty_as_nil(t) if t == "" then return nil @@ -27,4 +31,14 @@ function _M.mark_empty_as_nil(t) end end -return _M \ No newline at end of file +--- splits a string. +-- just a placeholder to the penlight `pl.stringx.split` function +-- @function split +_M.split = split + +--- strips whitespace from a string. +-- just a placeholder to the penlight `pl.stringx.strip` function +-- @function strip +_M.strip = strip + +return _M diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/lib/utils/svc_util.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/lib/utils/svc_util.lua index 226e31f..ce65e14 100644 --- a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/lib/utils/svc_util.lua +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/lib/utils/svc_util.lua @@ -1,6 +1,6 @@ --[[ - Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE) + Copyright (C) 2017-2018 ZTE, Inc. and others. All rights reserved. (ZTE) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -22,19 +22,18 @@ _M._VERSION = '1.0.0' local msbConf= require('conf.msbinit') local svcConf = require('conf.svcconf') local log_util = require('lib.utils.log_util') +local bit = require("bit") local log = log_util.log -local ngx_var = ngx.var - -local defaultport = msbConf.systemConf.defaultport -local defaulthttpsport = msbConf.systemConf.defaulthttpsport -local defaultprefix = msbConf.systemConf.defaultprefix -local router_subdomain = msbConf.routerConf.subdomain -local router_defaultprefix = msbConf.routerConf.defaultprefix local useconsultemplate = msbConf.systemConf.useconsultemplate local urlfieldMap = svcConf.urlfieldMap local apiRelatedTypes = svcConf.apiRelatedTypes +local SYS_SCENARIO_FLAG = { -- cos router + ["ROUTER"] = 1, -- 0 1 + ["COS"] = 2 -- 1 0 +} + function _M.isactive(svcinfo) if svcinfo["status"] == "1" then return true @@ -65,32 +64,76 @@ function _M.get_backend_protocol(svcinfo) end end -function _M.get_key_prefix() - --now assemble the key prefix according the svc_name and server_port - local key_prefix = "" - local server_port = ngx_var.server_port - local svc_name = ngx_var.svc_name - if (svc_name == "microservices" or svc_name == "msdiscover") then - key_prefix = defaultprefix - elseif (server_port == defaultport or server_port == defaulthttpsport) then - local m, err = ngx.re.match(ngx_var.host, "(?.+)\\."..router_subdomain,"o") - if m then - key_prefix = router_defaultprefix..":"..m["hostname"] +function _M.is_api_related_types(svc_type) + if(apiRelatedTypes[svc_type]) then + return true + else + return false + end +end + +function _M.get_connect_timeout(svcinfo) + local connect_timeout = svcinfo.spec["connect_timeout"] + if connect_timeout then + connect_timeout = tonumber(connect_timeout) + if connect_timeout and connect_timeout<=0 then + ngx.log(ngx.WARN, ngx.var.request_id.." ".."bad connect timeout!Zero and negative timeout values are not allowed.Input value:"..connect_timeout) + return nil else - key_prefix = defaultprefix + return connect_timeout end else - key_prefix = "msb:"..server_port + return nil end - return key_prefix end -function _M.is_api_related_types(svc_type) - if(apiRelatedTypes[svc_type]) then +function _M.get_send_timeout(svcinfo) + local send_timeout = svcinfo.spec["send_timeout"] + if send_timeout then + send_timeout = tonumber(send_timeout) + if send_timeout and send_timeout<=0 then + ngx.log(ngx.WARN, ngx.var.request_id.." ".."bad send timeout!Zero and negative timeout values are not allowed.Input value:"..send_timeout) + return nil + else + return send_timeout + end + else + return nil + end +end + +function _M.get_read_timeout(svcinfo) + local read_timeout = svcinfo.spec["read_timeout"] + if read_timeout then + read_timeout = tonumber(read_timeout) + if read_timeout and read_timeout <= 0 then + ngx.log(ngx.WARN, ngx.var.request_id.." ".."bad send timeout!Zero and negative timeout values are not allowed.Input value:"..read_timeout) + return nil + else + return read_timeout + end + else + return nil + end +end + +function _M.enable_refer_match(svcinfo) + local enable_refer_match = svcinfo.spec["enable_refer_match"] + --Be compatible with the old service info. If the field is not filled, the refer match is enabled by default. + if enable_refer_match == nil or enable_refer_match then return true else return false end end -return _M \ No newline at end of file +function _M.is_allow_access(system_tag,svcinfo) + local scenario = svcinfo.spec["scenario"] or 1 + local ok,res = pcall(function() return bit.band(SYS_SCENARIO_FLAG[system_tag], scenario) end) + if ok and res==SYS_SCENARIO_FLAG[system_tag] then + return true + else + return false + end +end +return _M diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/loadbalance/balancer.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/loadbalance/balancer.lua index 48dc1d8..ac9bb1d 100644 --- a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/loadbalance/balancer.lua +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/loadbalance/balancer.lua @@ -1,6 +1,6 @@ --[[ - Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE) + Copyright (C) 2018 ZTE, Inc. and others. All rights reserved. (ZTE) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,14 +18,16 @@ local b = require "ngx.balancer" local baseupstream = require "loadbalance.baseupstream" -local log_util = require('lib.utils.log_util') -local error_handler = require('lib.utils.error_handler') +local stats = require "monitor.stats" +local svc_util = require 'lib.utils.svc_util' + +local servers = ngx.ctx.backservers +local svc_key = ngx.ctx.svc_key +local svc_info = ngx.ctx.svc_info +local svc_get_connect_timeout = svc_util.get_connect_timeout +local svc_get_send_timeout = svc_util.get_send_timeout +local svc_get_read_timeout = svc_util.get_read_timeout -local log = log_util.log -local ngx_ctx = ngx.ctx -local servers = ngx_ctx.backservers -local svc_key = ngx_ctx.svc_key -local error_svc_not_found = error_handler.svc_not_found local status = b.get_last_failure() if status == nil then @@ -35,6 +37,7 @@ elseif status == "failed" then local last_peer = ngx.ctx.last_peer --mark the srv failed one time baseupstream.mark_srv_failed(svc_key,last_peer) + stats.backend_failed() end local server,err = baseupstream.get_backserver(svc_key,servers) @@ -46,5 +49,6 @@ if baseupstream.can_retry(svc_key,servers) then b.set_more_tries(1) end b.set_current_peer(server["ip"],server["port"]) ---log("upstreamserver",server["ip"]..":"..server["port"]) -ngx.ctx.last_peer = { ip=server["ip"], port=server["port"] } \ No newline at end of file +b.set_timeouts(svc_get_connect_timeout(svc_info), svc_get_send_timeout(svc_info), svc_get_read_timeout(svc_info)) +ngx.ctx.last_peer = { ip=server["ip"], port=server["port"] } +stats.forward_backend() diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/loadbalance/baseupstream.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/loadbalance/baseupstream.lua index 4af6dfa..9361eea 100644 --- a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/loadbalance/baseupstream.lua +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/loadbalance/baseupstream.lua @@ -1,6 +1,6 @@ --[[ - Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE) + Copyright (C) 2017-2018 ZTE, Inc. and others. All rights reserved. (ZTE) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -19,10 +19,14 @@ local _M = { _VERSION = '1.0.0' } -local policymodule = require "loadbalance.policy.roundrobin" + +local roundrobin = require "loadbalance.policy.roundrobin" +local consistent_hash = require "loadbalance.policy.consistent_hash" local tbl_util = require('lib.utils.table_util') -local peerwatcher = require "loadbalance.peerwatcher" +local svc_util = require('lib.utils.svc_util') +local peerwatcher = require "core.peerwatcher" local tbl_isempty = tbl_util.isempty +local svc_use_own_upstream = svc_util.use_own_upstream function _M.get_backserver(svc_key,servers) if tbl_isempty(servers) then return nil,"server list is empty" end @@ -42,14 +46,43 @@ function _M.get_backserver(svc_key,servers) return nil,"only one server but is not available" end end - for i=ngx.ctx.tried_num+1,servers_num do - ngx.ctx.tried_num = ngx.ctx.tried_num+1 - server = policymodule.select_backserver(servers,svc_key) - if peerwatcher.is_server_ok(svc_key,server) then - return server,"" - end + + -- A temporary solution, plase modify it when add lb_policy to svc_info + local svc_info = ngx.ctx.svc_info + if svc_use_own_upstream(svc_info) then + svc_info.lb_policy = "ip_hash" + else + svc_info.lb_policy = "roundrobin" end - return nil,"serveral server but no one is available" + + + local mode = svc_info.lb_policy + if mode ~= nil then + if mode == "ip_hash" then + -- iphash + for i=ngx.ctx.tried_num+1,servers_num do + ngx.ctx.tried_num = ngx.ctx.tried_num+1 + server = consistent_hash.select_backserver(servers,svc_key) + if peerwatcher.is_server_ok(svc_key,server) then + return server,"" + end + end + return nil,"serveral server but no one is available" + + elseif mode == "roundrobin" then + -- roundrobin + for i=ngx.ctx.tried_num+1,servers_num do + ngx.ctx.tried_num = ngx.ctx.tried_num+1 + server = roundrobin.select_backserver(servers,svc_key) + if peerwatcher.is_server_ok(svc_key,server) then + return server,"" + end + end + return nil,"serveral server but no one is available" + end + + end + end function _M.can_retry(svc_key,servers) @@ -63,4 +96,4 @@ end function _M.check_and_reset_srv_status_ifneed(svc_key, servers) peerwatcher.check_and_reset_srv_status_ifneed(svc_key,servers) end -return _M \ No newline at end of file +return _M diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/loadbalance/policy/consistent_hash.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/loadbalance/policy/consistent_hash.lua new file mode 100644 index 0000000..b3cd46e --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/loadbalance/policy/consistent_hash.lua @@ -0,0 +1,135 @@ +--[[ + + Copyright (C) 2018 ZTE, Inc. and others. All rights reserved. (ZTE) + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--]] + +local _M = {} +_M._VERSION = '1.0.0' + +local floor = math.floor +local str_byte = string.byte +local tab_sort = table.sort +local tab_insert = table.insert + +local MOD = 2 ^ 32 +local REPLICAS = 20 +local LUCKY_NUM = 13 + + +local tbl_util = require('lib.utils.table_util') +local tbl_isempty = tbl_util.isempty +local tbl_isequal = require('pl.tablex') +local peerwatcher = require "core.peerwatcher" +local ngx_var = ngx.var +local hash_data = {} + +local function hash_string(str) + local key = 0 + for i = 1, #str do + key = (key * 31 + str_byte(str, i)) % MOD + end + return key +end + + +local function init_consistent_hash_state(servers) + local weight_sum = 0 + local weight = 1 + for _, srv in ipairs(servers) do + if srv.weight and srv.weight ~= 0 then + weight = srv.weight + end + weight_sum = weight_sum + weight + end + + local circle, members = {}, 0 + for index, srv in ipairs(servers) do + local key = ("%s:%s"):format(srv.ip, srv.port) + local base_hash = hash_string(key) + for c = 1, REPLICAS * weight_sum do + local hash = (base_hash * c * LUCKY_NUM) % MOD + tab_insert(circle, { hash, index }) + end + + members = members + 1 + end + tab_sort(circle, function(a, b) return a[1] < b[1] end) + return { circle = circle, members = members } +end + +local function update_consistent_hash_state(hash_data,servers,svckey) + -- compare servers in ctx with servers in cache + -- update the hash data if changes occur + local serverscache = hash_data[svckey].servers + tab_sort(serverscache, function(a, b) return a.ip < b.ip end) + tab_sort(servers, function(a, b) return a.ip < b.ip end) + if not tbl_isequal.deepcompare(serverscache, servers, false) then + local tmp_chash = init_consistent_hash_state(servers) + hash_data[svckey].servers =servers + hash_data[svckey].chash = tmp_chash + end +end + +local function binary_search(circle, key) + local size = #circle + local st, ed, mid = 1, size + + while st <= ed do + mid = floor((st + ed) / 2) + if circle[mid][1] < key then + st = mid + 1 + else + ed = mid - 1 + end + end + + return st == size + 1 and 1 or st +end + + +function _M.select_backserver(servers,svckey) + + if hash_data[svckey] == nil then + local tbl = {} + tbl['servers'] = {} + tbl['chash'] = {} + hash_data[svckey] = tbl + end + + if tbl_isempty(hash_data[svckey].servers) then + local tmp_chash = init_consistent_hash_state(servers) + hash_data[svckey].servers = servers + hash_data[svckey].chash = tmp_chash + else + update_consistent_hash_state(hash_data,servers,svckey) + end + + local chash = hash_data[svckey].chash + local circle = chash.circle + local hash_key = ngx_var.remote_addr + local st = binary_search(circle, hash_string(hash_key)) + local size = #circle + local ed = st + size - 1 + for i = st, ed do + local idx = circle[(i - 1) % size + 1][2] + if peerwatcher.is_server_ok(svckey,hash_data[svckey].servers[idx]) then + return hash_data[svckey].servers[idx] + end + end + return nil, "consistent hash: no servers available" +end + +return _M diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/monitor/stats.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/monitor/stats.lua new file mode 100644 index 0000000..7a23e64 --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/monitor/stats.lua @@ -0,0 +1,177 @@ +--[[ + + Copyright (C) 2017 ZTE, Inc. and others. All rights reserved. (ZTE) + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--]] + +local _M = {} +_M._VERSION = '1.0.0' + +local cjson_safe = require "cjson.safe" +local dao = require('dao.dao') +local tbl_util = require('lib.utils.table_util') + +local stats_cache = ngx.shared.stats +local new_timer = ngx.timer.at +local tbl_isempty = tbl_util.isempty +local accept_preparing_forward_key = "accept_preparing_forward" +local forward_waiting_response_key = "forward_waiting_response" +local receive_resp_not_return_key = "receive_resp_not_return" +local req_num_stats_key = "req_num_stats" +local reset_delay = 60 -- in seconds +local remove_delay = 24*60*60 -- in seconds,24 hours +local lastday_stats = "" + + +function _M.accept_new_request() + local newval, err = stats_cache:incr(accept_preparing_forward_key, 1, 0) + if (not newval) then + ngx.log(ngx.ERR, "increase number of accept_not_forward_key error:", err) + end +end + +function _M.forward_backend() + if not ngx.ctx.not_first_forward then + local newval, err = stats_cache:incr(accept_preparing_forward_key, -1, 1) + if (not newval) then + ngx.log(ngx.ERR, "decrease number of accept_preparing_forward error:", err) + end + end + local newval, err = stats_cache:incr(forward_waiting_response_key, 1, 0) + if (not newval) then + ngx.log(ngx.ERR, "increase number of forward_waiting_response error:", err) + end + ngx.ctx.waiting_backend_resp = true + ngx.ctx.not_first_forward = true +end + +function _M.backend_failed() + ngx.ctx.waiting_backend_resp = false + local newval, err = stats_cache:incr(forward_waiting_response_key, -1, 1) + if (not newval) then + ngx.log(ngx.ERR, "decrease number of forward_waiting_response error:", err) + end +end + +function _M.receive_response() + if ngx.ctx.waiting_backend_resp then + ngx.ctx.waiting_backend_resp = false + local newval, err = stats_cache:incr(forward_waiting_response_key, -1, 1) + if (not newval) then + ngx.log(ngx.ERR, "decrease number of forward_waiting_response error:", err) + end + else + local newval, err = stats_cache:incr(accept_preparing_forward_key, -1, 1) + if (not newval) then + ngx.log(ngx.ERR, "decrease number of accept_preparing_forward error:", err) + end + end + local newval, err = stats_cache:incr(receive_resp_not_return_key, 1, 0) + if (not newval) then + ngx.log(ngx.ERR, "increase number of receive_resp_not_return error:", err) + end +end + +function _M.return_response() + local newval, err = stats_cache:incr(receive_resp_not_return_key, -1, 1) + if (not newval) then + ngx.log(ngx.ERR, "decrease number of receive_resp_not_return error:", err) + end + + local newval, err = stats_cache:incr(req_num_stats_key, 1, 0) + if (not newval) then + ngx.log(ngx.ERR, "increase number of total_req_num error:", err) + end +end + +function _M.format_req_status() + local req_status = {} + req_status[accept_preparing_forward_key] = stats_cache:get(accept_preparing_forward_key) or 0 + req_status[forward_waiting_response_key] = stats_cache:get(forward_waiting_response_key) or 0 + req_status[receive_resp_not_return_key] = stats_cache:get(receive_resp_not_return_key) or 0 + local value, err = cjson_safe.encode(req_status) + if err then + return "Collect real-time request status failed! Error:" .. err + end + return value +end + +function _M.format_conn_status() + local conn_status = {} + conn_status["connections_active"] = ngx.var.connections_active + conn_status["connections_reading"] = ngx.var.connections_reading + conn_status["connections_writing"] = ngx.var.connections_writing + conn_status["connections_waiting"] = ngx.var.connections_waiting + + local value, err = cjson_safe.encode(conn_status) + if err then + return "Collect real-time connection status failed! Error:" .. err + end + return value +end + +function _M.get_reqnum_stats(latest_num) + local resp = dao.get_reqnum_stats(ngx.today(), latest_num) + if tbl_isempty(resp) then + return "[]" + end + return cjson_safe.encode(resp) +end + +_reset_reqnum_stats = function(premature) + if premature then return end + local count = stats_cache:get(req_num_stats_key) or 0 + dao.save_reqnum_stats(ngx.today(), ngx.time().."|"..count) + stats_cache:set(req_num_stats_key, 0) + + local ok, err = new_timer(reset_delay, _reset_reqnum_stats) + if not ok then + ngx.log(ngx.ERR, "failed to create _reset_reqnum_stats timer: ", err) + return + end +end + +_remove_old_stats = function(premature) + if premature then return end + ngx.update_time() + if lastday_stats ~= "" and lastday_stats ~= ngx.today() then + ngx.log(ngx.ERR, "delete old data ") + dao.delete_reqnum_stats(lastday_stats) + end + local ok, err = new_timer(remove_delay, _remove_old_stats) + if not ok then + ngx.log(ngx.ERR, "failed to create _remove_old_stats timer: ", err) + return + end + lastday_stats = ngx.today() +end + +function _M.init_timer() + if 0 == ngx.worker.id() then + local ok, err = new_timer(reset_delay, _reset_reqnum_stats) + if not ok then + ngx.log(ngx.ERR, "failed to create _reset_reqnum_stats timer: ", err) + return + end + + ok, err = new_timer(10, _remove_old_stats) + if not ok then + ngx.log(ngx.ERR, "failed to create _remove_old_stats timer: ", err) + return + end + end +end + +return _M \ No newline at end of file diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/msb.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/msb.lua index f791bdb..c454173 100644 --- a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/msb.lua +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/msb.lua @@ -1,6 +1,6 @@ --[[ - Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE) + Copyright (C) 2017-2018 ZTE, Inc. and others. All rights reserved. (ZTE) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -22,9 +22,11 @@ _M._DESCRIPTION = 'msb plugins controller' local default_conf = require('plugins.config_default') local custom_conf = require('plugins.config_custom') +local msb_router= require('core.router') local table_insert = table.insert local string_find = string.find local str_low = string.lower +local ngx_var = ngx.var --- Borrowed from Kong --- Try to load a module. @@ -64,7 +66,7 @@ function _M.load_plugins() if not loaded then error("The following plugin has been enabled in the configuration but it is not installed on the system: "..v) end - ngx.log(ngx.DEBUG, "Loading plugin: "..v) + ngx.log(ngx.WARN, "Loading plugin: "..v) table_insert(plugins, { name = v, handler = plugin_handler_mod() @@ -73,6 +75,52 @@ function _M.load_plugins() package.loaded.plugins = plugins end +function _M.filter_websocket_req() + local http_upgrade = ngx_var.http_upgrade + if http_upgrade and str_low(http_upgrade) == "websocket" then + --ngx.log(ngx.ERR, "Websocket request and redirect to @commonwebsocket") + return ngx.exec("@websocket"); + end +end + +function _M.route() + msb_router.execute(ngx_var.server_port,"ROUTER") +end + +local function prepare_route() + local uri = ngx_var.uri + local m, err = ngx.re.match(uri, "^/(api|admin|apijson)(/[Vv]\\d+(?:\\.\\d+)*)?/([^/]+)(/[Vv]\\d+(?:\\.\\d+)*)?(.*)", "o") + if m then + ngx_var.svc_type = m[1] + ngx_var.svc_name = m[3] + ngx_var.svc_version1 = m[2] or "" + ngx_var.svc_version2 = m[4] or "" + ngx_var.req_res = m[5] + return + end + local m, err = ngx.re.match(uri, "^/iui/([^/]+)(.*)", "o") + if m then + ngx_var.svc_type = "iui" + ngx_var.svc_name = m[1] + ngx_var.req_res = m[2] + return + end + ngx_var.svc_type = "custom" + return +end + +function _M.external_route(server_port,system_tag) + if not server_port or not system_tag then + local err = "server_port and system_tag are required while routing!" + ngx.log(ngx.WARN, ngx.var.request_id.." "..err) + ngx.status = ngx.HTTP_BAD_GATEWAY + ngx.print(err) + return ngx.exit(ngx.status) + end + prepare_route() + msb_router.execute(server_port,system_tag) +end + function _M.access() local plugins = package.loaded.plugins for _, plugin in ipairs(plugins) do @@ -87,4 +135,4 @@ function _M.header_filter() end end -return _M \ No newline at end of file +return _M diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/plugins/config_default.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/plugins/config_default.lua index cbb3107..3790453 100644 --- a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/plugins/config_default.lua +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/plugins/config_default.lua @@ -1,6 +1,6 @@ --[[ - Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE) + Copyright (C) 2017-2018 ZTE, Inc. and others. All rights reserved. (ZTE) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -19,9 +19,10 @@ local _M = {} _M._VERSION = '1.0.0' _M._DESCRIPTION = 'config_default' - +local str_util = require('lib.utils.str_util') +local mark_empty_as_nil = str_util.mark_empty_as_nil _M.plugins_default = { - {["name"] = "redirect-transformer",["status"] = "on"} + {["name"] = "redirect-transformer",["status"] = mark_empty_as_nil(os.getenv("MSB_REDIRECT_TRANSFORMER_PLUGIN")) or "on"} } -return _M \ No newline at end of file +return _M diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/plugins/redirect-transformer/handler.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/plugins/redirect-transformer/handler.lua index 72e3330..f469aa0 100644 --- a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/plugins/redirect-transformer/handler.lua +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/plugins/redirect-transformer/handler.lua @@ -1,6 +1,6 @@ --[[ - Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE) + Copyright (C) 201-2018 ZTE, Inc. and others. All rights reserved. (ZTE) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -19,8 +19,9 @@ local BasePlugin = require "plugins.base_plugin" local msbConf = require('conf.msbinit') local log_util = require('lib.utils.log_util') +local url_matcher = require "plugins.redirect-transformer.url_matcher" local log = log_util.log - +local url_match_msb_route = url_matcher.is_match_msb_route local RedirectTransformerPluginHandler = BasePlugin:extend() function RedirectTransformerPluginHandler:new() @@ -30,13 +31,40 @@ end function RedirectTransformerPluginHandler:header_filter() RedirectTransformerPluginHandler.super.header_filter(self) local originloc = ngx.header.Location + local newloc if(originloc) then - local newloc = ngx.re.sub(originloc, "^(https|http)(.*)", ngx.var.scheme.."$2", "oi") + log("origin location:",originloc) + local patten_conform,route_match = url_match_msb_route(originloc) + if not patten_conform then + log("redirect-transformer output:","The redirect address may be outside msb, do nothing temporarily.") + return + end + + if route_match then + --if the redirect address can be forwarded by msb,then donot modify it's url + newloc = ngx.re.sub(originloc, "^(https|http)(.*)", ngx.var.scheme.."$2", "oi") + else + --if the redirect address can not be forwarded by msb,then try to modify it's url + local svc_pub_url = ngx.ctx.svc_pub_url + local svc_url = ngx.ctx.svc_url + if(svc_pub_url and svc_pub_url == "/") then + --replace $svc_url with "" + newloc = ngx.re.sub(originloc, "^(https|http)://([^/]+)"..svc_url, ngx.var.scheme.."://".."$2", "oi") + else + --replace $svc_url with $svc_pub_url + newloc = ngx.re.sub(originloc, "^(https|http)://([^/]+)"..svc_url, ngx.var.scheme.."://".."$2"..svc_pub_url, "oi") + end + end + -- replace the backend server with the host of msb + local last_peer = ngx.ctx.last_peer + if last_peer then + local backend_ip = ngx.re.gsub(last_peer.ip, "\\.", "\\.", "o") + newloc = ngx.re.sub(newloc, "^(https://|http://)"..backend_ip..":"..last_peer.port, "$1"..ngx.var.host..":"..ngx.var.server_port, "o") + end ngx.header["Location"] = newloc - log("origin Location:",originloc) - log("req scheme:",ngx.var.scheme) - log("new Location:",newloc) + log("redirect-transformer output:","replace the redirect address to :"..newloc) + ngx.log(ngx.WARN, "redirect-transformer replace the redirect address to:"..newloc, " origin location:",originloc) end end -return RedirectTransformerPluginHandler \ No newline at end of file +return RedirectTransformerPluginHandler diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/plugins/redirect-transformer/url_matcher.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/plugins/redirect-transformer/url_matcher.lua new file mode 100644 index 0000000..41f7602 --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/plugins/redirect-transformer/url_matcher.lua @@ -0,0 +1,164 @@ +--[[ + + Copyright (C) 2018 ZTE, Inc. and others. All rights reserved. (ZTE) + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--]] + +local _M = {} +_M._VERSION = '1.0.0' + +local tbl_util = require('lib.utils.table_util') +local dbclient = require('dao.db_access') +local msbConf = require('conf.msbinit') + +local tbl_concat = table.concat +local defaultport = msbConf.systemConf.defaultport +local defaulthttpsport = msbConf.systemConf.defaulthttpsport +local defaultprefix = msbConf.systemConf.defaultprefix +local router_subdomain = msbConf.routerConf.subdomain +local router_defaultprefix = msbConf.routerConf.defaultprefix +local tbl_isempty = tbl_util.isempty + +local function _get_key_prefix(scheme,host,server_port) + if(not server_port) then + if(scheme == "https") then + server_port = 443 + else + server_port = 80 + end + end + if (server_port == defaultport or server_port == defaulthttpsport) then + local m, err = ngx.re.match(host, "(?.+)\\."..router_subdomain,"o") + if m then + return router_defaultprefix..":"..m["hostname"] + else + return defaultprefix + end + else + return "msb:"..server_port + end +end + +local function _is_match_route_api(uri,key_prefix) + local svc_name,svc_version + local m, err = ngx.re.match(uri, "^/(api|admin|apijson)(/[Vv]\\d+(?:\\.\\d+)*)?/([^/]+)(/[Vv]\\d+(?:\\.\\d+)*)?(.*)", "o") + if m then + svc_name = m[3] + local svc_version1 = m[2] or "" + local svc_version2 = m[4] or "" + if(not svc_version1 or svc_version1 == "") then + svc_version = svc_version2 + else + svc_version = svc_version1 + end + local svc_key = tbl_concat({key_prefix,"api",svc_name,svc_version},":") + local svcinfo = dbclient.load_serviceinfo(svc_key) + if tbl_isempty(svcinfo) then + return false + else + return true + end + end + return false +end + +local function _is_match_route_iui(uri,key_prefix) + local m, err = ngx.re.match(uri, "^/iui/([^/]+)(.*)", "o") + if m then + local svc_name = m[1] + local svc_key = tbl_concat({key_prefix,"iui",svc_name},":") + local svcinfo = dbclient.load_serviceinfo(svc_key) + if tbl_isempty(svcinfo) then + return false + else + return true + end + end + return false +end + +local function _is_match_route_custom(uri,key_prefix) + --[[ + local custom_svc_keypattern = tbl_concat({key_prefix,"custom","*"},":") + local svcnames,err = dbclient.load_customsvcnames(custom_svc_keypattern) + ]] + local svcnames = ngx.ctx.svcnames + if not svcnames then + return false + end + for _, svcname in ipairs(svcnames) do + if (svcname == "/") then + return true + end + local from, to, err = ngx.re.find(uri, "^"..svcname.."(/(.*))?$", "jo") + --check whether svcname is the prefix of the req uri + if from then + return true + else + --do nothing + end + end + return false +end + +local function _is_patten_conform(host) + local m1, err = ngx.re.match(host, "^([0-9a-zA-Z]([0-9a-zA-Z-]+[\\.]{1})+[a-zA-Z-]+)$","o") + if m1 then + -- domain + local m2, err = ngx.re.match(host, "(?.+)\\."..router_subdomain,"o") + if m2 then + return true + else + return false + end + else + --ip + if host == ngx.var.host then + return true + else + local last_peer = ngx.ctx.last_peer + if(last_peer and host == last_peer.ip) then + return true + else + return false + end + end + end +end + +-- syntax: patten_conform,route_match = is_match_msb_route(location) +function _M.is_match_msb_route(location) + local m, err = ngx.re.match(location, "^(\\w+)://([^/:]*)(?::(\\d+))?([^/?]*).*", "o") + local scheme,host,port,uri + if m then + scheme = m[1] + host = m[2] + port = m[3] + uri = m[4] + else + return false,false --It is not normal to enter this branch. This match result just let redirect transformer ignore this request(do nothing) + end + + -- check the host whether conform to msb rules + if not _is_patten_conform(host) then + return false,false + end + + local key_prefix = _get_key_prefix(scheme,host,port) + --return true,_is_match_route_api(uri,key_prefix) or _is_match_route_iui(uri,key_prefix) or _is_match_route_custom(uri,key_prefix) + return true,_is_match_route_custom(uri,key_prefix) +end + +return _M diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/pl/List.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/pl/List.lua new file mode 100644 index 0000000..95d8c0e --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/pl/List.lua @@ -0,0 +1,566 @@ +--- Python-style list class. +-- +-- **Please Note**: methods that change the list will return the list. +-- This is to allow for method chaining, but please note that `ls = ls:sort()` +-- does not mean that a new copy of the list is made. In-place (mutable) methods +-- are marked as returning 'the list' in this documentation. +-- +-- See the Guide for further @{02-arrays.md.Python_style_Lists|discussion} +-- +-- See http://www.python.org/doc/current/tut/tut.html, section 5.1 +-- +-- **Note**: The comments before some of the functions are from the Python docs +-- and contain Python code. +-- +-- Written for Lua version Nick Trout 4.0; Redone for Lua 5.1, Steve Donovan. +-- +-- Dependencies: `pl.utils`, `pl.tablex`, `pl.class` +-- @classmod pl.List +-- @pragma nostrip + +local tinsert,tremove,concat,tsort = table.insert,table.remove,table.concat,table.sort +local setmetatable, getmetatable,type,tostring,string = setmetatable,getmetatable,type,tostring,string +local tablex = require 'pl.tablex' +local filter,imap,imap2,reduce,transform,tremovevalues = tablex.filter,tablex.imap,tablex.imap2,tablex.reduce,tablex.transform,tablex.removevalues +local tsub = tablex.sub +local utils = require 'pl.utils' +local class = require 'pl.class' + +local array_tostring,split,assert_arg,function_arg = utils.array_tostring,utils.split,utils.assert_arg,utils.function_arg +local normalize_slice = tablex._normalize_slice + +-- metatable for our list and map objects has already been defined.. +local Multimap = utils.stdmt.MultiMap +local List = utils.stdmt.List + +local iter + +class(nil,nil,List) + +-- we want the result to be _covariant_, i.e. t must have type of obj if possible +local function makelist (t,obj) + local klass = List + if obj then + klass = getmetatable(obj) + end + return setmetatable(t,klass) +end + +local function simple_table(t) + return type(t) == 'table' and not getmetatable(t) and #t > 0 +end + +function List._create (src) + if simple_table(src) then return src end +end + +function List:_init (src) + if self == src then return end -- existing table used as self! + if src then + for v in iter(src) do + tinsert(self,v) + end + end +end + +--- Create a new list. Can optionally pass a table; +-- passing another instance of List will cause a copy to be created; +-- this will return a plain table with an appropriate metatable. +-- we pass anything which isn't a simple table to iterate() to work out +-- an appropriate iterator +-- @see List.iterate +-- @param[opt] t An optional list-like table +-- @return a new List +-- @usage ls = List(); ls = List {1,2,3,4} +-- @function List.new + +List.new = List + +--- Make a copy of an existing list. +-- The difference from a plain 'copy constructor' is that this returns +-- the actual List subtype. +function List:clone() + local ls = makelist({},self) + ls:extend(self) + return ls +end + +---Add an item to the end of the list. +-- @param i An item +-- @return the list +function List:append(i) + tinsert(self,i) + return self +end + +List.push = tinsert + +--- Extend the list by appending all the items in the given list. +-- equivalent to 'a[len(a):] = L'. +-- @tparam List L Another List +-- @return the list +function List:extend(L) + assert_arg(1,L,'table') + for i = 1,#L do tinsert(self,L[i]) end + return self +end + +--- Insert an item at a given position. i is the index of the +-- element before which to insert. +-- @int i index of element before whichh to insert +-- @param x A data item +-- @return the list +function List:insert(i, x) + assert_arg(1,i,'number') + tinsert(self,i,x) + return self +end + +--- Insert an item at the begining of the list. +-- @param x a data item +-- @return the list +function List:put (x) + return self:insert(1,x) +end + +--- Remove an element given its index. +-- (equivalent of Python's del s[i]) +-- @int i the index +-- @return the list +function List:remove (i) + assert_arg(1,i,'number') + tremove(self,i) + return self +end + +--- Remove the first item from the list whose value is given. +-- (This is called 'remove' in Python; renamed to avoid confusion +-- with table.remove) +-- Return nil if there is no such item. +-- @param x A data value +-- @return the list +function List:remove_value(x) + for i=1,#self do + if self[i]==x then tremove(self,i) return self end + end + return self + end + +--- Remove the item at the given position in the list, and return it. +-- If no index is specified, a:pop() returns the last item in the list. +-- The item is also removed from the list. +-- @int[opt] i An index +-- @return the item +function List:pop(i) + if not i then i = #self end + assert_arg(1,i,'number') + return tremove(self,i) +end + +List.get = List.pop + +--- Return the index in the list of the first item whose value is given. +-- Return nil if there is no such item. +-- @function List:index +-- @param x A data value +-- @int[opt=1] idx where to start search +-- @return the index, or nil if not found. + +local tfind = tablex.find +List.index = tfind + +--- does this list contain the value?. +-- @param x A data value +-- @return true or false +function List:contains(x) + return tfind(self,x) and true or false +end + +--- Return the number of times value appears in the list. +-- @param x A data value +-- @return number of times x appears +function List:count(x) + local cnt=0 + for i=1,#self do + if self[i]==x then cnt=cnt+1 end + end + return cnt +end + +--- Sort the items of the list, in place. +-- @func[opt='<'] cmp an optional comparison function +-- @return the list +function List:sort(cmp) + if cmp then cmp = function_arg(1,cmp) end + tsort(self,cmp) + return self +end + +--- return a sorted copy of this list. +-- @func[opt='<'] cmp an optional comparison function +-- @return a new list +function List:sorted(cmp) + return List(self):sort(cmp) +end + +--- Reverse the elements of the list, in place. +-- @return the list +function List:reverse() + local t = self + local n = #t + for i = 1,n/2 do + t[i],t[n] = t[n],t[i] + n = n - 1 + end + return self +end + +--- return the minimum and the maximum value of the list. +-- @return minimum value +-- @return maximum value +function List:minmax() + local vmin,vmax = 1e70,-1e70 + for i = 1,#self do + local v = self[i] + if v < vmin then vmin = v end + if v > vmax then vmax = v end + end + return vmin,vmax +end + +--- Emulate list slicing. like 'list[first:last]' in Python. +-- If first or last are negative then they are relative to the end of the list +-- eg. slice(-2) gives last 2 entries in a list, and +-- slice(-4,-2) gives from -4th to -2nd +-- @param first An index +-- @param last An index +-- @return a new List +function List:slice(first,last) + return tsub(self,first,last) +end + +--- empty the list. +-- @return the list +function List:clear() + for i=1,#self do tremove(self) end + return self +end + +local eps = 1.0e-10 + +--- Emulate Python's range(x) function. +-- Include it in List table for tidiness +-- @int start A number +-- @int[opt] finish A number greater than start; if absent, +-- then start is 1 and finish is start +-- @int[opt=1] incr an increment (may be less than 1) +-- @return a List from start .. finish +-- @usage List.range(0,3) == List{0,1,2,3} +-- @usage List.range(4) = List{1,2,3,4} +-- @usage List.range(5,1,-1) == List{5,4,3,2,1} +function List.range(start,finish,incr) + if not finish then + finish = start + start = 1 + end + if incr then + assert_arg(3,incr,'number') + if math.ceil(incr) ~= incr then finish = finish + eps end + else + incr = 1 + end + assert_arg(1,start,'number') + assert_arg(2,finish,'number') + local t = List() + for i=start,finish,incr do tinsert(t,i) end + return t +end + +--- list:len() is the same as #list. +function List:len() + return #self +end + +-- Extended operations -- + +--- Remove a subrange of elements. +-- equivalent to 'del s[i1:i2]' in Python. +-- @int i1 start of range +-- @int i2 end of range +-- @return the list +function List:chop(i1,i2) + return tremovevalues(self,i1,i2) +end + +--- Insert a sublist into a list +-- equivalent to 's[idx:idx] = list' in Python +-- @int idx index +-- @tparam List list list to insert +-- @return the list +-- @usage l = List{10,20}; l:splice(2,{21,22}); assert(l == List{10,21,22,20}) +function List:splice(idx,list) + assert_arg(1,idx,'number') + idx = idx - 1 + local i = 1 + for v in iter(list) do + tinsert(self,i+idx,v) + i = i + 1 + end + return self +end + +--- general slice assignment s[i1:i2] = seq. +-- @int i1 start index +-- @int i2 end index +-- @tparam List seq a list +-- @return the list +function List:slice_assign(i1,i2,seq) + assert_arg(1,i1,'number') + assert_arg(1,i2,'number') + i1,i2 = normalize_slice(self,i1,i2) + if i2 >= i1 then self:chop(i1,i2) end + self:splice(i1,seq) + return self +end + +--- concatenation operator. +-- @within metamethods +-- @tparam List L another List +-- @return a new list consisting of the list with the elements of the new list appended +function List:__concat(L) + assert_arg(1,L,'table') + local ls = self:clone() + ls:extend(L) + return ls +end + +--- equality operator ==. True iff all elements of two lists are equal. +-- @within metamethods +-- @tparam List L another List +-- @return true or false +function List:__eq(L) + if #self ~= #L then return false end + for i = 1,#self do + if self[i] ~= L[i] then return false end + end + return true +end + +--- join the elements of a list using a delimiter. +-- This method uses tostring on all elements. +-- @string[opt=''] delim a delimiter string, can be empty. +-- @return a string +function List:join (delim) + delim = delim or '' + assert_arg(1,delim,'string') + return concat(array_tostring(self),delim) +end + +--- join a list of strings.
+-- Uses `table.concat` directly. +-- @function List:concat +-- @string[opt=''] delim a delimiter +-- @return a string +List.concat = concat + +local function tostring_q(val) + local s = tostring(val) + if type(val) == 'string' then + s = '"'..s..'"' + end + return s +end + +--- how our list should be rendered as a string. Uses join(). +-- @within metamethods +-- @see List:join +function List:__tostring() + return '{'..self:join(',',tostring_q)..'}' +end + +--- call the function on each element of the list. +-- @func fun a function or callable object +-- @param ... optional values to pass to function +function List:foreach (fun,...) + fun = function_arg(1,fun) + for i = 1,#self do + fun(self[i],...) + end +end + +local function lookup_fun (obj,name) + local f = obj[name] + if not f then error(type(obj).." does not have method "..name,3) end + return f +end + +--- call the named method on each element of the list. +-- @string name the method name +-- @param ... optional values to pass to function +function List:foreachm (name,...) + for i = 1,#self do + local obj = self[i] + local f = lookup_fun(obj,name) + f(obj,...) + end +end + +--- create a list of all elements which match a function. +-- @func fun a boolean function +-- @param[opt] arg optional argument to be passed as second argument of the predicate +-- @return a new filtered list. +function List:filter (fun,arg) + return makelist(filter(self,fun,arg),self) +end + +--- split a string using a delimiter. +-- @string s the string +-- @string[opt] delim the delimiter (default spaces) +-- @return a List of strings +-- @see pl.utils.split +function List.split (s,delim) + assert_arg(1,s,'string') + return makelist(split(s,delim)) +end + +--- apply a function to all elements. +-- Any extra arguments will be passed to the function. +-- @func fun a function of at least one argument +-- @param ... arbitrary extra arguments. +-- @return a new list: {f(x) for x in self} +-- @usage List{'one','two'}:map(string.upper) == {'ONE','TWO'} +-- @see pl.tablex.imap +function List:map (fun,...) + return makelist(imap(fun,self,...),self) +end + +--- apply a function to all elements, in-place. +-- Any extra arguments are passed to the function. +-- @func fun A function that takes at least one argument +-- @param ... arbitrary extra arguments. +-- @return the list. +function List:transform (fun,...) + transform(fun,self,...) + return self +end + +--- apply a function to elements of two lists. +-- Any extra arguments will be passed to the function +-- @func fun a function of at least two arguments +-- @tparam List ls another list +-- @param ... arbitrary extra arguments. +-- @return a new list: {f(x,y) for x in self, for x in arg1} +-- @see pl.tablex.imap2 +function List:map2 (fun,ls,...) + return makelist(imap2(fun,self,ls,...),self) +end + +--- apply a named method to all elements. +-- Any extra arguments will be passed to the method. +-- @string name name of method +-- @param ... extra arguments +-- @return a new list of the results +-- @see pl.seq.mapmethod +function List:mapm (name,...) + local res = {} + for i = 1,#self do + local val = self[i] + local fn = lookup_fun(val,name) + res[i] = fn(val,...) + end + return makelist(res,self) +end + +local function composite_call (method,f) + return function(self,...) + return self[method](self,f,...) + end +end + +function List.default_map_with(T) + return function(self,name) + local m + if T then + local f = lookup_fun(T,name) + m = composite_call('map',f) + else + m = composite_call('mapn',name) + end + getmetatable(self)[name] = m -- and cache.. + return m + end +end + +List.default_map = List.default_map_with + +--- 'reduce' a list using a binary function. +-- @func fun a function of two arguments +-- @return result of the function +-- @see pl.tablex.reduce +function List:reduce (fun) + return reduce(fun,self) +end + +--- partition a list using a classifier function. +-- The function may return nil, but this will be converted to the string key ''. +-- @func fun a function of at least one argument +-- @param ... will also be passed to the function +-- @treturn MultiMap a table where the keys are the returned values, and the values are Lists +-- of values where the function returned that key. +-- @see pl.MultiMap +function List:partition (fun,...) + fun = function_arg(1,fun) + local res = {} + for i = 1,#self do + local val = self[i] + local klass = fun(val,...) + if klass == nil then klass = '' end + if not res[klass] then res[klass] = List() end + res[klass]:append(val) + end + return setmetatable(res,Multimap) +end + +--- return an iterator over all values. +function List:iter () + return iter(self) +end + +--- Create an iterator over a seqence. +-- This captures the Python concept of 'sequence'. +-- For tables, iterates over all values with integer indices. +-- @param seq a sequence; a string (over characters), a table, a file object (over lines) or an iterator function +-- @usage for x in iterate {1,10,22,55} do io.write(x,',') end ==> 1,10,22,55 +-- @usage for ch in iterate 'help' do do io.write(ch,' ') end ==> h e l p +function List.iterate(seq) + if type(seq) == 'string' then + local idx = 0 + local n = #seq + local sub = string.sub + return function () + idx = idx + 1 + if idx > n then return nil + else + return sub(seq,idx,idx) + end + end + elseif type(seq) == 'table' then + local idx = 0 + local n = #seq + return function() + idx = idx + 1 + if idx > n then return nil + else + return seq[idx] + end + end + elseif type(seq) == 'function' then + return seq + elseif type(seq) == 'userdata' and io.type(seq) == 'file' then + return seq:lines() + end +end +iter = List.iterate + +return List + diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/pl/class.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/pl/class.lua new file mode 100644 index 0000000..a57ac71 --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/pl/class.lua @@ -0,0 +1,261 @@ +--- Provides a reuseable and convenient framework for creating classes in Lua. +-- Two possible notations: +-- +-- B = class(A) +-- class.B(A) +-- +-- The latter form creates a named class within the current environment. Note +-- that this implicitly brings in `pl.utils` as a dependency. +-- +-- See the Guide for further @{01-introduction.md.Simplifying_Object_Oriented_Programming_in_Lua|discussion} +-- @module pl.class + +local error, getmetatable, io, pairs, rawget, rawset, setmetatable, tostring, type = + _G.error, _G.getmetatable, _G.io, _G.pairs, _G.rawget, _G.rawset, _G.setmetatable, _G.tostring, _G.type +local compat + +-- this trickery is necessary to prevent the inheritance of 'super' and +-- the resulting recursive call problems. +local function call_ctor (c,obj,...) + -- nice alias for the base class ctor + local base = rawget(c,'_base') + if base then + local parent_ctor = rawget(base,'_init') + while not parent_ctor do + base = rawget(base,'_base') + if not base then break end + parent_ctor = rawget(base,'_init') + end + if parent_ctor then + rawset(obj,'super',function(obj,...) + call_ctor(base,obj,...) + end) + end + end + local res = c._init(obj,...) + rawset(obj,'super',nil) + return res +end + +--- initializes an __instance__ upon creation. +-- @function class:_init +-- @param ... parameters passed to the constructor +-- @usage local Cat = class() +-- function Cat:_init(name) +-- --self:super(name) -- call the ancestor initializer if needed +-- self.name = name +-- end +-- +-- local pussycat = Cat("pussycat") +-- print(pussycat.name) --> pussycat + +--- checks whether an __instance__ is derived from some class. +-- Works the other way around as `class_of`. It has two ways of using; +-- 1) call with a class to check against, 2) call without params. +-- @function instance:is_a +-- @param some_class class to check against, or `nil` to return the class +-- @return `true` if `instance` is derived from `some_class`, or if `some_class == nil` then +-- it returns the class table of the instance +-- @usage local pussycat = Lion() -- assuming Lion derives from Cat +-- if pussycat:is_a(Cat) then +-- -- it's true, it is a Lion, but also a Cat +-- end +-- +-- if pussycat:is_a() == Lion then +-- -- It's true +-- end +local function is_a(self,klass) + if klass == nil then + -- no class provided, so return the class this instance is derived from + return getmetatable(self) + end + local m = getmetatable(self) + if not m then return false end --*can't be an object! + while m do + if m == klass then return true end + m = rawget(m,'_base') + end + return false +end + +--- checks whether an __instance__ is derived from some class. +-- Works the other way around as `is_a`. +-- @function some_class:class_of +-- @param some_instance instance to check against +-- @return `true` if `some_instance` is derived from `some_class` +-- @usage local pussycat = Lion() -- assuming Lion derives from Cat +-- if Cat:class_of(pussycat) then +-- -- it's true +-- end +local function class_of(klass,obj) + if type(klass) ~= 'table' or not rawget(klass,'is_a') then return false end + return klass.is_a(obj,klass) +end + +--- cast an object to another class. +-- It is not clever (or safe!) so use carefully. +-- @param some_instance the object to be changed +-- @function some_class:cast +local function cast (klass, obj) + return setmetatable(obj,klass) +end + + +local function _class_tostring (obj) + local mt = obj._class + local name = rawget(mt,'_name') + setmetatable(obj,nil) + local str = tostring(obj) + setmetatable(obj,mt) + if name then str = name ..str:gsub('table','') end + return str +end + +local function tupdate(td,ts,dont_override) + for k,v in pairs(ts) do + if not dont_override or td[k] == nil then + td[k] = v + end + end +end + +local function _class(base,c_arg,c) + -- the class `c` will be the metatable for all its objects, + -- and they will look up their methods in it. + local mt = {} -- a metatable for the class to support __call and _handler + -- can define class by passing it a plain table of methods + local plain = type(base) == 'table' and not getmetatable(base) + if plain then + c = base + base = c._base + else + c = c or {} + end + + if type(base) == 'table' then + -- our new class is a shallow copy of the base class! + -- but be careful not to wipe out any methods we have been given at this point! + tupdate(c,base,plain) + c._base = base + -- inherit the 'not found' handler, if present + if rawget(c,'_handler') then mt.__index = c._handler end + elseif base ~= nil then + error("must derive from a table type",3) + end + + c.__index = c + setmetatable(c,mt) + if not plain then + c._init = nil + end + + if base and rawget(base,'_class_init') then + base._class_init(c,c_arg) + end + + -- expose a ctor which can be called by () + mt.__call = function(class_tbl,...) + local obj + if rawget(c,'_create') then obj = c._create(...) end + if not obj then obj = {} end + setmetatable(obj,c) + + if rawget(c,'_init') then -- explicit constructor + local res = call_ctor(c,obj,...) + if res then -- _if_ a ctor returns a value, it becomes the object... + obj = res + setmetatable(obj,c) + end + elseif base and rawget(base,'_init') then -- default constructor + -- make sure that any stuff from the base class is initialized! + call_ctor(base,obj,...) + end + + if base and rawget(base,'_post_init') then + base._post_init(obj) + end + + return obj + end + -- Call Class.catch to set a handler for methods/properties not found in the class! + c.catch = function(self, handler) + if type(self) == "function" then + -- called using . instead of : + handler = self + end + c._handler = handler + mt.__index = handler + end + c.is_a = is_a + c.class_of = class_of + c.cast = cast + c._class = c + + if not rawget(c,'__tostring') then + c.__tostring = _class_tostring + end + + return c +end + +--- create a new class, derived from a given base class. +-- Supporting two class creation syntaxes: +-- either `Name = class(base)` or `class.Name(base)`. +-- The first form returns the class directly and does not set its `_name`. +-- The second form creates a variable `Name` in the current environment set +-- to the class, and also sets `_name`. +-- @function class +-- @param base optional base class +-- @param c_arg optional parameter to class constructor +-- @param c optional table to be used as class +local class +class = setmetatable({},{ + __call = function(fun,...) + return _class(...) + end, + __index = function(tbl,key) + if key == 'class' then + io.stderr:write('require("pl.class").class is deprecated. Use require("pl.class")\n') + return class + end + compat = compat or require 'pl.compat' + local env = compat.getfenv(2) + return function(...) + local c = _class(...) + c._name = key + rawset(env,key,c) + return c + end + end +}) + +class.properties = class() + +function class.properties._class_init(klass) + klass.__index = function(t,key) + -- normal class lookup! + local v = klass[key] + if v then return v end + -- is it a getter? + v = rawget(klass,'get_'..key) + if v then + return v(t) + end + -- is it a field? + return rawget(t,'_'..key) + end + klass.__newindex = function (t,key,value) + -- if there's a setter, use that, otherwise directly set table + local p = 'set_'..key + local setter = klass[p] + if setter then + setter(t,value) + else + rawset(t,key,value) + end + end +end + + +return class + diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/pl/compat.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/pl/compat.lua new file mode 100644 index 0000000..c9adc4c --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/pl/compat.lua @@ -0,0 +1,143 @@ +---------------- +--- Lua 5.1/5.2/5.3 compatibility. +-- Ensures that `table.pack` and `package.searchpath` are available +-- for Lua 5.1 and LuaJIT. +-- The exported function `load` is Lua 5.2 compatible. +-- `compat.setfenv` and `compat.getfenv` are available for Lua 5.2, although +-- they are not always guaranteed to work. +-- @module pl.compat + +local compat = {} + +compat.lua51 = _VERSION == 'Lua 5.1' + +local isJit = (tostring(assert):match('builtin') ~= nil) +if isJit then + -- 'goto' is a keyword when 52 compatibility is enabled in LuaJit + compat.jit52 = not loadstring("local goto = 1") +end + +--- execute a shell command. +-- This is a compatibility function that returns the same for Lua 5.1 and Lua 5.2 +-- @param cmd a shell command +-- @return true if successful +-- @return actual return code +function compat.execute (cmd) + local res1,_,res3 = os.execute(cmd) + if compat.lua51 and not compat.jit52 then + return res1==0,res1 + else + return not not res1,res3 + end +end + +---------------- +-- Load Lua code as a text or binary chunk. +-- @param ld code string or loader +-- @param[opt] source name of chunk for errors +-- @param[opt] mode 'b', 't' or 'bt' +-- @param[opt] env environment to load the chunk in +-- @function compat.load + +--------------- +-- Get environment of a function. +-- With Lua 5.2, may return nil for a function with no global references! +-- Based on code by [Sergey Rozhenko](http://lua-users.org/lists/lua-l/2010-06/msg00313.html) +-- @param f a function or a call stack reference +-- @function compat.getfenv + +--------------- +-- Set environment of a function +-- @param f a function or a call stack reference +-- @param env a table that becomes the new environment of `f` +-- @function compat.setfenv + +if compat.lua51 then -- define Lua 5.2 style load() + if not isJit then -- but LuaJIT's load _is_ compatible + local lua51_load = load + function compat.load(str,src,mode,env) + local chunk,err + if type(str) == 'string' then + if str:byte(1) == 27 and not (mode or 'bt'):find 'b' then + return nil,"attempt to load a binary chunk" + end + chunk,err = loadstring(str,src) + else + chunk,err = lua51_load(str,src) + end + if chunk and env then setfenv(chunk,env) end + return chunk,err + end + else + compat.load = load + end + compat.setfenv, compat.getfenv = setfenv, getfenv +else + compat.load = load + -- setfenv/getfenv replacements for Lua 5.2 + -- by Sergey Rozhenko + -- http://lua-users.org/lists/lua-l/2010-06/msg00313.html + -- Roberto Ierusalimschy notes that it is possible for getfenv to return nil + -- in the case of a function with no globals: + -- http://lua-users.org/lists/lua-l/2010-06/msg00315.html + function compat.setfenv(f, t) + f = (type(f) == 'function' and f or debug.getinfo(f + 1, 'f').func) + local name + local up = 0 + repeat + up = up + 1 + name = debug.getupvalue(f, up) + until name == '_ENV' or name == nil + if name then + debug.upvaluejoin(f, up, function() return name end, 1) -- use unique upvalue + debug.setupvalue(f, up, t) + end + if f ~= 0 then return f end + end + + function compat.getfenv(f) + local f = f or 0 + f = (type(f) == 'function' and f or debug.getinfo(f + 1, 'f').func) + local name, val + local up = 0 + repeat + up = up + 1 + name, val = debug.getupvalue(f, up) + until name == '_ENV' or name == nil + return val + end +end + +--- Lua 5.2 Functions Available for 5.1 +-- @section lua52 + +--- pack an argument list into a table. +-- @param ... any arguments +-- @return a table with field n set to the length +-- @return the length +-- @function table.pack +if not table.pack then + function table.pack (...) + return {n=select('#',...); ...} + end +end + +------ +-- return the full path where a Lua module name would be matched. +-- @param mod module name, possibly dotted +-- @param path a path in the same form as package.path or package.cpath +-- @see path.package_path +-- @function package.searchpath +if not package.searchpath then + local sep = package.config:sub(1,1) + function package.searchpath (mod,path) + mod = mod:gsub('%.',sep) + for m in path:gmatch('[^;]+') do + local nm = m:gsub('?',mod) + local f = io.open(nm,'r') + if f then f:close(); return nm end + end + end +end + +return compat diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/pl/stringx.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/pl/stringx.lua new file mode 100644 index 0000000..f5a27d4 --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/pl/stringx.lua @@ -0,0 +1,548 @@ +--- Python-style extended string library. +-- +-- see 3.6.1 of the Python reference. +-- If you want to make these available as string methods, then say +-- `stringx.import()` to bring them into the standard `string` table. +-- +-- See @{03-strings.md|the Guide} +-- +-- Dependencies: `pl.utils` +-- @module pl.stringx +local utils = require 'pl.utils' +local string = string +local find = string.find +local type,setmetatable,ipairs = type,setmetatable,ipairs +local error = error +local gsub = string.gsub +local rep = string.rep +local sub = string.sub +local concat = table.concat +local append = table.insert +local escape = utils.escape +local ceil, max = math.ceil, math.max +local assert_arg,usplit = utils.assert_arg,utils.split +local lstrip + +local function assert_string (n,s) + assert_arg(n,s,'string') +end + +local function non_empty(s) + return #s > 0 +end + +local function assert_nonempty_string(n,s) + assert_arg(n,s,'string',non_empty,'must be a non-empty string') +end + +local function makelist(l) + return setmetatable(l, require('pl.List')) +end + +local stringx = {} + +------------------ +-- String Predicates +-- @section predicates + +--- does s only contain alphabetic characters? +-- @string s a string +function stringx.isalpha(s) + assert_string(1,s) + return find(s,'^%a+$') == 1 +end + +--- does s only contain digits? +-- @string s a string +function stringx.isdigit(s) + assert_string(1,s) + return find(s,'^%d+$') == 1 +end + +--- does s only contain alphanumeric characters? +-- @string s a string +function stringx.isalnum(s) + assert_string(1,s) + return find(s,'^%w+$') == 1 +end + +--- does s only contain spaces? +-- @string s a string +function stringx.isspace(s) + assert_string(1,s) + return find(s,'^%s+$') == 1 +end + +--- does s only contain lower case characters? +-- @string s a string +function stringx.islower(s) + assert_string(1,s) + return find(s,'^[%l%s]+$') == 1 +end + +--- does s only contain upper case characters? +-- @string s a string +function stringx.isupper(s) + assert_string(1,s) + return find(s,'^[%u%s]+$') == 1 +end + +local function raw_startswith(s, prefix) + return find(s,prefix,1,true) == 1 +end + +local function raw_endswith(s, suffix) + return #s >= #suffix and find(s, suffix, #s-#suffix+1, true) and true or false +end + +local function test_affixes(s, affixes, fn) + if type(affixes) == 'string' then + return fn(s,affixes) + elseif type(affixes) == 'table' then + for _,affix in ipairs(affixes) do + if fn(s,affix) then return true end + end + return false + else + error(("argument #2 expected a 'string' or a 'table', got a '%s'"):format(type(affixes))) + end +end + +--- does s start with prefix or one of prefixes? +-- @string s a string +-- @param prefix a string or an array of strings +function stringx.startswith(s,prefix) + assert_string(1,s) + return test_affixes(s,prefix,raw_startswith) +end + +--- does s end with suffix or one of suffixes? +-- @string s a string +-- @param suffix a string or an array of strings +function stringx.endswith(s,suffix) + assert_string(1,s) + return test_affixes(s,suffix,raw_endswith) +end + +--- Strings and Lists +-- @section lists + +--- concatenate the strings using this string as a delimiter. +-- @string s the string +-- @param seq a table of strings or numbers +-- @usage (' '):join {1,2,3} == '1 2 3' +function stringx.join(s,seq) + assert_string(1,s) + return concat(seq,s) +end + +--- Split a string into a list of lines. +-- `"\r"`, `"\n"`, and `"\r\n"` are considered line ends. +-- They are not included in the lines unless `keepends` is passed. +-- Terminal line end does not produce an extra line. +-- Splitting an empty string results in an empty list. +-- @string s the string. +-- @bool[opt] keep_ends include line ends. +function stringx.splitlines(s, keep_ends) + assert_string(1, s) + local res = {} + local pos = 1 + while true do + local line_end_pos = find(s, '[\r\n]', pos) + if not line_end_pos then + break + end + + local line_end = sub(s, line_end_pos, line_end_pos) + if line_end == '\r' and sub(s, line_end_pos + 1, line_end_pos + 1) == '\n' then + line_end = '\r\n' + end + + local line = sub(s, pos, line_end_pos - 1) + if keep_ends then + line = line .. line_end + end + append(res, line) + + pos = line_end_pos + #line_end + end + + if pos <= #s then + append(res, sub(s, pos)) + end + return makelist(res) +end + +--- split a string into a list of strings using a delimiter. +-- @function split +-- @string s the string +-- @string[opt] re a delimiter (defaults to whitespace) +-- @int[opt] n maximum number of results +-- @usage #(('one two'):split()) == 2 +-- @usage ('one,two,three'):split(',') == List{'one','two','three'} +-- @usage ('one,two,three'):split(',',2) == List{'one','two,three'} +function stringx.split(s,re,n) + assert_string(1,s) + local plain = true + if not re then -- default spaces + s = lstrip(s) + plain = false + end + local res = usplit(s,re,plain,n) + if re and re ~= '' and find(s,re,-#re,true) then + res[#res+1] = "" + end + return makelist(res) +end + +--- replace all tabs in s with tabsize spaces. If not specified, tabsize defaults to 8. +-- with 0.9.5 this now correctly expands to the next tab stop (if you really +-- want to just replace tabs, use :gsub('\t',' ') etc) +-- @string s the string +-- @int tabsize[opt=8] number of spaces to expand each tab +function stringx.expandtabs(s,tabsize) + assert_string(1,s) + tabsize = tabsize or 8 + return (s:gsub("([^\t\r\n]*)\t", function(before_tab) + return before_tab .. (" "):rep(tabsize - #before_tab % tabsize) + end)) +end + +--- Finding and Replacing +-- @section find + +local function _find_all(s,sub,first,last) + first = first or 1 + last = last or #s + if sub == '' then return last+1,last-first+1 end + local i1,i2 = find(s,sub,first,true) + local res + local k = 0 + while i1 do + if last and i2 > last then break end + res = i1 + k = k + 1 + i1,i2 = find(s,sub,i2+1,true) + end + return res,k +end + +--- find index of first instance of sub in s from the left. +-- @string s the string +-- @string sub substring +-- @int[opt] first first index +-- @int[opt] last last index +function stringx.lfind(s,sub,first,last) + assert_string(1,s) + assert_string(2,sub) + local i1, i2 = find(s,sub,first,true) + + if i1 and (not last or i2 <= last) then + return i1 + else + return nil + end +end + +--- find index of first instance of sub in s from the right. +-- @string s the string +-- @string sub substring +-- @int[opt] first first index +-- @int[opt] last last index +function stringx.rfind(s,sub,first,last) + assert_string(1,s) + assert_string(2,sub) + return (_find_all(s,sub,first,last)) +end + +--- replace up to n instances of old by new in the string s. +-- if n is not present, replace all instances. +-- @string s the string +-- @string old the target substring +-- @string new the substitution +-- @int[opt] n optional maximum number of substitutions +-- @return result string +function stringx.replace(s,old,new,n) + assert_string(1,s) + assert_string(2,old) + assert_string(3,new) + return (gsub(s,escape(old),new:gsub('%%','%%%%'),n)) +end + +--- count all instances of substring in string. +-- @string s the string +-- @string sub substring +function stringx.count(s,sub) + assert_string(1,s) + local i,k = _find_all(s,sub,1) + return k +end + +--- Stripping and Justifying +-- @section strip + +local function _just(s,w,ch,left,right) + local n = #s + if w > n then + if not ch then ch = ' ' end + local f1,f2 + if left and right then + local rn = ceil((w-n)/2) + local ln = w - n - rn + f1 = rep(ch,ln) + f2 = rep(ch,rn) + elseif right then + f1 = rep(ch,w-n) + f2 = '' + else + f2 = rep(ch,w-n) + f1 = '' + end + return f1..s..f2 + else + return s + end +end + +--- left-justify s with width w. +-- @string s the string +-- @int w width of justification +-- @string[opt=' '] ch padding character +function stringx.ljust(s,w,ch) + assert_string(1,s) + assert_arg(2,w,'number') + return _just(s,w,ch,true,false) +end + +--- right-justify s with width w. +-- @string s the string +-- @int w width of justification +-- @string[opt=' '] ch padding character +function stringx.rjust(s,w,ch) + assert_string(1,s) + assert_arg(2,w,'number') + return _just(s,w,ch,false,true) +end + +--- center-justify s with width w. +-- @string s the string +-- @int w width of justification +-- @string[opt=' '] ch padding character +function stringx.center(s,w,ch) + assert_string(1,s) + assert_arg(2,w,'number') + return _just(s,w,ch,true,true) +end + +local function _strip(s,left,right,chrs) + if not chrs then + chrs = '%s' + else + chrs = '['..escape(chrs)..']' + end + if left then + local i1,i2 = find(s,'^'..chrs..'*') + if i2 >= i1 then + s = sub(s,i2+1) + end + end + if right then + local i1,i2 = find(s,chrs..'*$') + if i2 >= i1 then + s = sub(s,1,i1-1) + end + end + return s +end + +--- trim any whitespace on the left of s. +-- @string s the string +-- @string[opt='%s'] chrs default any whitespace character, +-- but can be a string of characters to be trimmed +function stringx.lstrip(s,chrs) + assert_string(1,s) + return _strip(s,true,false,chrs) +end +lstrip = stringx.lstrip + +--- trim any whitespace on the right of s. +-- @string s the string +-- @string[opt='%s'] chrs default any whitespace character, +-- but can be a string of characters to be trimmed +function stringx.rstrip(s,chrs) + assert_string(1,s) + return _strip(s,false,true,chrs) +end + +--- trim any whitespace on both left and right of s. +-- @string s the string +-- @string[opt='%s'] chrs default any whitespace character, +-- but can be a string of characters to be trimmed +function stringx.strip(s,chrs) + assert_string(1,s) + return _strip(s,true,true,chrs) +end + +--- Partioning Strings +-- @section partioning + +--- split a string using a pattern. Note that at least one value will be returned! +-- @string s the string +-- @string[opt='%s'] re a Lua string pattern (defaults to whitespace) +-- @return the parts of the string +-- @usage a,b = line:splitv('=') +function stringx.splitv(s,re) + assert_string(1,s) + return utils.splitv(s,re) +end + +-- The partition functions split a string using a delimiter into three parts: +-- the part before, the delimiter itself, and the part afterwards +local function _partition(p,delim,fn) + local i1,i2 = fn(p,delim) + if not i1 or i1 == -1 then + return p,'','' + else + if not i2 then i2 = i1 end + return sub(p,1,i1-1),sub(p,i1,i2),sub(p,i2+1) + end +end + +--- partition the string using first occurance of a delimiter +-- @string s the string +-- @string ch delimiter +-- @return part before ch +-- @return ch +-- @return part after ch +function stringx.partition(s,ch) + assert_string(1,s) + assert_nonempty_string(2,ch) + return _partition(s,ch,stringx.lfind) +end + +--- partition the string p using last occurance of a delimiter +-- @string s the string +-- @string ch delimiter +-- @return part before ch +-- @return ch +-- @return part after ch +function stringx.rpartition(s,ch) + assert_string(1,s) + assert_nonempty_string(2,ch) + return _partition(s,ch,stringx.rfind) +end + +--- return the 'character' at the index. +-- @string s the string +-- @int idx an index (can be negative) +-- @return a substring of length 1 if successful, empty string otherwise. +function stringx.at(s,idx) + assert_string(1,s) + assert_arg(2,idx,'number') + return sub(s,idx,idx) +end + +--- Miscelaneous +-- @section misc + +--- return an iterator over all lines in a string +-- @string s the string +-- @return an iterator +function stringx.lines(s) + assert_string(1,s) + if not s:find '\n$' then s = s..'\n' end + return s:gmatch('([^\n]*)\n') +end + +--- iniital word letters uppercase ('title case'). +-- Here 'words' mean chunks of non-space characters. +-- @string s the string +-- @return a string with each word's first letter uppercase +function stringx.title(s) + assert_string(1,s) + return (s:gsub('(%S)(%S*)',function(f,r) + return f:upper()..r:lower() + end)) +end + +stringx.capitalize = stringx.title + +local ellipsis = '...' +local n_ellipsis = #ellipsis + +--- Return a shortened version of a string. +-- Fits string within w characters. Removed characters are marked with ellipsis. +-- @string s the string +-- @int w the maxinum size allowed +-- @bool tail true if we want to show the end of the string (head otherwise) +-- @usage ('1234567890'):shorten(8) == '12345...' +-- @usage ('1234567890'):shorten(8, true) == '...67890' +-- @usage ('1234567890'):shorten(20) == '1234567890' +function stringx.shorten(s,w,tail) + assert_string(1,s) + if #s > w then + if w < n_ellipsis then return ellipsis:sub(1,w) end + if tail then + local i = #s - w + 1 + n_ellipsis + return ellipsis .. s:sub(i) + else + return s:sub(1,w-n_ellipsis) .. ellipsis + end + end + return s +end + +--- Utility function that finds any patterns that match a long string's an open or close. +-- Note that having this function use the least number of equal signs that is possible is a harder algorithm to come up with. +-- Right now, it simply returns the greatest number of them found. +-- @param s The string +-- @return 'nil' if not found. If found, the maximum number of equal signs found within all matches. +local function has_lquote(s) + local lstring_pat = '([%[%]])(=*)%1' + local equals + local start, finish, bracket, new_equals = nil, 1, nil, nil + + repeat + start, finish, bracket, new_equals = s:find(lstring_pat, finish) + if new_equals then + equals = max(equals or 0, #new_equals) + end + until not new_equals + + return equals +end + +--- Quote the given string and preserve any control or escape characters, such that reloading the string in Lua returns the same result. +-- @param s The string to be quoted. +-- @return The quoted string. +function stringx.quote_string(s) + assert_string(1,s) + -- Find out if there are any embedded long-quote sequences that may cause issues. + -- This is important when strings are embedded within strings, like when serializing. + -- Append a closing bracket to catch unfinished long-quote sequences at the end of the string. + local equal_signs = has_lquote(s .. "]") + + -- Note that strings containing "\r" can't be quoted using long brackets + -- as Lua lexer converts all newlines to "\n" within long strings. + if (s:find("\n") or equal_signs) and not s:find("\r") then + -- If there is an embedded sequence that matches a long quote, then + -- find the one with the maximum number of = signs and add one to that number. + equal_signs = ("="):rep((equal_signs or -1) + 1) + -- Long strings strip out leading newline. We want to retain that, when quoting. + if s:find("^\n") then s = "\n" .. s end + local lbracket, rbracket = + "[" .. equal_signs .. "[", + "]" .. equal_signs .. "]" + s = lbracket .. s .. rbracket + else + -- Escape funny stuff. Lua 5.1 does not handle "\r" correctly. + s = ("%q"):format(s):gsub("\r", "\\r") + end + return s +end + +function stringx.import() + utils.import(stringx,string) +end + +return stringx diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/pl/tablex.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/pl/tablex.lua new file mode 100644 index 0000000..03ecba9 --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/pl/tablex.lua @@ -0,0 +1,927 @@ +--- Extended operations on Lua tables. +-- +-- See @{02-arrays.md.Useful_Operations_on_Tables|the Guide} +-- +-- Dependencies: `pl.utils`, `pl.types` +-- @module pl.tablex +local utils = require ('pl.utils') +local types = require ('pl.types') +local getmetatable,setmetatable,require = getmetatable,setmetatable,require +local tsort,append,remove = table.sort,table.insert,table.remove +local min = math.min +local pairs,type,unpack,select,tostring = pairs,type,utils.unpack,select,tostring +local function_arg = utils.function_arg +local assert_arg = utils.assert_arg + +local tablex = {} + +-- generally, functions that make copies of tables try to preserve the metatable. +-- However, when the source has no obvious type, then we attach appropriate metatables +-- like List, Map, etc to the result. +local function setmeta (res,tbl,pl_class) + local mt = getmetatable(tbl) or pl_class and require('pl.' .. pl_class) + return mt and setmetatable(res, mt) or res +end + +local function makelist(l) + return setmetatable(l, require('pl.List')) +end + +local function makemap(m) + return setmetatable(m, require('pl.Map')) +end + +local function complain (idx,msg) + error(('argument %d is not %s'):format(idx,msg),3) +end + +local function assert_arg_indexable (idx,val) + if not types.is_indexable(val) then + complain(idx,"indexable") + end +end + +local function assert_arg_iterable (idx,val) + if not types.is_iterable(val) then + complain(idx,"iterable") + end +end + +local function assert_arg_writeable (idx,val) + if not types.is_writeable(val) then + complain(idx,"writeable") + end +end + +--- copy a table into another, in-place. +-- @within Copying +-- @tab t1 destination table +-- @tab t2 source (actually any iterable object) +-- @return first table +function tablex.update (t1,t2) + assert_arg_writeable(1,t1) + assert_arg_iterable(2,t2) + for k,v in pairs(t2) do + t1[k] = v + end + return t1 +end + +--- total number of elements in this table. +-- Note that this is distinct from `#t`, which is the number +-- of values in the array part; this value will always +-- be greater or equal. The difference gives the size of +-- the hash part, for practical purposes. Works for any +-- object with a __pairs metamethod. +-- @tab t a table +-- @return the size +function tablex.size (t) + assert_arg_iterable(1,t) + local i = 0 + for k in pairs(t) do i = i + 1 end + return i +end + +--- make a shallow copy of a table +-- @within Copying +-- @tab t an iterable source +-- @return new table +function tablex.copy (t) + assert_arg_iterable(1,t) + local res = {} + for k,v in pairs(t) do + res[k] = v + end + return res +end + +--- make a deep copy of a table, recursively copying all the keys and fields. +-- This will also set the copied table's metatable to that of the original. +-- @within Copying +-- @tab t A table +-- @return new table +function tablex.deepcopy(t) + if type(t) ~= 'table' then return t end + assert_arg_iterable(1,t) + local mt = getmetatable(t) + local res = {} + for k,v in pairs(t) do + if type(v) == 'table' then + v = tablex.deepcopy(v) + end + res[k] = v + end + setmetatable(res,mt) + return res +end + +local abs, deepcompare = math.abs + +--- compare two values. +-- if they are tables, then compare their keys and fields recursively. +-- @within Comparing +-- @param t1 A value +-- @param t2 A value +-- @bool[opt] ignore_mt if true, ignore __eq metamethod (default false) +-- @number[opt] eps if defined, then used for any number comparisons +-- @return true or false +function tablex.deepcompare(t1,t2,ignore_mt,eps) + local ty1 = type(t1) + local ty2 = type(t2) + if ty1 ~= ty2 then return false end + -- non-table types can be directly compared + if ty1 ~= 'table' then + if ty1 == 'number' and eps then return abs(t1-t2) < eps end + return t1 == t2 + end + -- as well as tables which have the metamethod __eq + local mt = getmetatable(t1) + if not ignore_mt and mt and mt.__eq then return t1 == t2 end + for k1 in pairs(t1) do + if t2[k1]==nil then return false end + end + for k2 in pairs(t2) do + if t1[k2]==nil then return false end + end + for k1,v1 in pairs(t1) do + local v2 = t2[k1] + if not deepcompare(v1,v2,ignore_mt,eps) then return false end + end + + return true +end + +deepcompare = tablex.deepcompare + +--- compare two arrays using a predicate. +-- @within Comparing +-- @array t1 an array +-- @array t2 an array +-- @func cmp A comparison function +function tablex.compare (t1,t2,cmp) + assert_arg_indexable(1,t1) + assert_arg_indexable(2,t2) + if #t1 ~= #t2 then return false end + cmp = function_arg(3,cmp) + for k = 1,#t1 do + if not cmp(t1[k],t2[k]) then return false end + end + return true +end + +--- compare two list-like tables using an optional predicate, without regard for element order. +-- @within Comparing +-- @array t1 a list-like table +-- @array t2 a list-like table +-- @param cmp A comparison function (may be nil) +function tablex.compare_no_order (t1,t2,cmp) + assert_arg_indexable(1,t1) + assert_arg_indexable(2,t2) + if cmp then cmp = function_arg(3,cmp) end + if #t1 ~= #t2 then return false end + local visited = {} + for i = 1,#t1 do + local val = t1[i] + local gotcha + for j = 1,#t2 do if not visited[j] then + local match + if cmp then match = cmp(val,t2[j]) else match = val == t2[j] end + if match then + gotcha = j + break + end + end end + if not gotcha then return false end + visited[gotcha] = true + end + return true +end + + +--- return the index of a value in a list. +-- Like string.find, there is an optional index to start searching, +-- which can be negative. +-- @within Finding +-- @array t A list-like table +-- @param val A value +-- @int idx index to start; -1 means last element,etc (default 1) +-- @return index of value or nil if not found +-- @usage find({10,20,30},20) == 2 +-- @usage find({'a','b','a','c'},'a',2) == 3 +function tablex.find(t,val,idx) + assert_arg_indexable(1,t) + idx = idx or 1 + if idx < 0 then idx = #t + idx + 1 end + for i = idx,#t do + if t[i] == val then return i end + end + return nil +end + +--- return the index of a value in a list, searching from the end. +-- Like string.find, there is an optional index to start searching, +-- which can be negative. +-- @within Finding +-- @array t A list-like table +-- @param val A value +-- @param idx index to start; -1 means last element,etc (default 1) +-- @return index of value or nil if not found +-- @usage rfind({10,10,10},10) == 3 +function tablex.rfind(t,val,idx) + assert_arg_indexable(1,t) + idx = idx or #t + if idx < 0 then idx = #t + idx + 1 end + for i = idx,1,-1 do + if t[i] == val then return i end + end + return nil +end + + +--- return the index (or key) of a value in a table using a comparison function. +-- @within Finding +-- @tab t A table +-- @func cmp A comparison function +-- @param arg an optional second argument to the function +-- @return index of value, or nil if not found +-- @return value returned by comparison function +function tablex.find_if(t,cmp,arg) + assert_arg_iterable(1,t) + cmp = function_arg(2,cmp) + for k,v in pairs(t) do + local c = cmp(v,arg) + if c then return k,c end + end + return nil +end + +--- return a list of all values in a table indexed by another list. +-- @tab tbl a table +-- @array idx an index table (a list of keys) +-- @return a list-like table +-- @usage index_by({10,20,30,40},{2,4}) == {20,40} +-- @usage index_by({one=1,two=2,three=3},{'one','three'}) == {1,3} +function tablex.index_by(tbl,idx) + assert_arg_indexable(1,tbl) + assert_arg_indexable(2,idx) + local res = {} + for i = 1,#idx do + res[i] = tbl[idx[i]] + end + return setmeta(res,tbl,'List') +end + +--- apply a function to all values of a table. +-- This returns a table of the results. +-- Any extra arguments are passed to the function. +-- @within MappingAndFiltering +-- @func fun A function that takes at least one argument +-- @tab t A table +-- @param ... optional arguments +-- @usage map(function(v) return v*v end, {10,20,30,fred=2}) is {100,400,900,fred=4} +function tablex.map(fun,t,...) + assert_arg_iterable(1,t) + fun = function_arg(1,fun) + local res = {} + for k,v in pairs(t) do + res[k] = fun(v,...) + end + return setmeta(res,t) +end + +--- apply a function to all values of a list. +-- This returns a table of the results. +-- Any extra arguments are passed to the function. +-- @within MappingAndFiltering +-- @func fun A function that takes at least one argument +-- @array t a table (applies to array part) +-- @param ... optional arguments +-- @return a list-like table +-- @usage imap(function(v) return v*v end, {10,20,30,fred=2}) is {100,400,900} +function tablex.imap(fun,t,...) + assert_arg_indexable(1,t) + fun = function_arg(1,fun) + local res = {} + for i = 1,#t do + res[i] = fun(t[i],...) or false + end + return setmeta(res,t,'List') +end + +--- apply a named method to values from a table. +-- @within MappingAndFiltering +-- @string name the method name +-- @array t a list-like table +-- @param ... any extra arguments to the method +function tablex.map_named_method (name,t,...) + utils.assert_string(1,name) + assert_arg_indexable(2,t) + local res = {} + for i = 1,#t do + local val = t[i] + local fun = val[name] + res[i] = fun(val,...) + end + return setmeta(res,t,'List') +end + +--- apply a function to all values of a table, in-place. +-- Any extra arguments are passed to the function. +-- @func fun A function that takes at least one argument +-- @tab t a table +-- @param ... extra arguments +function tablex.transform (fun,t,...) + assert_arg_iterable(1,t) + fun = function_arg(1,fun) + for k,v in pairs(t) do + t[k] = fun(v,...) + end +end + +--- generate a table of all numbers in a range. +-- This is consistent with a numerical for loop. +-- @int start number +-- @int finish number +-- @int[opt=1] step make this negative for start < finish +function tablex.range (start,finish,step) + local res + step = step or 1 + if start == finish then + res = {start} + elseif (start > finish and step > 0) or (finish > start and step < 0) then + res = {} + else + local k = 1 + res = {} + for i=start,finish,step do res[k]=i; k=k+1 end + end + return makelist(res) +end + +--- apply a function to values from two tables. +-- @within MappingAndFiltering +-- @func fun a function of at least two arguments +-- @tab t1 a table +-- @tab t2 a table +-- @param ... extra arguments +-- @return a table +-- @usage map2('+',{1,2,3,m=4},{10,20,30,m=40}) is {11,22,23,m=44} +function tablex.map2 (fun,t1,t2,...) + assert_arg_iterable(1,t1) + assert_arg_iterable(2,t2) + fun = function_arg(1,fun) + local res = {} + for k,v in pairs(t1) do + res[k] = fun(v,t2[k],...) + end + return setmeta(res,t1,'List') +end + +--- apply a function to values from two arrays. +-- The result will be the length of the shortest array. +-- @within MappingAndFiltering +-- @func fun a function of at least two arguments +-- @array t1 a list-like table +-- @array t2 a list-like table +-- @param ... extra arguments +-- @usage imap2('+',{1,2,3,m=4},{10,20,30,m=40}) is {11,22,23} +function tablex.imap2 (fun,t1,t2,...) + assert_arg_indexable(2,t1) + assert_arg_indexable(3,t2) + fun = function_arg(1,fun) + local res,n = {},math.min(#t1,#t2) + for i = 1,n do + res[i] = fun(t1[i],t2[i],...) + end + return res +end + +--- 'reduce' a list using a binary function. +-- @func fun a function of two arguments +-- @array t a list-like table +-- @array memo optional initial memo value. Defaults to first value in table. +-- @return the result of the function +-- @usage reduce('+',{1,2,3,4}) == 10 +function tablex.reduce (fun,t,memo) + assert_arg_indexable(2,t) + fun = function_arg(1,fun) + local n = #t + if n == 0 then + return memo + end + local res = memo and fun(memo, t[1]) or t[1] + for i = 2,n do + res = fun(res,t[i]) + end + return res +end + +--- apply a function to all elements of a table. +-- The arguments to the function will be the value, +-- the key and _finally_ any extra arguments passed to this function. +-- Note that the Lua 5.0 function table.foreach passed the _key_ first. +-- @within Iterating +-- @tab t a table +-- @func fun a function with at least one argument +-- @param ... extra arguments +function tablex.foreach(t,fun,...) + assert_arg_iterable(1,t) + fun = function_arg(2,fun) + for k,v in pairs(t) do + fun(v,k,...) + end +end + +--- apply a function to all elements of a list-like table in order. +-- The arguments to the function will be the value, +-- the index and _finally_ any extra arguments passed to this function +-- @within Iterating +-- @array t a table +-- @func fun a function with at least one argument +-- @param ... optional arguments +function tablex.foreachi(t,fun,...) + assert_arg_indexable(1,t) + fun = function_arg(2,fun) + for i = 1,#t do + fun(t[i],i,...) + end +end + +--- Apply a function to a number of tables. +-- A more general version of map +-- The result is a table containing the result of applying that function to the +-- ith value of each table. Length of output list is the minimum length of all the lists +-- @within MappingAndFiltering +-- @func fun a function of n arguments +-- @tab ... n tables +-- @usage mapn(function(x,y,z) return x+y+z end, {1,2,3},{10,20,30},{100,200,300}) is {111,222,333} +-- @usage mapn(math.max, {1,20,300},{10,2,3},{100,200,100}) is {100,200,300} +-- @param fun A function that takes as many arguments as there are tables +function tablex.mapn(fun,...) + fun = function_arg(1,fun) + local res = {} + local lists = {...} + local minn = 1e40 + for i = 1,#lists do + minn = min(minn,#(lists[i])) + end + for i = 1,minn do + local args,k = {},1 + for j = 1,#lists do + args[k] = lists[j][i] + k = k + 1 + end + res[#res+1] = fun(unpack(args)) + end + return res +end + +--- call the function with the key and value pairs from a table. +-- The function can return a value and a key (note the order!). If both +-- are not nil, then this pair is inserted into the result: if the key already exists, we convert the value for that +-- key into a table and append into it. If only value is not nil, then it is appended to the result. +-- @within MappingAndFiltering +-- @func fun A function which will be passed each key and value as arguments, plus any extra arguments to pairmap. +-- @tab t A table +-- @param ... optional arguments +-- @usage pairmap(function(k,v) return v end,{fred=10,bonzo=20}) is {10,20} _or_ {20,10} +-- @usage pairmap(function(k,v) return {k,v},k end,{one=1,two=2}) is {one={'one',1},two={'two',2}} +function tablex.pairmap(fun,t,...) + assert_arg_iterable(1,t) + fun = function_arg(1,fun) + local res = {} + for k,v in pairs(t) do + local rv,rk = fun(k,v,...) + if rk then + if res[rk] then + if type(res[rk]) == 'table' then + table.insert(res[rk],rv) + else + res[rk] = {res[rk], rv} + end + else + res[rk] = rv + end + else + res[#res+1] = rv + end + end + return res +end + +local function keys_op(i,v) return i end + +--- return all the keys of a table in arbitrary order. +-- @within Extraction +-- @tab t A table +function tablex.keys(t) + assert_arg_iterable(1,t) + return makelist(tablex.pairmap(keys_op,t)) +end + +local function values_op(i,v) return v end + +--- return all the values of the table in arbitrary order +-- @within Extraction +-- @tab t A table +function tablex.values(t) + assert_arg_iterable(1,t) + return makelist(tablex.pairmap(values_op,t)) +end + +local function index_map_op (i,v) return i,v end + +--- create an index map from a list-like table. The original values become keys, +-- and the associated values are the indices into the original list. +-- @array t a list-like table +-- @return a map-like table +function tablex.index_map (t) + assert_arg_indexable(1,t) + return makemap(tablex.pairmap(index_map_op,t)) +end + +local function set_op(i,v) return true,v end + +--- create a set from a list-like table. A set is a table where the original values +-- become keys, and the associated values are all true. +-- @array t a list-like table +-- @return a set (a map-like table) +function tablex.makeset (t) + assert_arg_indexable(1,t) + return setmetatable(tablex.pairmap(set_op,t),require('pl.Set')) +end + +--- combine two tables, either as union or intersection. Corresponds to +-- set operations for sets () but more general. Not particularly +-- useful for list-like tables. +-- @within Merging +-- @tab t1 a table +-- @tab t2 a table +-- @bool dup true for a union, false for an intersection. +-- @usage merge({alice=23,fred=34},{bob=25,fred=34}) is {fred=34} +-- @usage merge({alice=23,fred=34},{bob=25,fred=34},true) is {bob=25,fred=34,alice=23} +-- @see tablex.index_map +function tablex.merge (t1,t2,dup) + assert_arg_iterable(1,t1) + assert_arg_iterable(2,t2) + local res = {} + for k,v in pairs(t1) do + if dup or t2[k] then res[k] = v end + end + if dup then + for k,v in pairs(t2) do + res[k] = v + end + end + return setmeta(res,t1,'Map') +end + +--- the union of two map-like tables. +-- If there are duplicate keys, the second table wins. +-- @tab t1 a table +-- @tab t2 a table +-- @treturn tab +-- @see tablex.merge +function tablex.union(t1, t2) + return tablex.merge(t1, t2, true) +end + +--- the intersection of two map-like tables. +-- @tab t1 a table +-- @tab t2 a table +-- @treturn tab +-- @see tablex.merge +function tablex.intersection(t1, t2) + return tablex.merge(t1, t2, false) +end + +--- a new table which is the difference of two tables. +-- With sets (where the values are all true) this is set difference and +-- symmetric difference depending on the third parameter. +-- @within Merging +-- @tab s1 a map-like table or set +-- @tab s2 a map-like table or set +-- @bool symm symmetric difference (default false) +-- @return a map-like table or set +function tablex.difference (s1,s2,symm) + assert_arg_iterable(1,s1) + assert_arg_iterable(2,s2) + local res = {} + for k,v in pairs(s1) do + if s2[k] == nil then res[k] = v end + end + if symm then + for k,v in pairs(s2) do + if s1[k] == nil then res[k] = v end + end + end + return setmeta(res,s1,'Map') +end + +--- A table where the key/values are the values and value counts of the table. +-- @array t a list-like table +-- @func cmp a function that defines equality (otherwise uses ==) +-- @return a map-like table +-- @see seq.count_map +function tablex.count_map (t,cmp) + assert_arg_indexable(1,t) + local res,mask = {},{} + cmp = function_arg(2,cmp or '==') + local n = #t + for i = 1,#t do + local v = t[i] + if not mask[v] then + mask[v] = true + -- check this value against all other values + res[v] = 1 -- there's at least one instance + for j = i+1,n do + local w = t[j] + local ok = cmp(v,w) + if ok then + res[v] = res[v] + 1 + mask[w] = true + end + end + end + end + return makemap(res) +end + +--- filter an array's values using a predicate function +-- @within MappingAndFiltering +-- @array t a list-like table +-- @func pred a boolean function +-- @param arg optional argument to be passed as second argument of the predicate +function tablex.filter (t,pred,arg) + assert_arg_indexable(1,t) + pred = function_arg(2,pred) + local res,k = {},1 + for i = 1,#t do + local v = t[i] + if pred(v,arg) then + res[k] = v + k = k + 1 + end + end + return setmeta(res,t,'List') +end + +--- return a table where each element is a table of the ith values of an arbitrary +-- number of tables. It is equivalent to a matrix transpose. +-- @within Merging +-- @usage zip({10,20,30},{100,200,300}) is {{10,100},{20,200},{30,300}} +-- @array ... arrays to be zipped +function tablex.zip(...) + return tablex.mapn(function(...) return {...} end,...) +end + +local _copy +function _copy (dest,src,idest,isrc,nsrc,clean_tail) + idest = idest or 1 + isrc = isrc or 1 + local iend + if not nsrc then + nsrc = #src + iend = #src + else + iend = isrc + min(nsrc-1,#src-isrc) + end + if dest == src then -- special case + if idest > isrc and iend >= idest then -- overlapping ranges + src = tablex.sub(src,isrc,nsrc) + isrc = 1; iend = #src + end + end + for i = isrc,iend do + dest[idest] = src[i] + idest = idest + 1 + end + if clean_tail then + tablex.clear(dest,idest) + end + return dest +end + +--- copy an array into another one, clearing `dest` after `idest+nsrc`, if necessary. +-- @within Copying +-- @array dest a list-like table +-- @array src a list-like table +-- @int[opt=1] idest where to start copying values into destination +-- @int[opt=1] isrc where to start copying values from source +-- @int[opt=#src] nsrc number of elements to copy from source +function tablex.icopy (dest,src,idest,isrc,nsrc) + assert_arg_indexable(1,dest) + assert_arg_indexable(2,src) + return _copy(dest,src,idest,isrc,nsrc,true) +end + +--- copy an array into another one. +-- @within Copying +-- @array dest a list-like table +-- @array src a list-like table +-- @int[opt=1] idest where to start copying values into destination +-- @int[opt=1] isrc where to start copying values from source +-- @int[opt=#src] nsrc number of elements to copy from source +function tablex.move (dest,src,idest,isrc,nsrc) + assert_arg_indexable(1,dest) + assert_arg_indexable(2,src) + return _copy(dest,src,idest,isrc,nsrc,false) +end + +function tablex._normalize_slice(self,first,last) + local sz = #self + if not first then first=1 end + if first<0 then first=sz+first+1 end + -- make the range _inclusive_! + if not last then last=sz end + if last < 0 then last=sz+1+last end + return first,last +end + +--- Extract a range from a table, like 'string.sub'. +-- If first or last are negative then they are relative to the end of the list +-- eg. sub(t,-2) gives last 2 entries in a list, and +-- sub(t,-4,-2) gives from -4th to -2nd +-- @within Extraction +-- @array t a list-like table +-- @int first An index +-- @int last An index +-- @return a new List +function tablex.sub(t,first,last) + assert_arg_indexable(1,t) + first,last = tablex._normalize_slice(t,first,last) + local res={} + for i=first,last do append(res,t[i]) end + return setmeta(res,t,'List') +end + +--- set an array range to a value. If it's a function we use the result +-- of applying it to the indices. +-- @array t a list-like table +-- @param val a value +-- @int[opt=1] i1 start range +-- @int[opt=#t] i2 end range +function tablex.set (t,val,i1,i2) + assert_arg_indexable(1,t) + i1,i2 = i1 or 1,i2 or #t + if types.is_callable(val) then + for i = i1,i2 do + t[i] = val(i) + end + else + for i = i1,i2 do + t[i] = val + end + end +end + +--- create a new array of specified size with initial value. +-- @int n size +-- @param val initial value (can be `nil`, but don't expect `#` to work!) +-- @return the table +function tablex.new (n,val) + local res = {} + tablex.set(res,val,1,n) + return res +end + +--- clear out the contents of a table. +-- @array t a list +-- @param istart optional start position +function tablex.clear(t,istart) + istart = istart or 1 + for i = istart,#t do remove(t) end +end + +--- insert values into a table. +-- similar to `table.insert` but inserts values from given table `values`, +-- not the object itself, into table `t` at position `pos`. +-- @within Copying +-- @array t the list +-- @int[opt] position (default is at end) +-- @array values +function tablex.insertvalues(t, ...) + assert_arg(1,t,'table') + local pos, values + if select('#', ...) == 1 then + pos,values = #t+1, ... + else + pos,values = ... + end + if #values > 0 then + for i=#t,pos,-1 do + t[i+#values] = t[i] + end + local offset = 1 - pos + for i=pos,pos+#values-1 do + t[i] = values[i + offset] + end + end + return t +end + +--- remove a range of values from a table. +-- End of range may be negative. +-- @array t a list-like table +-- @int i1 start index +-- @int i2 end index +-- @return the table +function tablex.removevalues (t,i1,i2) + assert_arg(1,t,'table') + i1,i2 = tablex._normalize_slice(t,i1,i2) + for i = i1,i2 do + remove(t,i1) + end + return t +end + +local _find +_find = function (t,value,tables) + for k,v in pairs(t) do + if v == value then return k end + end + for k,v in pairs(t) do + if not tables[v] and type(v) == 'table' then + tables[v] = true + local res = _find(v,value,tables) + if res then + res = tostring(res) + if type(k) ~= 'string' then + return '['..k..']'..res + else + return k..'.'..res + end + end + end + end +end + +--- find a value in a table by recursive search. +-- @within Finding +-- @tab t the table +-- @param value the value +-- @array[opt] exclude any tables to avoid searching +-- @usage search(_G,math.sin,{package.path}) == 'math.sin' +-- @return a fieldspec, e.g. 'a.b' or 'math.sin' +function tablex.search (t,value,exclude) + assert_arg_iterable(1,t) + local tables = {[t]=true} + if exclude then + for _,v in pairs(exclude) do tables[v] = true end + end + return _find(t,value,tables) +end + +--- return an iterator to a table sorted by its keys +-- @within Iterating +-- @tab t the table +-- @func f an optional comparison function (f(x,y) is true if x < y) +-- @usage for k,v in tablex.sort(t) do print(k,v) end +-- @return an iterator to traverse elements sorted by the keys +function tablex.sort(t,f) + local keys = {} + for k in pairs(t) do keys[#keys + 1] = k end + tsort(keys,f) + local i = 0 + return function() + i = i + 1 + return keys[i], t[keys[i]] + end +end + +--- return an iterator to a table sorted by its values +-- @within Iterating +-- @tab t the table +-- @func f an optional comparison function (f(x,y) is true if x < y) +-- @usage for k,v in tablex.sortv(t) do print(k,v) end +-- @return an iterator to traverse elements sorted by the values +function tablex.sortv(t,f) + f = function_arg(2, f or '<') + local keys = {} + for k in pairs(t) do keys[#keys + 1] = k end + tsort(keys,function(x, y) return f(t[x], t[y]) end) + local i = 0 + return function() + i = i + 1 + return keys[i], t[keys[i]] + end +end + +--- modifies a table to be read only. +-- This only offers weak protection. Tables can still be modified with +-- `table.insert` and `rawset`. +-- @tab t the table +-- @return the table read only. +function tablex.readonly(t) + local mt = { + __index=t, + __newindex=function(t, k, v) error("Attempt to modify read-only table", 2) end, + __pairs=function() return pairs(t) end, + __ipairs=function() return ipairs(t) end, + __len=function() return #t end, + __metatable=false + } + return setmetatable({}, mt) +end + +return tablex diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/pl/types.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/pl/types.lua new file mode 100644 index 0000000..f4ab236 --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/pl/types.lua @@ -0,0 +1,145 @@ +---- Dealing with Detailed Type Information + +-- Dependencies `pl.utils` +-- @module pl.types + +local utils = require 'pl.utils' +local types = {} + +--- is the object either a function or a callable object?. +-- @param obj Object to check. +function types.is_callable (obj) + return type(obj) == 'function' or getmetatable(obj) and getmetatable(obj).__call and true +end + +--- is the object of the specified type?. +-- If the type is a string, then use type, otherwise compare with metatable +-- @param obj An object to check +-- @param tp String of what type it should be +-- @function is_type +types.is_type = utils.is_type + +local fileMT = getmetatable(io.stdout) + +--- a string representation of a type. +-- For tables with metatables, we assume that the metatable has a `_name` +-- field. Knows about Lua file objects. +-- @param obj an object +-- @return a string like 'number', 'table' or 'List' +function types.type (obj) + local t = type(obj) + if t == 'table' or t == 'userdata' then + local mt = getmetatable(obj) + if mt == fileMT then + return 'file' + elseif mt == nil then + return t + else + return mt._name or "unknown "..t + end + else + return t + end +end + +--- is this number an integer? +-- @param x a number +-- @raise error if x is not a number +function types.is_integer (x) + return math.ceil(x)==x +end + +--- Check if the object is "empty". +-- An object is considered empty if it is nil, a table with out any items (key, +-- value pairs or indexes), or a string with no content (""). +-- @param o The object to check if it is empty. +-- @param ignore_spaces If the object is a string and this is true the string is +-- considered empty is it only contains spaces. +-- @return true if the object is empty, otherwise false. +function types.is_empty(o, ignore_spaces) + if o == nil or (type(o) == "table" and not next(o)) or (type(o) == "string" and (o == "" or (ignore_spaces and o:match("^%s+$")))) then + return true + end + return false +end + +local function check_meta (val) + if type(val) == 'table' then return true end + return getmetatable(val) +end + +--- is an object 'array-like'? +-- @param val any value. +function types.is_indexable (val) + local mt = check_meta(val) + if mt == true then return true end + return mt and mt.__len and mt.__index and true +end + +--- can an object be iterated over with `pairs`? +-- @param val any value. +function types.is_iterable (val) + local mt = check_meta(val) + if mt == true then return true end + return mt and mt.__pairs and true +end + +--- can an object accept new key/pair values? +-- @param val any value. +function types.is_writeable (val) + local mt = check_meta(val) + if mt == true then return true end + return mt and mt.__newindex and true +end + +-- Strings that should evaluate to true. +local trues = { yes=true, y=true, ["true"]=true, t=true, ["1"]=true } +-- Conditions types should evaluate to true. +local true_types = { + boolean=function(o, true_strs, check_objs) return o end, + string=function(o, true_strs, check_objs) + if trues[o:lower()] then + return true + end + -- Check alternative user provided strings. + for _,v in ipairs(true_strs or {}) do + if type(v) == "string" and o == v:lower() then + return true + end + end + return false + end, + number=function(o, true_strs, check_objs) return o ~= 0 end, + table=function(o, true_strs, check_objs) if check_objs and next(o) ~= nil then return true end return false end +} +--- Convert to a boolean value. +-- True values are: +-- +-- * boolean: true. +-- * string: 'yes', 'y', 'true', 't', '1' or additional strings specified by `true_strs`. +-- * number: Any non-zero value. +-- * table: Is not empty and `check_objs` is true. +-- * object: Is not `nil` and `check_objs` is true. +-- +-- @param o The object to evaluate. +-- @param[opt] true_strs optional Additional strings that when matched should evaluate to true. Comparison is case insensitive. +-- This should be a List of strings. E.g. "ja" to support German. +-- @param[opt] check_objs True if objects should be evaluated. Default is to evaluate objects as true if not nil +-- or if it is a table and it is not empty. +-- @return true if the input evaluates to true, otherwise false. +function types.to_bool(o, true_strs, check_objs) + local true_func + if true_strs then + utils.assert_arg(2, true_strs, "table") + end + true_func = true_types[type(o)] + if true_func then + return true_func(o, true_strs, check_objs) + elseif check_objs and o ~= nil then + return true + end + return false +end + + +return types diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/pl/utils.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/pl/utils.lua new file mode 100644 index 0000000..d3d6e11 --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/pl/utils.lua @@ -0,0 +1,516 @@ +--- Generally useful routines. +-- See @{01-introduction.md.Generally_useful_functions|the Guide}. +-- +-- Dependencies: `pl.compat` +-- +-- @module pl.utils +local format = string.format +local compat = require 'pl.compat' +local stdout = io.stdout +local append = table.insert +local unpack = rawget(_G,'unpack') or rawget(table,'unpack') + +local utils = { + _VERSION = "1.4.1", + lua51 = compat.lua51, + setfenv = compat.setfenv, + getfenv = compat.getfenv, + load = compat.load, + execute = compat.execute, + dir_separator = _G.package.config:sub(1,1), + unpack = unpack +} + +--- end this program gracefully. +-- @param code The exit code or a message to be printed +-- @param ... extra arguments for message's format' +-- @see utils.fprintf +function utils.quit(code,...) + if type(code) == 'string' then + utils.fprintf(io.stderr,code,...) + code = -1 + else + utils.fprintf(io.stderr,...) + end + io.stderr:write('\n') + os.exit(code) +end + +--- print an arbitrary number of arguments using a format. +-- @param fmt The format (see string.format) +-- @param ... Extra arguments for format +function utils.printf(fmt,...) + utils.assert_string(1,fmt) + utils.fprintf(stdout,fmt,...) +end + +--- write an arbitrary number of arguments to a file using a format. +-- @param f File handle to write to. +-- @param fmt The format (see string.format). +-- @param ... Extra arguments for format +function utils.fprintf(f,fmt,...) + utils.assert_string(2,fmt) + f:write(format(fmt,...)) +end + +local function import_symbol(T,k,v,libname) + local key = rawget(T,k) + -- warn about collisions! + if key and k ~= '_M' and k ~= '_NAME' and k ~= '_PACKAGE' and k ~= '_VERSION' then + utils.fprintf(io.stderr,"warning: '%s.%s' will not override existing symbol\n",libname,k) + return + end + rawset(T,k,v) +end + +local function lookup_lib(T,t) + for k,v in pairs(T) do + if v == t then return k end + end + return '?' +end + +local already_imported = {} + +--- take a table and 'inject' it into the local namespace. +-- @param t The Table +-- @param T An optional destination table (defaults to callers environment) +function utils.import(t,T) + T = T or _G + t = t or utils + if type(t) == 'string' then + t = require (t) + end + local libname = lookup_lib(T,t) + if already_imported[t] then return end + already_imported[t] = libname + for k,v in pairs(t) do + import_symbol(T,k,v,libname) + end +end + +utils.patterns = { + FLOAT = '[%+%-%d]%d*%.?%d*[eE]?[%+%-]?%d*', + INTEGER = '[+%-%d]%d*', + IDEN = '[%a_][%w_]*', + FILE = '[%a%.\\][:%][%w%._%-\\]*' +} + +--- escape any 'magic' characters in a string +-- @param s The input string +function utils.escape(s) + utils.assert_string(1,s) + return (s:gsub('[%-%.%+%[%]%(%)%$%^%%%?%*]','%%%1')) +end + +--- return either of two values, depending on a condition. +-- @param cond A condition +-- @param value1 Value returned if cond is true +-- @param value2 Value returned if cond is false (can be optional) +function utils.choose(cond,value1,value2) + if cond then return value1 + else return value2 + end +end + +local raise + +--- return the contents of a file as a string +-- @param filename The file path +-- @param is_bin open in binary mode +-- @return file contents +function utils.readfile(filename,is_bin) + local mode = is_bin and 'b' or '' + utils.assert_string(1,filename) + local f,open_err = io.open(filename,'r'..mode) + if not f then return utils.raise (open_err) end + local res,read_err = f:read('*a') + f:close() + if not res then + -- Errors in io.open have "filename: " prefix, + -- error in file:read don't, add it. + return raise (filename..": "..read_err) + end + return res +end + +--- write a string to a file +-- @param filename The file path +-- @param str The string +-- @param is_bin open in binary mode +-- @return true or nil +-- @return error message +-- @raise error if filename or str aren't strings +function utils.writefile(filename,str,is_bin) + local mode = is_bin and 'b' or '' + utils.assert_string(1,filename) + utils.assert_string(2,str) + local f,err = io.open(filename,'w'..mode) + if not f then return raise(err) end + f:write(str) + f:close() + return true +end + +--- return the contents of a file as a list of lines +-- @param filename The file path +-- @return file contents as a table +-- @raise errror if filename is not a string +function utils.readlines(filename) + utils.assert_string(1,filename) + local f,err = io.open(filename,'r') + if not f then return raise(err) end + local res = {} + for line in f:lines() do + append(res,line) + end + f:close() + return res +end + +--- split a string into a list of strings separated by a delimiter. +-- @param s The input string +-- @param re A Lua string pattern; defaults to '%s+' +-- @param plain don't use Lua patterns +-- @param n optional maximum number of splits +-- @return a list-like table +-- @raise error if s is not a string +function utils.split(s,re,plain,n) + utils.assert_string(1,s) + local find,sub,append = string.find, string.sub, table.insert + local i1,ls = 1,{} + if not re then re = '%s+' end + if re == '' then return {s} end + while true do + local i2,i3 = find(s,re,i1,plain) + if not i2 then + local last = sub(s,i1) + if last ~= '' then append(ls,last) end + if #ls == 1 and ls[1] == '' then + return {} + else + return ls + end + end + append(ls,sub(s,i1,i2-1)) + if n and #ls == n then + ls[#ls] = sub(s,i1) + return ls + end + i1 = i3+1 + end +end + +--- split a string into a number of values. +-- @param s the string +-- @param re the delimiter, default space +-- @return n values +-- @usage first,next = splitv('jane:doe',':') +-- @see split +function utils.splitv (s,re) + return unpack(utils.split(s,re)) +end + +--- convert an array of values to strings. +-- @param t a list-like table +-- @param temp buffer to use, otherwise allocate +-- @param tostr custom tostring function, called with (value,index). +-- Otherwise use `tostring` +-- @return the converted buffer +function utils.array_tostring (t,temp,tostr) + temp, tostr = temp or {}, tostr or tostring + for i = 1,#t do + temp[i] = tostr(t[i],i) + end + return temp +end + +local is_windows = package.config:sub(1, 1) == "\\" + +--- Quote an argument of a command. +-- Quotes a single argument of a command to be passed +-- to `os.execute`, `pl.utils.execute` or `pl.utils.executeex`. +-- @string argument the argument. +-- @return quoted argument. +function utils.quote_arg(argument) + if is_windows then + if argument == "" or argument:find('[ \f\t\v]') then + -- Need to quote the argument. + -- Quotes need to be escaped with backslashes; + -- additionally, backslashes before a quote, escaped or not, + -- need to be doubled. + -- See documentation for CommandLineToArgvW Windows function. + argument = '"' .. argument:gsub([[(\*)"]], [[%1%1\"]]):gsub([[\+$]], "%0%0") .. '"' + end + + -- os.execute() uses system() C function, which on Windows passes command + -- to cmd.exe. Escape its special characters. + return (argument:gsub('["^<>!|&%%]', "^%0")) + else + if argument == "" or argument:find('[^a-zA-Z0-9_@%+=:,./-]') then + -- To quote arguments on posix-like systems use single quotes. + -- To represent an embedded single quote close quoted string ('), + -- add escaped quote (\'), open quoted string again ('). + argument = "'" .. argument:gsub("'", [['\'']]) .. "'" + end + + return argument + end +end + +--- execute a shell command and return the output. +-- This function redirects the output to tempfiles and returns the content of those files. +-- @param cmd a shell command +-- @param bin boolean, if true, read output as binary file +-- @return true if successful +-- @return actual return code +-- @return stdout output (string) +-- @return errout output (string) +function utils.executeex(cmd, bin) + local mode + local outfile = os.tmpname() + local errfile = os.tmpname() + + if is_windows and not outfile:find(':') then + outfile = os.getenv('TEMP')..outfile + errfile = os.getenv('TEMP')..errfile + end + cmd = cmd .. " > " .. utils.quote_arg(outfile) .. " 2> " .. utils.quote_arg(errfile) + + local success, retcode = utils.execute(cmd) + local outcontent = utils.readfile(outfile, bin) + local errcontent = utils.readfile(errfile, bin) + os.remove(outfile) + os.remove(errfile) + return success, retcode, (outcontent or ""), (errcontent or "") +end + +--- 'memoize' a function (cache returned value for next call). +-- This is useful if you have a function which is relatively expensive, +-- but you don't know in advance what values will be required, so +-- building a table upfront is wasteful/impossible. +-- @param func a function of at least one argument +-- @return a function with at least one argument, which is used as the key. +function utils.memoize(func) + local cache = {} + return function(k) + local res = cache[k] + if res == nil then + res = func(k) + cache[k] = res + end + return res + end +end + + +utils.stdmt = { + List = {_name='List'}, Map = {_name='Map'}, + Set = {_name='Set'}, MultiMap = {_name='MultiMap'} +} + +local _function_factories = {} + +--- associate a function factory with a type. +-- A function factory takes an object of the given type and +-- returns a function for evaluating it +-- @tab mt metatable +-- @func fun a callable that returns a function +function utils.add_function_factory (mt,fun) + _function_factories[mt] = fun +end + +local function _string_lambda(f) + local raise = utils.raise + if f:find '^|' or f:find '_' then + local args,body = f:match '|([^|]*)|(.+)' + if f:find '_' then + args = '_' + body = f + else + if not args then return raise 'bad string lambda' end + end + local fstr = 'return function('..args..') return '..body..' end' + local fn,err = utils.load(fstr) + if not fn then return raise(err) end + fn = fn() + return fn + else return raise 'not a string lambda' + end +end + +--- an anonymous function as a string. This string is either of the form +-- '|args| expression' or is a function of one argument, '_' +-- @param lf function as a string +-- @return a function +-- @usage string_lambda '|x|x+1' (2) == 3 +-- @usage string_lambda '_+1' (2) == 3 +-- @function utils.string_lambda +utils.string_lambda = utils.memoize(_string_lambda) + +local ops + +--- process a function argument. +-- This is used throughout Penlight and defines what is meant by a function: +-- Something that is callable, or an operator string as defined by pl.operator, +-- such as '>' or '#'. If a function factory has been registered for the type, it will +-- be called to get the function. +-- @param idx argument index +-- @param f a function, operator string, or callable object +-- @param msg optional error message +-- @return a callable +-- @raise if idx is not a number or if f is not callable +function utils.function_arg (idx,f,msg) + utils.assert_arg(1,idx,'number') + local tp = type(f) + if tp == 'function' then return f end -- no worries! + -- ok, a string can correspond to an operator (like '==') + if tp == 'string' then + if not ops then ops = require 'pl.operator'.optable end + local fn = ops[f] + if fn then return fn end + local fn, err = utils.string_lambda(f) + if not fn then error(err..': '..f) end + return fn + elseif tp == 'table' or tp == 'userdata' then + local mt = getmetatable(f) + if not mt then error('not a callable object',2) end + local ff = _function_factories[mt] + if not ff then + if not mt.__call then error('not a callable object',2) end + return f + else + return ff(f) -- we have a function factory for this type! + end + end + if not msg then msg = " must be callable" end + if idx > 0 then + error("argument "..idx..": "..msg,2) + else + error(msg,2) + end +end + +--- bind the first argument of the function to a value. +-- @param fn a function of at least two values (may be an operator string) +-- @param p a value +-- @return a function such that f(x) is fn(p,x) +-- @raise same as @{function_arg} +-- @see func.bind1 +function utils.bind1 (fn,p) + fn = utils.function_arg(1,fn) + return function(...) return fn(p,...) end +end + +--- bind the second argument of the function to a value. +-- @param fn a function of at least two values (may be an operator string) +-- @param p a value +-- @return a function such that f(x) is fn(x,p) +-- @raise same as @{function_arg} +function utils.bind2 (fn,p) + fn = utils.function_arg(1,fn) + return function(x,...) return fn(x,p,...) end +end + + +--- assert that the given argument is in fact of the correct type. +-- @param n argument index +-- @param val the value +-- @param tp the type +-- @param verify an optional verification function +-- @param msg an optional custom message +-- @param lev optional stack position for trace, default 2 +-- @raise if the argument n is not the correct type +-- @usage assert_arg(1,t,'table') +-- @usage assert_arg(n,val,'string',path.isdir,'not a directory') +function utils.assert_arg (n,val,tp,verify,msg,lev) + if type(val) ~= tp then + error(("argument %d expected a '%s', got a '%s'"):format(n,tp,type(val)),lev or 2) + end + if verify and not verify(val) then + error(("argument %d: '%s' %s"):format(n,val,msg),lev or 2) + end +end + +--- assert the common case that the argument is a string. +-- @param n argument index +-- @param val a value that must be a string +-- @raise val must be a string +function utils.assert_string (n,val) + utils.assert_arg(n,val,'string',nil,nil,3) +end + +local err_mode = 'default' + +--- control the error strategy used by Penlight. +-- Controls how utils.raise works; the default is for it +-- to return nil and the error string, but if the mode is 'error' then +-- it will throw an error. If mode is 'quit' it will immediately terminate +-- the program. +-- @param mode - either 'default', 'quit' or 'error' +-- @see utils.raise +function utils.on_error (mode) + if ({['default'] = 1, ['quit'] = 2, ['error'] = 3})[mode] then + err_mode = mode + else + -- fail loudly + if err_mode == 'default' then err_mode = 'error' end + utils.raise("Bad argument expected string; 'default', 'quit', or 'error'. Got '"..tostring(mode).."'") + end +end + +--- used by Penlight functions to return errors. Its global behaviour is controlled +-- by utils.on_error +-- @param err the error string. +-- @see utils.on_error +function utils.raise (err) + if err_mode == 'default' then return nil,err + elseif err_mode == 'quit' then utils.quit(err) + else error(err,2) + end +end + +--- is the object of the specified type?. +-- If the type is a string, then use type, otherwise compare with metatable +-- @param obj An object to check +-- @param tp String of what type it should be +function utils.is_type (obj,tp) + if type(tp) == 'string' then return type(obj) == tp end + local mt = getmetatable(obj) + return tp == mt +end + +raise = utils.raise + +--- load a code string or bytecode chunk. +-- @param code Lua code as a string or bytecode +-- @param name for source errors +-- @param mode kind of chunk, 't' for text, 'b' for bytecode, 'bt' for all (default) +-- @param env the environment for the new chunk (default nil) +-- @return compiled chunk +-- @return error message (chunk is nil) +-- @function utils.load + +--------------- +-- Get environment of a function. +-- With Lua 5.2, may return nil for a function with no global references! +-- Based on code by [Sergey Rozhenko](http://lua-users.org/lists/lua-l/2010-06/msg00313.html) +-- @param f a function or a call stack reference +-- @function utils.getfenv + +--------------- +-- Set environment of a function +-- @param f a function or a call stack reference +-- @param env a table that becomes the new environment of `f` +-- @function utils.setfenv + +--- execute a shell command. +-- This is a compatibility function that returns the same for Lua 5.1 and Lua 5.2 +-- @param cmd a shell command +-- @return true if successful +-- @return actual return code +-- @function utils.execute + +return utils + + diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/msb-enabled/location-default/msblocations.conf b/openresty-ext/src/assembly/resources/openresty/nginx/msb-enabled/location-default/msblocations.conf index 745c290..2150e70 100644 --- a/openresty-ext/src/assembly/resources/openresty/nginx/msb-enabled/location-default/msblocations.conf +++ b/openresty-ext/src/assembly/resources/openresty/nginx/msb-enabled/location-default/msblocations.conf @@ -1,5 +1,5 @@ # -# Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE) +# Copyright (C) 2017-2018 ZTE, Inc. and others. All rights reserved. (ZTE) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,10 +16,7 @@ default_type text/html; - # the flag identify whether to check doing internal redirect or not - set $websocket_internal_redirect "on"; set $http_protocol "http"; - location = /iui/microservices { try_files $uri @addslash; @@ -37,23 +34,29 @@ set $svc_url ""; set $backend "defaultbackend"; - #rewrite by the lua file - rewrite_by_lua_file luaext/rewrite/customrewrite.lua; access_by_lua_block { + ngx.log(ngx.INFO, ngx.var.request_id..":receive the request") + msb.filter_websocket_req() + msb.route() msb.access() + if ngx.ctx.use_ownupstream then + stats.forward_backend() + end } header_filter_by_lua_block { + stats.receive_response() msb.header_filter() + stats.return_response() } #log by the lua file log_by_lua_file luaext/log/logger.lua; proxy_pass $http_protocol://$backend; - proxy_redirect $http_protocol://$host:$server_port$svc_url $http_protocol://$host:$server_port$svc_name; + #proxy_redirect $http_protocol://$host:$server_port$svc_url $http_protocol://$host:$server_port$svc_name; } - location @customwebsocket { - set $websocket_internal_redirect "off"; + location @websocket { + set $stats_new_req "false"; #set header for websocket proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; @@ -63,38 +66,18 @@ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Host $host:$server_port; - #rewrite by the lua file - rewrite_by_lua_file luaext/rewrite/customrewrite.lua; - access_by_lua_block { - msb.access() - } - header_filter_by_lua_block { - msb.header_filter() - } - #log by the lua file - log_by_lua_file luaext/log/logger.lua; - - proxy_pass $http_protocol://$backend; - } - - location @commonwebsocket { - set $websocket_internal_redirect "off"; - #set header for websocket - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $host:$server_port; - - #rewrite by the lua file - rewrite_by_lua_file luaext/rewrite/commonrewrite.lua; access_by_lua_block { + ngx.log(ngx.INFO, ngx.var.request_id..":receive the request") + msb.route() msb.access() + if ngx.ctx.use_ownupstream then + stats.forward_backend() + end } header_filter_by_lua_block { + stats.receive_response() msb.header_filter() + stats.return_response() } #log by the lua file log_by_lua_file luaext/log/logger.lua; @@ -108,19 +91,26 @@ set $svc_url ""; set $backend "defaultbackend"; - #rewrite by the lua file - rewrite_by_lua_file luaext/rewrite/customrewrite.lua; access_by_lua_block { + ngx.log(ngx.INFO, ngx.var.request_id..":receive the request") + stats.accept_new_request() + msb.filter_websocket_req() + msb.route() msb.access() + if ngx.ctx.use_ownupstream then + stats.forward_backend() + end } header_filter_by_lua_block { + stats.receive_response() msb.header_filter() + stats.return_response() } #log by the lua file log_by_lua_file luaext/log/logger.lua; proxy_pass $http_protocol://$backend; - proxy_redirect $http_protocol://$host:$server_port$svc_url $http_protocol://$host:$server_port$svc_name; + #proxy_redirect $http_protocol://$host:$server_port$svc_url $http_protocol://$host:$server_port$svc_name; } location ~ ^/(api|admin|apijson)(/[Vv]\d+(?:\.\d+)*)?/([^/]+)(/[Vv]\d+(?:\.\d+)*)?(.*) { @@ -131,13 +121,20 @@ set $req_res $5; set $backend "defaultbackend"; - #rewrite by the lua file - rewrite_by_lua_file luaext/rewrite/commonrewrite.lua; access_by_lua_block { + ngx.log(ngx.INFO, ngx.var.request_id..":receive the request") + stats.accept_new_request() + msb.filter_websocket_req() + msb.route() msb.access() + if ngx.ctx.use_ownupstream then + stats.forward_backend() + end } header_filter_by_lua_block { + stats.receive_response() msb.header_filter() + stats.return_response() } #log by the lua file log_by_lua_file luaext/log/logger.lua; @@ -151,13 +148,20 @@ set $req_res $2.$3; set $backend "defaultbackend"; - #rewrite by the lua file - rewrite_by_lua_file luaext/rewrite/commonrewrite.lua; access_by_lua_block { + ngx.log(ngx.INFO, ngx.var.request_id..":receive the request") + stats.accept_new_request() + msb.filter_websocket_req() + msb.route() msb.access() + if ngx.ctx.use_ownupstream then + stats.forward_backend() + end } header_filter_by_lua_block { + stats.receive_response() msb.header_filter() + stats.return_response() } #log by the lua file log_by_lua_file luaext/log/logger.lua; @@ -173,13 +177,20 @@ set $req_res $2; set $backend "defaultbackend"; - #rewrite by the lua file - rewrite_by_lua_file luaext/rewrite/commonrewrite.lua; access_by_lua_block { + ngx.log(ngx.INFO, ngx.var.request_id..":receive the request") + stats.accept_new_request() + msb.filter_websocket_req() + msb.route() msb.access() + if ngx.ctx.use_ownupstream then + stats.forward_backend() + end } header_filter_by_lua_block { + stats.receive_response() msb.header_filter() + stats.return_response() } #log by the lua file log_by_lua_file luaext/log/logger.lua; @@ -196,13 +207,20 @@ set $svc_url ""; set $backend "defaultbackend"; - #rewrite by the lua file - rewrite_by_lua_file luaext/rewrite/customrewrite.lua; access_by_lua_block { + ngx.log(ngx.INFO, ngx.var.request_id..":receive the request") + stats.accept_new_request() + msb.filter_websocket_req() + msb.route() msb.access() + if ngx.ctx.use_ownupstream then + stats.forward_backend() + end } header_filter_by_lua_block { + stats.receive_response() msb.header_filter() + stats.return_response() } #log by the lua file log_by_lua_file luaext/log/logger.lua; @@ -210,9 +228,9 @@ proxy_cache nginx_cache; add_header X-Cache-Status $upstream_cache_status; proxy_pass $http_protocol://$backend; - proxy_redirect http://$host:$server_port$svc_url http://$host:$server_port$svc_name; + #proxy_redirect http://$host:$server_port$svc_url http://$host:$server_port$svc_name; } location = /favicon.ico { log_not_found off; - } \ No newline at end of file + } diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/msb-enabled/location-ext/monitor.conf b/openresty-ext/src/assembly/resources/openresty/nginx/msb-enabled/location-ext/monitor.conf new file mode 100644 index 0000000..c1ae2d2 --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/msb-enabled/location-ext/monitor.conf @@ -0,0 +1,56 @@ +# +# Copyright (C) 2018 ZTE, Inc. and others. All rights reserved. (ZTE) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +location = /admin/microservices/v1/status/connection { + access_log off; + content_by_lua_block{ + if ngx.req.get_method() == "GET" then + ngx.print(stats.format_conn_status()) + else + ngx.status = ngx.HTTP_NOT_FOUND + ngx.say("request method not supported!") + return ngx.exit(ngx.status) + end + } +} +location = /admin/microservices/v1/status/request { + access_log off; + content_by_lua_block{ + if ngx.req.get_method() == "GET" then + ngx.print(stats.format_req_status()) + else + ngx.status = ngx.HTTP_NOT_FOUND + ngx.say("request method not supported!") + return ngx.exit(ngx.status) + end + } +} +location = /admin/microservices/v1/statistics/request { + access_log off; + content_by_lua_block{ + if ngx.req.get_method() == "GET" then + --if input latestNum is empty or illegal, then set to 1 + local latest_num = tonumber(ngx.var.arg_latestNum) or 1 + if latest_num<=0 then + latest_num =1 + end + ngx.print(stats.get_reqnum_stats(latest_num)) + else + ngx.status = ngx.HTTP_NOT_FOUND + ngx.say("request method not supported!") + return ngx.exit(ngx.status) + end + } +} diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/msb-enabled/msb.conf b/openresty-ext/src/assembly/resources/openresty/nginx/msb-enabled/msb.conf index 8e854bb..3e7038b 100644 --- a/openresty-ext/src/assembly/resources/openresty/nginx/msb-enabled/msb.conf +++ b/openresty-ext/src/assembly/resources/openresty/nginx/msb-enabled/msb.conf @@ -1,5 +1,5 @@ # -# Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE) +# Copyright (C) 2017-2018 ZTE, Inc. and others. All rights reserved. (ZTE) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -41,13 +41,15 @@ proxy_next_upstream error timeout; proxy_next_upstream_tries 5; # Lua settings -lua_package_path "$prefix/../lualib/?.lua;$prefix/luaext/?.lua;;"; +lua_package_path "$prefix/../lualib/?.lua;$prefix/luaext/?.lua;$prefix/luaext/vendor/?.lua;;"; lua_package_cpath "$prefix/../lualib/?.so;;"; #lua_shared_dict rr_cache 1M; #lua_shared_dict rr_locks 100k; lua_shared_dict svc_cache 5M; lua_shared_dict locks 200k; +lua_shared_dict stats 1M; +lua_shared_dict dns_cache 1M; lua_code_cache on; @@ -60,8 +62,12 @@ upstream defaultbackend { init_by_lua_block { msb = require('msb') msb.load_plugins() + stats = require ('monitor.stats') +} +init_worker_by_lua_block { + stats.init_timer() } server { listen 80; include ../msb-enabled/location-default/msblocations.conf; -} \ No newline at end of file +} diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/msb-enabled/msbhttps.conf b/openresty-ext/src/assembly/resources/openresty/nginx/msb-enabled/msbhttps.conf index 430c05d..9015d65 100644 --- a/openresty-ext/src/assembly/resources/openresty/nginx/msb-enabled/msbhttps.conf +++ b/openresty-ext/src/assembly/resources/openresty/nginx/msb-enabled/msbhttps.conf @@ -1,5 +1,5 @@ # -# Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE) +# Copyright (C) 2017-2018 ZTE, Inc. and others. All rights reserved. (ZTE) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ server { listen 443 ssl; ssl_certificate ../ssl/cert/cert.crt; ssl_certificate_key ../ssl/cert/cert.key; + ssl_protocols TLSv1.1 TLSv1.2; ssl_dhparam ../ssl/dh-pubkey/dhparams.pem; include ../msb-enabled/location-default/msblocations.conf; -} \ No newline at end of file +} diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/sites-enabled/cosSample b/openresty-ext/src/assembly/resources/openresty/nginx/sites-enabled/cosSample new file mode 100644 index 0000000..b5891f8 --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/sites-enabled/cosSample @@ -0,0 +1,88 @@ +# +# Copyright (C) 2018 ZTE, Inc. and others. All rights reserved. (ZTE) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +server { + listen 8000; + default_type text/html; + + set $http_protocol "http"; + + location / { + set $svc_type ""; + set $svc_name ""; + set $svc_version1 ""; + set $svc_version2 ""; + set $req_res ""; + set $svc_url ""; + set $backend "defaultbackend"; + access_by_lua_block { + ngx.log(ngx.INFO, ngx.var.request_id..":receive the request") + stats.accept_new_request() + msb.filter_websocket_req() + -- 80 is the publish port in the msb, COS is the system_tag. These two parameters are required + msb.external_route("80","COS") + msb.access() + if ngx.ctx.use_ownupstream then + stats.forward_backend() + end + } + header_filter_by_lua_block { + stats.receive_response() + msb.header_filter() + stats.return_response() + } + #log by the lua file + log_by_lua_file luaext/log/logger.lua; + + proxy_pass $http_protocol://$backend; + + } + + location @websocket { + #set header for websocket + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host:$server_port; + access_by_lua_block { + ngx.log(ngx.INFO, ngx.var.request_id..":receive the request") + msb.external_route("80","COS") + msb.access() + if ngx.ctx.use_ownupstream then + stats.forward_backend() + end + } + header_filter_by_lua_block { + stats.receive_response() + msb.header_filter() + stats.return_response() + } + #log by the lua file + log_by_lua_file luaext/log/logger.lua; + + proxy_pass $http_protocol://$backend; + } + + location @commonnotfound { + return 502; + } + + location = /favicon.ico { + log_not_found off; + } +} diff --git a/openresty-ext/src/assembly/resources/openresty/reload.sh b/openresty-ext/src/assembly/resources/openresty/reload.sh index 240b4b0..601df14 100644 --- a/openresty-ext/src/assembly/resources/openresty/reload.sh +++ b/openresty-ext/src/assembly/resources/openresty/reload.sh @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE) +# Copyright (C) 2017-2018 ZTE, Inc. and others. All rights reserved. (ZTE) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,7 +20,8 @@ HOME=`cd $DIRNAME/nginx; pwd` _NGINXCMD="$HOME/sbin/nginx" cd $HOME; pwd -echo =========== begin to reload =============== +SYSTEM_TIME=`date '+%Y-%m-%d %T'` +echo ${SYSTEM_TIME} ========= begin to reload =============== echo @WORK_DIR@ $HOME echo @C_CMD@ $_NGINXCMD -p $HOME/ -s reload -$_NGINXCMD -p $HOME/ -s reload \ No newline at end of file +$_NGINXCMD -p $HOME/ -s reload -- cgit 1.2.3-korg