diff options
author | HuabingZhao <zhao.huabing@zte.com.cn> | 2017-07-25 15:18:33 +0800 |
---|---|---|
committer | HuabingZhao <zhao.huabing@zte.com.cn> | 2017-07-25 18:11:59 +0800 |
commit | 672f3d40be83d9e380fd7be4b674d5e8d5fa36de (patch) | |
tree | 43105e1d5e2ba8e8accea8648e57e1cf87db3f00 /openresty-ext/src/assembly/resources/openresty/nginx/luaext | |
parent | 41d3db15a8e1a0496f9c2a5e15db2998a32bb9bf (diff) |
Divide the MSB source codes into two repos
Change-Id: Ie76d545b214a8ce5191f215350a623e1529983d9
Issue-id: MSB-5
Signed-off-by: HuabingZhao <zhao.huabing@zte.com.cn>
Diffstat (limited to 'openresty-ext/src/assembly/resources/openresty/nginx/luaext')
27 files changed, 2260 insertions, 0 deletions
diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/conf/msbinit.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/conf/msbinit.lua new file mode 100644 index 0000000..bc8a13f --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/conf/msbinit.lua @@ -0,0 +1,58 @@ +--[[ + + Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE) + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--]] + +local modulename = "msbinit" +local _M = {} +_M._VERSION = '1.0.0' + +local str_util = require('lib.utils.str_util') +local mark_empty_as_nil = str_util.mark_empty_as_nil + +_M.systemConf = { + ["defaultport"] = mark_empty_as_nil(os.getenv("HTTP_OVERWRITE_PORT")) or "80", + ["defaulthttpsport"] = mark_empty_as_nil(os.getenv("HTTPS_OVERWRITE_PORT")) or "443", + ["defaultprefix"] = "msb:routing", + ["enablefullsearch"] = true, --whether test against the custom services after common match processing + ["enablerefercheck"] = true, --whether use refer to test against the service names as the last solution + ["useconsultemplate"] = true --whether using consul template or not +} +_M.redisConf = { + ["host"] = "127.0.0.1", + ["port"] = 6379, + ["poolsize"] = 100, + ["idletimeout"] = 90000, + ["timeout"] = 1000, + ["dbid"] = 0 +} +_M.cacheConf = { + ["positive_ttl"] = 5, --shcache use,in seconds + ["negative_ttl"] = 2, --shcache use,in seconds + ["actualize_ttl"] = 120, --shcache use,in seconds + ["lru_ttl"] = 2 --in seconds +} +_M.routerConf = { + -- cross-domain plugin uses this field in init_by_lua context, ngx.re.gsub is not allowed in this context, using string.gsub instead + --["subdomain"] = ngx.re.gsub(os.getenv("ROUTER_SBUDOMAIN") or "openpalette.zte.com.cn", "\\.", "\\.", "o"), + ["subdomain"] = string.gsub(mark_empty_as_nil(os.getenv("ROUTER_SUBDOMAIN")) or "openpalette.zte.com.cn", "%.", "\\."), + ["defaultprefix"]= "msb:host" +} +_M.server = { + ["fail_timeout"] = 10, + ["max_fails"] = 1 +} +return _M
\ No newline at end of file diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/conf/svcconf.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/conf/svcconf.lua new file mode 100644 index 0000000..df6356e --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/conf/svcconf.lua @@ -0,0 +1,34 @@ +--[[
+
+ Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE)
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+--]]
+
+local _M = {}
+_M._VERSION = '1.0.0'
+
+_M.apiRelatedTypes = {
+ ["api"] = true,
+ ["admin"] = true,
+ ["apijson"] = true
+}
+_M.urlfieldMap = {
+ ["api"] = "url",
+ ["admin"] = "metricsUrl",
+ ["apijson"] = "apijson",
+ ["iui"] = "url",
+ ["custom"] = "url"
+}
+return _M
\ No newline at end of file diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/dao/dao.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/dao/dao.lua new file mode 100644 index 0000000..58d058c --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/dao/dao.lua @@ -0,0 +1,123 @@ +--[[
+
+ Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE)
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+--]]
+
+--This module integrates the shcache with an abstraction for various databases(redis and so on)
+local _M = {}
+_M._VERSION = '1.0.0'
+
+local shcache = require("vendor.shcache")
+local cjson_safe = require "cjson.safe"
+local msbConf = require('conf.msbinit')
+
+local DB = require('dao.redis_db')
+local options = msbConf.redisConf
+
+local cacheConf = msbConf.cacheConf
+local positive_ttl = cacheConf.positive_ttl or 60
+local negative_ttl = cacheConf.negative_ttl or 2
+local actualize_ttl = cacheConf.actualize_ttl or 120
+
+local svc_shcache = ngx.shared.svc_cache
+
+local function load_serviceinfo(key)
+
+ -- closure to perform external lookup to redis
+ local fetch_svcinfo_from_db = function ()
+ local _db = DB:new(options)
+ return _db:getserviceinfo(key)
+ end
+
+ local svcinfo_cache_table = shcache:new(
+ svc_shcache,
+ { external_lookup = fetch_svcinfo_from_db,
+ --encode = cmsgpack.pack,
+ --encode = cjson_safe.encode,
+ --decode = cmsgpack.unpack
+ --decode = cjson_safe.decode
+ },
+ { positive_ttl = positive_ttl, -- default cache good data for 60s
+ negative_ttl = negative_ttl, -- default cache failed lookup for 5s
+ actualize_ttl = actualize_ttl,
+ name = 'svcinfo_cache' -- "named" cache, useful for debug / report
+ }
+ )
+
+ local serviceinfo, from_cache = svcinfo_cache_table:load(key)
+
+ return serviceinfo
+end
+_M.load_serviceinfo = load_serviceinfo
+
+local function load_backservers(keypattern)
+
+ -- closure to perform external lookup to redis
+ local fetch_servers_from_db = function ()
+ local _db = DB:new(options)
+ return _db:getbackservers(keypattern)
+ end
+
+ local servers_cache_table = shcache:new(
+ svc_shcache,
+ { external_lookup = fetch_servers_from_db,
+ --encode = cmsgpack.pack,
+ encode = cjson_safe.encode,
+ --decode = cmsgpack.unpack
+ decode = cjson_safe.decode
+ },
+ { positive_ttl = positive_ttl, -- default cache good data for 60s
+ negative_ttl = negative_ttl, -- default cache failed lookup for 5s
+ name = 'servers_cache' -- "named" cache, useful for debug / report
+ }
+ )
+
+ local servers_table, from_cache = servers_cache_table:load(keypattern)
+
+ return servers_table
+end
+_M.load_backservers = load_backservers
+
+
+local function load_customsvcnames(keypattern)
+ -- closure to perform external lookup to redis
+ local fetch_svcnames_from_db = function ()
+ local _db = DB:new(options)
+ return _db:getcustomsvcnames(keypattern)
+ end
+
+ local svcnames_cache_table = shcache:new(
+ svc_shcache,
+ { external_lookup = fetch_svcnames_from_db,
+ --encode = cmsgpack.pack,
+ encode = cjson_safe.encode,
+ --decode = cmsgpack.unpack
+ decode = cjson_safe.decode
+ },
+ { positive_ttl = positive_ttl, -- default cache good data for 60s
+ negative_ttl = negative_ttl, -- default cache failed lookup for 5s
+ actualize_ttl = actualize_ttl,
+ name = 'svcnames_cache' -- "named" cache, useful for debug / report
+ }
+ )
+
+ local service_names, from_cache = svcnames_cache_table:load(keypattern)
+
+ return service_names
+end
+_M.load_customsvcnames = load_customsvcnames
+
+return _M
\ No newline at end of file diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/dao/db_access.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/dao/db_access.lua new file mode 100644 index 0000000..35572cc --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/dao/db_access.lua @@ -0,0 +1,85 @@ +--[[
+
+ Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE)
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+--]]
+
+-- unified layer to access back DB, using two levels of cache mechanism(LRUCache and shcache)
+local _M = {}
+_M._VERSION = '1.0.0'
+
+local cache = require('lib.tools.db_cache')
+local dao = require('dao.dao')
+local msbConf = require('conf.msbinit')
+local cjson_safe = require "cjson.safe"
+local cacheConf = msbConf.cacheConf
+local lru_ttl = cacheConf.lru_ttl or 10
+
+function _M.load_serviceinfo(key)
+ -- Try to get from cache
+ local cached_value = cache.get(key)
+ if cached_value then
+ if cache.is_empty(cached_value) then
+ return nil
+ end
+ return cached_value
+ end
+ -- Get from shcache or backend redis
+ local svcinfo_str = dao.load_serviceinfo(key)
+ local value,err
+ if svcinfo_str then
+ value, err = cjson_safe.decode(svcinfo_str)
+ if err then
+ value = nil
+ ngx.log(ngx.WARN, "decode service info error: ", err, " service info: ", svcinfo_str)
+ end
+ else
+ value = nil
+ end
+ cache.set(key, value, lru_ttl)
+ return value
+end
+
+function _M.load_backservers(keypattern)
+ local value, err
+ -- Try to get from cache
+ value = cache.get(keypattern)
+ if not value then
+ -- Get from shcache or backend redis
+ value = dao.load_backservers(keypattern)
+ cache.set(keypattern, value, lru_ttl)
+ end
+ if cache.is_empty(value) then
+ return nil
+ end
+ return value
+end
+
+function _M.load_customsvcnames(keypattern)
+ local value, err
+ -- Try to get from cache
+ value = cache.get(keypattern)
+ if not value then
+ -- Get from shcache or backend redis
+ value = dao.load_customsvcnames(keypattern)
+ cache.set(keypattern, value, lru_ttl)
+ end
+ if cache.is_empty(value) then
+ return nil
+ end
+ return value
+end
+
+return _M
\ No newline at end of file diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/dao/redis_db.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/dao/redis_db.lua new file mode 100644 index 0000000..d35cb8e --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/dao/redis_db.lua @@ -0,0 +1,199 @@ +--[[
+
+ Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE)
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+--]]
+
+-- the client for redis, include the connection pool management and api implements
+local redis = require('resty.redis')
+local tbl_util = require('lib.utils.table_util')
+
+local _M = {
+}
+_M._VERSION = '0.0.1'
+_M._DESCRIPTION = 'msb_redis_module'
+
+local mt = { __index = _M }
+
+local tbl_insert = table.insert
+local tbl_sort = table.sort
+local tbl_isempty = tbl_util.isempty
+
+function _M.new(self, conf)
+ self.host = conf.host
+ self.port = conf.port
+ self.timeout = conf.timeout
+ self.dbid = conf.dbid
+ self.poolsize = conf.poolsize
+ self.idletimeout = conf.idletimeout
+
+ local red = redis:new()
+ return setmetatable({redis = red}, mt)
+end
+
+function _M.connectdb(self)
+ local host = self.host
+ local port = self.port
+ local dbid = self.dbid
+ local red = self.redis
+
+ if not (host and port) then
+ return nil, 'no host:port avaliable provided'
+ end
+
+ --set default value
+ if not dbid then dbid = 0 end
+ local timeout = self.timeout
+ if not timeout then
+ timeout = 1000 -- 1s
+ end
+
+ red:set_timeout(timeout)
+
+ local ok, err
+ if host and port then
+ ok, err = red:connect(host, port)
+ if ok then return red:select(dbid) end
+ end
+
+ return ok, err
+end
+
+function _M.keepalivedb(self)
+ local max_idle_timeout = self.idletimeout --ms
+ local pool_size = self.poolsize
+
+ if not pool_size then pool_size = 100 end
+ if not max_idle_timeout then max_idle_timeout = 90000 end --90s
+
+ local ok, err = self.redis:set_keepalive(max_idle_timeout, pool_size)
+ if not ok then
+ ngx.log(ngx.ERR, "redis pool keepalive error",err)
+ return
+ end
+ return
+end
+
+--inner function,only used in this module
+local function _hgetall(red,key)
+ local resp,err = red:hgetall(key)
+ --if not resp or next(resp) == nil then
+ if tbl_isempty(resp) then
+ return nil, "key "..key.." not found"
+ end
+ local hashinfo = red:array_to_hash(resp)
+ return hashinfo,nil
+end
+
+function _M.getserviceinfo(self,key)
+ if not key then
+ return nil,'no key is provided'
+ end
+ local c, err = self:connectdb()
+ if not c then
+ return nil, err
+ end
+
+ local red = self.redis
+ local resp,err = red:get(key) --the key will create dynamically
+ self:keepalivedb()
+ if not resp or resp == ngx.null then
+ return nil, "key "..key.." not found"
+ else
+ return resp,nil
+ end
+end
+
+function _M.getbackservers(self,keypattern)
+ if not keypattern then
+ return nil,'no keypattern is provided'
+ end
+ local c, err = self:connectdb()
+ if not c then
+ return nil, err
+ end
+
+ local red = self.redis
+
+ local resp, err = red:keys(keypattern)
+ if tbl_isempty(resp) then
+ self:keepalivedb()
+ return nil, "no server matched"
+ end
+
+ local servers = {}
+ for i, v in ipairs(resp) do
+ local serverinfo,err = _hgetall(red,v)
+ if serverinfo then
+ tbl_insert(servers,serverinfo)
+ end
+ end
+ self:keepalivedb()
+ return servers,nil
+end
+
+function _M.getcustomsvcnames(self,keypattern)
+ if not keypattern then
+ return nil,'no keypattern is provided'
+ end
+ local c, err = self:connectdb()
+ if not c then
+ return nil, err
+ end
+
+ local red = self.redis
+ --store svc names into the Set
+ local svcname_set={}
+
+ local res, err = red:scan("0","count",50,"match",keypattern)
+ if not res then
+ self:keepalivedb()
+ return {}, "failed to scan in getcustomsvcnames()"
+ end
+
+ local cursor, keys, err = unpack(res)
+ for _, svckey in ipairs(keys) do
+ local m, err = ngx.re.match(svckey, "^.+:custom:([^:]+)", "o")
+ if m then
+ svcname_set[m[1]]=true
+ end
+ end
+
+ while( cursor ~= "0" )
+ do
+ res = red:scan(cursor,"count",50,"match",keypattern)
+ if not res then
+ break
+ end
+ cursor, keys, err = unpack(res)
+ for _, svckey in ipairs(keys) do
+ local m, err = ngx.re.match(svckey, "^.+:custom:([^:]+)", "o")
+ if m then
+ svcname_set[m[1]]=true
+ end
+ end
+ end
+ self:keepalivedb()
+ local svcnames = {}
+ for svcname,_ in pairs(svcname_set) do
+ tbl_insert(svcnames,svcname)
+ end
+ --sort the key_table in reverse order
+ tbl_sort(svcnames, function (a, b)
+ return a > b
+ end)
+ return svcnames,nil
+end
+return _M
\ No newline at end of file diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/lib/tools/db_cache.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/lib/tools/db_cache.lua new file mode 100644 index 0000000..7f40590 --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/lib/tools/db_cache.lua @@ -0,0 +1,54 @@ +--[[
+
+ Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE)
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+--]]
+
+-- db cache over LRUCache, used within one worker processes
+local _M = {}
+_M._VERSION = '1.0.0'
+
+local lrucache = require "resty.lrucache"
+
+local EMPTY_DATA = '_EMPTY_'
+
+-- we need to initialize the cache on the lua module level so that
+-- it can be shared by all the requests served by each nginx worker process:
+local cache,err = lrucache.new(200) -- allow up to 200 items in the cache
+if not cache then
+ return ngx.log(ngx.ERR,"failed to create the cache: " .. (err or "unknown"))
+end
+
+function _M.get(key)
+ return cache:get(key)
+end
+
+function _M.set(key,value)
+ return _M:set(key, value, nil)
+end
+
+function _M.set(key,value,ttl)
+ if not value then
+ value = EMPTY_DATA
+ end
+ return cache:set(key, value, ttl)
+end
+
+-- check if the data returned by get() is considered empty
+function _M.is_empty(data)
+ return data == EMPTY_DATA
+end
+
+return _M
\ No newline at end of file diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/lib/tools/rr_cache.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/lib/tools/rr_cache.lua new file mode 100644 index 0000000..bfcc157 --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/lib/tools/rr_cache.lua @@ -0,0 +1,40 @@ +--[[
+
+ Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE)
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+--]]
+
+-- round-robin cache over LRUCache, used within one worker processes
+local _M = {}
+_M._VERSION = '1.0.0'
+
+local lrucache = require "resty.lrucache"
+
+-- we need to initialize the cache on the lua module level so that
+-- it can be shared by all the requests served by each nginx worker process:
+local rrcache,err = lrucache.new(500) -- allow up to 200 items in the cache
+if not rrcache then
+ return ngx.log(ngx.ERR,"failed to create the cache: " .. (err or "unknown"))
+end
+
+function _M.get(key)
+ return rrcache:get(key)
+end
+
+function _M.set(key,value)
+ rrcache:set(key, value)
+end
+
+return _M
\ No newline at end of file diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/lib/utils/error_handler.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/lib/utils/error_handler.lua new file mode 100644 index 0000000..784eb86 --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/lib/utils/error_handler.lua @@ -0,0 +1,48 @@ +--[[
+
+ Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE)
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+--]]
+
+local _M = {}
+_M._VERSION = '1.0.0'
+local msbConf = require('conf.msbinit')
+local enablefullsearch = msbConf.systemConf.enablefullsearch
+local ngx_var = ngx.var
+local error_page_head = '<html><head><title>502 Bad Gateway</title></head><body bgcolor="white"><center><h1>502 Bad Gateway</h1></center><center>error message:'
+local error_page_foot = '</center><hr><center>nginx</center></body></html>'
+local upstream_not_found_err = "service info is incorrect:using own upstream flag is on but upstream name is empty"
+
+function _M.svc_not_found(err_info,detail_info)
+ ngx.log(ngx.WARN, ngx.var.request_id.." "..(err_info or "").." detail_info:"..(detail_info or ""))
+ if enablefullsearch and ngx_var.svc_type ~= "custom" then
+ -- test against the custom services after the commonrewrite phase
+ --ngx.status = ngx.HTTP_GONE
+ return ngx.exec("@commonnotfound");
+ else
+ ngx.status = ngx.HTTP_BAD_GATEWAY
+ ngx.print(error_page_head..err_info..error_page_foot)
+ end
+ return ngx.exit(ngx.status)
+end
+
+function _M.upstream_not_found()
+ ngx.log(ngx.WARN, ngx.var.request_id.." "..upstream_not_found_err)
+ ngx.status = ngx.HTTP_BAD_GATEWAY
+ ngx.print(error_page_head..upstream_not_found_err..error_page_foot)
+ return ngx.exit(ngx.status)
+end
+
+return _M
\ No newline at end of file diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/lib/utils/log_util.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/lib/utils/log_util.lua new file mode 100644 index 0000000..67b0e26 --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/lib/utils/log_util.lua @@ -0,0 +1,28 @@ +--[[
+
+ Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE)
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+--]]
+
+local _M = {}
+_M._VERSION = '1.0.0'
+
+function _M.log(k, v)
+ --if empty,initialize it
+ if not ngx.ctx.logtbl then ngx.ctx.logtbl = {} end
+ ngx.ctx.logtbl[k] = v
+end
+
+return _M
\ No newline at end of file diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/lib/utils/str_util.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/lib/utils/str_util.lua new file mode 100644 index 0000000..6fa8d39 --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/lib/utils/str_util.lua @@ -0,0 +1,30 @@ +--[[
+
+ Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE)
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+--]]
+
+local _M = {}
+_M._VERSION = '1.0.0'
+
+function _M.mark_empty_as_nil(t)
+ if t == "" then
+ return nil
+ else
+ return t
+ end
+end
+
+return _M
\ No newline at end of file diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/lib/utils/svc_util.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/lib/utils/svc_util.lua new file mode 100644 index 0000000..226e31f --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/lib/utils/svc_util.lua @@ -0,0 +1,96 @@ +--[[
+
+ Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE)
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+--]]
+
+local _M = {}
+_M._VERSION = '1.0.0'
+
+local msbConf= require('conf.msbinit')
+local svcConf = require('conf.svcconf')
+local log_util = require('lib.utils.log_util')
+
+local log = log_util.log
+local ngx_var = ngx.var
+
+local defaultport = msbConf.systemConf.defaultport
+local defaulthttpsport = msbConf.systemConf.defaulthttpsport
+local defaultprefix = msbConf.systemConf.defaultprefix
+local router_subdomain = msbConf.routerConf.subdomain
+local router_defaultprefix = msbConf.routerConf.defaultprefix
+local useconsultemplate = msbConf.systemConf.useconsultemplate
+local urlfieldMap = svcConf.urlfieldMap
+local apiRelatedTypes = svcConf.apiRelatedTypes
+
+function _M.isactive(svcinfo)
+ if svcinfo["status"] == "1" then
+ return true
+ else
+ return false
+ end
+end
+
+function _M.use_own_upstream(svcinfo)
+ if useconsultemplate and svcinfo.spec.useOwnUpstream == "1" then
+ log("useOwnUpstream",true)
+ return true
+ else
+ return false
+ end
+end
+
+function _M.get_url(svcinfo,svc_type)
+ return svcinfo.spec[urlfieldMap[svc_type]]
+end
+
+function _M.get_backend_protocol(svcinfo)
+ local svc_enable_ssl = svcinfo.spec["enable_ssl"]
+ if svc_enable_ssl then
+ return "https"
+ else
+ return "http"
+ end
+end
+
+function _M.get_key_prefix()
+ --now assemble the key prefix according the svc_name and server_port
+ local key_prefix = ""
+ local server_port = ngx_var.server_port
+ local svc_name = ngx_var.svc_name
+ if (svc_name == "microservices" or svc_name == "msdiscover") then
+ key_prefix = defaultprefix
+ elseif (server_port == defaultport or server_port == defaulthttpsport) then
+ local m, err = ngx.re.match(ngx_var.host, "(?<hostname>.+)\\."..router_subdomain,"o")
+ if m then
+ key_prefix = router_defaultprefix..":"..m["hostname"]
+ else
+ key_prefix = defaultprefix
+ end
+ else
+ key_prefix = "msb:"..server_port
+ end
+ return key_prefix
+end
+
+function _M.is_api_related_types(svc_type)
+ if(apiRelatedTypes[svc_type]) then
+ return true
+ else
+ return false
+ end
+end
+
+return _M
\ No newline at end of file diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/lib/utils/table_util.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/lib/utils/table_util.lua new file mode 100644 index 0000000..c1ddbfe --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/lib/utils/table_util.lua @@ -0,0 +1,30 @@ +--[[
+
+ Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE)
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+--]]
+
+local _M = {}
+_M._VERSION = '1.0.0'
+
+function _M.isempty(t)
+ if t == nil or next(t) == nil then
+ return true
+ else
+ return false
+ end
+end
+
+return _M
\ No newline at end of file diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/loadbalance/balancer.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/loadbalance/balancer.lua new file mode 100644 index 0000000..48dc1d8 --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/loadbalance/balancer.lua @@ -0,0 +1,50 @@ +--[[
+
+ Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE)
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+--]]
+
+local b = require "ngx.balancer"
+local baseupstream = require "loadbalance.baseupstream"
+local log_util = require('lib.utils.log_util')
+local error_handler = require('lib.utils.error_handler')
+
+local log = log_util.log
+local ngx_ctx = ngx.ctx
+local servers = ngx_ctx.backservers
+local svc_key = ngx_ctx.svc_key
+local error_svc_not_found = error_handler.svc_not_found
+
+local status = b.get_last_failure()
+if status == nil then
+ --only reset the server status table of the service in the first attempt
+ baseupstream.check_and_reset_srv_status_ifneed(svc_key,servers)
+elseif status == "failed" then
+ local last_peer = ngx.ctx.last_peer
+ --mark the srv failed one time
+ baseupstream.mark_srv_failed(svc_key,last_peer)
+end
+
+local server,err = baseupstream.get_backserver(svc_key,servers)
+if server == nil then
+ ngx.log(ngx.WARN, ngx.var.request_id.." ".."No active backend server found! detail_info: key--"..svc_key.." "..(err or ""))
+ return
+end
+if baseupstream.can_retry(svc_key,servers) then
+ b.set_more_tries(1)
+end
+b.set_current_peer(server["ip"],server["port"])
+--log("upstreamserver",server["ip"]..":"..server["port"])
+ngx.ctx.last_peer = { ip=server["ip"], port=server["port"] }
\ No newline at end of file diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/loadbalance/baseupstream.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/loadbalance/baseupstream.lua new file mode 100644 index 0000000..4af6dfa --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/loadbalance/baseupstream.lua @@ -0,0 +1,66 @@ +--[[
+
+ Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE)
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+--]]
+
+local _M = {
+ _VERSION = '1.0.0'
+}
+local policymodule = require "loadbalance.policy.roundrobin"
+local tbl_util = require('lib.utils.table_util')
+local peerwatcher = require "loadbalance.peerwatcher"
+local tbl_isempty = tbl_util.isempty
+
+function _M.get_backserver(svc_key,servers)
+ if tbl_isempty(servers) then return nil,"server list is empty" end
+
+ local servers_num = #servers
+ if not ngx.ctx.tried_num then
+ ngx.ctx.tried_num = 0
+ end
+ local server
+ if servers_num==1 then
+ ngx.ctx.tried_num = ngx.ctx.tried_num+1
+ -- return it directly if there is only one server
+ server = servers[1]
+ if peerwatcher.is_server_ok(svc_key,server) then
+ return server,""
+ else
+ return nil,"only one server but is not available"
+ end
+ end
+ for i=ngx.ctx.tried_num+1,servers_num do
+ ngx.ctx.tried_num = ngx.ctx.tried_num+1
+ server = policymodule.select_backserver(servers,svc_key)
+ if peerwatcher.is_server_ok(svc_key,server) then
+ return server,""
+ end
+ end
+ return nil,"serveral server but no one is available"
+end
+
+function _M.can_retry(svc_key,servers)
+ return ngx.ctx.tried_num < #servers
+end
+
+function _M.mark_srv_failed(svc_key, srv)
+ peerwatcher.set_srv_status(svc_key, srv, true)
+end
+
+function _M.check_and_reset_srv_status_ifneed(svc_key, servers)
+ peerwatcher.check_and_reset_srv_status_ifneed(svc_key,servers)
+end
+return _M
\ No newline at end of file diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/loadbalance/peerwatcher.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/loadbalance/peerwatcher.lua new file mode 100644 index 0000000..6b7e522 --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/loadbalance/peerwatcher.lua @@ -0,0 +1,107 @@ +--[[
+
+ Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE)
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+--]]
+
+local _M = {
+ _VERSION = '1.0.0',
+ STATUS_OK = 0, STATUS_UNSTABLE = 1, STATUS_ERR = 2
+}
+local msbConf = require('conf.msbinit')
+local str_format = string.format
+local now = ngx.now
+local fail_timeout = msbConf.server.fail_timeout or 10
+local max_fails = msbConf.server.max_fails or 1
+
+local cluster_status = {}
+_M.cluster_status = cluster_status
+
+function _M.is_server_ok(skey, srv)
+ return _M.get_srv_status(skey, srv)==_M.STATUS_OK
+end
+
+function _M.get_srv_status(skey, srv)
+ local server_status = cluster_status[skey]
+ if not server_status then
+ return _M.STATUS_OK
+ end
+
+ local srv_key = str_format("%s:%d", srv.ip, srv.port)
+ local srv_status = server_status[srv_key]
+
+ if srv_status and srv_status.lastmodify + fail_timeout > now() then
+ return srv_status.status
+ end
+
+ return _M.STATUS_OK
+end
+
+function _M.set_srv_status(skey, srv, failed)
+ local server_status = cluster_status[skey]
+ if not server_status then
+ server_status = {}
+ cluster_status[skey] = server_status
+ end
+
+ local time_now = now()
+ local srv_key = str_format("%s:%d", srv.ip, srv.port)
+ local srv_status = server_status[srv_key]
+ if not srv_status then -- first set
+ srv_status = {
+ status = _M.STATUS_OK,
+ failed_count = 0,
+ lastmodify = time_now
+ }
+ server_status[srv_key] = srv_status
+ elseif srv_status.lastmodify + fail_timeout < time_now then -- srv_status expired
+ srv_status.status = _M.STATUS_OK
+ srv_status.failed_count = 0
+ srv_status.lastmodify = time_now
+ end
+
+ if failed then
+ srv_status.failed_count = srv_status.failed_count + 1
+ if srv_status.failed_count >= max_fails then
+ srv_status.status = _M.STATUS_ERR
+ end
+ end
+end
+
+function _M.check_and_reset_srv_status_ifneed(skey,servers)
+ local server_status = cluster_status[skey]
+ --if disabled servers of the service is empty,do nothing
+ if not server_status then
+ ngx.log(ngx.DEBUG, "service:",skey," server_status is nil")
+ return
+ end
+ local need_reset = true
+ for _, srv in ipairs(servers) do
+ local srv_key = str_format("%s:%d", srv.ip, srv.port)
+ local srv_status = server_status[srv_key]
+ if not (srv_status and srv_status.status == _M.STATUS_ERR and srv_status.lastmodify + fail_timeout > now()) then
+ --once find the server is not disabled now, no need to reset the status table. break the loop
+ ngx.log(ngx.DEBUG, "service:",skey," donot need reset,break the loop")
+ need_reset = false
+ break
+ end
+ end
+ if need_reset then
+ ngx.log(ngx.DEBUG, "service:",skey," need reset")
+ cluster_status[skey] = {}
+ end
+end
+
+return _M
\ No newline at end of file diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/loadbalance/policy/roundrobin.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/loadbalance/policy/roundrobin.lua new file mode 100644 index 0000000..0a1b8e8 --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/loadbalance/policy/roundrobin.lua @@ -0,0 +1,68 @@ +--[[
+
+ Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE)
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+--]]
+
+local _M = {}
+_M._VERSION = '1.0.0'
+
+local tbl_util = require('lib.utils.table_util')
+local tbl_isempty = tbl_util.isempty
+local rrcache = require('lib.tools.rr_cache')
+
+function _M.select_backserver(servers,svckey)
+ local length = #servers
+ local index = ngx.ctx.last_peer_index
+
+ if index then
+ --if it is a retry request,use the index in the context as the base
+ index = index%length+1
+ else
+ --if it is a normal request,fetch index from cache as the base
+ index = rrcache.get(svckey) or 0
+ index = index%length+1
+ rrcache.set(svckey,index)
+ end
+ ngx.ctx.last_peer_index = index
+
+ --[[
+ local resty_lock = require "resty.lock"
+ local roundrobin_cache = ngx.shared.rr_cache
+
+ --step1:acquire lock
+ local opts = {["timeout"] = 0.002,["exptime"] = 0.05}--this can be set using the conf file
+ local rrlock = resty_lock:new("rr_locks",opts)
+ local elapsed, err = rrlock:lock(svckey)
+ if not elapsed then
+ --return fail("failed to acquire the lock: ", err)
+ end
+ --step2:lock successfully acquired!incr the index
+ local index, err = roundrobin_cache:get(svckey)
+ if not index then
+ index = 0
+ end
+ index = index%length+1
+
+ --step3:update the shm cache with the new index
+ roundrobin_cache:set(svckey,index)
+
+ --step4:release the lock
+ local ok, err = rrlock:unlock()
+ ]]
+ return servers[index],nil
+end
+
+return _M
\ No newline at end of file diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/log/logger.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/log/logger.lua new file mode 100644 index 0000000..6d1e8b5 --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/log/logger.lua @@ -0,0 +1,24 @@ +--[[
+
+ Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE)
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+--]]
+
+local cjson = require "cjson"
+
+if ngx.ctx.logtbl then
+ local jsonData = cjson.encode(ngx.ctx.logtbl)
+ ngx.log(ngx.INFO, ngx.var.request_id.." "..string.gsub(jsonData,"\\/","/"))
+end
\ No newline at end of file diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/msb.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/msb.lua new file mode 100644 index 0000000..f791bdb --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/msb.lua @@ -0,0 +1,90 @@ +--[[
+
+ Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE)
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+--]]
+local _M = {
+}
+_M._VERSION = '1.0.0'
+_M._DESCRIPTION = 'msb plugins controller'
+
+local default_conf = require('plugins.config_default')
+local custom_conf = require('plugins.config_custom')
+local table_insert = table.insert
+local string_find = string.find
+local str_low = string.lower
+
+--- Borrowed from Kong
+--- Try to load a module.
+-- Will not throw an error if the module was not found, but will throw an error if the
+-- loading failed for another reason (eg: syntax error).
+-- @param module_name Path of the module to load (ex: kong.plugins.keyauth.api).
+-- @return success A boolean indicating wether the module was found.
+-- @return module The retrieved module.
+local load_module_if_exists = function(module_name)
+ local status, res = pcall(require, module_name)
+ if status then
+ return true, res
+ -- Here we match any character because if a module has a dash '-' in its name, we would need to escape it.
+ elseif type(res) == "string" and string_find(res, "module '"..module_name.."' not found", nil, true) then
+ return false
+ else
+ error(res)
+ end
+end
+
+
+function _M.load_plugins()
+ local pluginnames = {}
+ for _, plugin in ipairs(default_conf.plugins_default) do
+ if(str_low(plugin.status) =="on") then
+ table_insert(pluginnames,plugin.name)
+ end
+ end
+ for _, plugin in ipairs(custom_conf.plugins_custom) do
+ if(str_low(plugin.status) =="on") then
+ table_insert(pluginnames,plugin.name)
+ end
+ end
+ local plugins = {}
+ for _, v in ipairs(pluginnames) do
+ local loaded, plugin_handler_mod = load_module_if_exists("plugins."..v..".handler")
+ if not loaded then
+ error("The following plugin has been enabled in the configuration but it is not installed on the system: "..v)
+ end
+ ngx.log(ngx.DEBUG, "Loading plugin: "..v)
+ table_insert(plugins, {
+ name = v,
+ handler = plugin_handler_mod()
+ })
+ end
+ package.loaded.plugins = plugins
+end
+
+function _M.access()
+ local plugins = package.loaded.plugins
+ for _, plugin in ipairs(plugins) do
+ plugin.handler:access()
+ end
+end
+
+function _M.header_filter()
+ local plugins = package.loaded.plugins
+ for _, plugin in ipairs(plugins) do
+ plugin.handler:header_filter()
+ end
+end
+
+return _M
\ No newline at end of file diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/plugins/base_plugin.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/plugins/base_plugin.lua new file mode 100644 index 0000000..5ad9c78 --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/plugins/base_plugin.lua @@ -0,0 +1,35 @@ +--[[ + + Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE) + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--]] + +local Object = require "vendor.classic" +local BasePlugin = Object:extend() + +function BasePlugin:new(name) + self._name = name + ngx.log(ngx.DEBUG, "executing plugin \""..self._name.."\": new") +end + +function BasePlugin:access() + ngx.log(ngx.DEBUG, "executing plugin \""..self._name.."\": access") +end + +function BasePlugin:header_filter() + ngx.log(ngx.DEBUG, " executing plugin \""..self._name.."\": header_filter") +end + +return BasePlugin
\ No newline at end of file diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/plugins/config_custom.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/plugins/config_custom.lua new file mode 100644 index 0000000..bc84ebc --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/plugins/config_custom.lua @@ -0,0 +1,27 @@ +--[[ + + Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE) + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--]] + +local _M = {} +_M._VERSION = '1.0.0' +_M._DESCRIPTION = 'config_custom' + +_M.plugins_custom = { + {["name"] = "sampleplugin",["status"] = "off"} +} + +return _M
\ No newline at end of file diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/plugins/config_default.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/plugins/config_default.lua new file mode 100644 index 0000000..cbb3107 --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/plugins/config_default.lua @@ -0,0 +1,27 @@ +--[[ + + Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE) + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--]] +local _M = {} +_M._VERSION = '1.0.0' +_M._DESCRIPTION = 'config_default' + + +_M.plugins_default = { + {["name"] = "redirect-transformer",["status"] = "on"} +} + +return _M
\ No newline at end of file diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/plugins/redirect-transformer/handler.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/plugins/redirect-transformer/handler.lua new file mode 100644 index 0000000..72e3330 --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/plugins/redirect-transformer/handler.lua @@ -0,0 +1,42 @@ +--[[ + + Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE) + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--]] + +local BasePlugin = require "plugins.base_plugin" +local msbConf = require('conf.msbinit') +local log_util = require('lib.utils.log_util') +local log = log_util.log + +local RedirectTransformerPluginHandler = BasePlugin:extend() + +function RedirectTransformerPluginHandler:new() + RedirectTransformerPluginHandler.super.new(self, "redirect-transformer-plugin") +end + +function RedirectTransformerPluginHandler:header_filter() + RedirectTransformerPluginHandler.super.header_filter(self) + local originloc = ngx.header.Location + if(originloc) then + local newloc = ngx.re.sub(originloc, "^(https|http)(.*)", ngx.var.scheme.."$2", "oi") + ngx.header["Location"] = newloc + log("origin Location:",originloc) + log("req scheme:",ngx.var.scheme) + log("new Location:",newloc) + end +end + +return RedirectTransformerPluginHandler
\ No newline at end of file diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/plugins/sampleplugin/handler.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/plugins/sampleplugin/handler.lua new file mode 100644 index 0000000..539053a --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/plugins/sampleplugin/handler.lua @@ -0,0 +1,50 @@ +--[[ + + Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE) + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--]] + +local BasePlugin = require "plugins.base_plugin" + +local SamplePluginHandler = BasePlugin:extend() + +function SamplePluginHandler:new() + SamplePluginHandler.super.new(self, "sampleplugin") +end + +function SamplePluginHandler:access() + SamplePluginHandler.super.access(self) + --[[more about the use of APIs please refer to github doc + https://github.com/openresty/lua-nginx-module + ]] + --validate and rewrite + if(ngx.req.get_method() == "GET") then + ngx.req.set_uri("/sayhello") + ngx.var.backend = "127.0.0.1:10089" + else + ngx.status = ngx.HTTP_NOT_ALLOWED + ngx.exit(ngx.status) + end + --access + --[[ + local client_ip = ngx.var.remote_addr + if(client_ip ~= "127.0.0.1") then + ngx.status = ngx.HTTP_FORBIDDEN + ngx.exit(ngx.status) + end + ]] +end + +return SamplePluginHandler
\ No newline at end of file diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/rewrite/commonrewrite.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/rewrite/commonrewrite.lua new file mode 100644 index 0000000..6bea893 --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/rewrite/commonrewrite.lua @@ -0,0 +1,138 @@ +--[[
+
+ Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE)
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+--]]
+
+local dbclient = require('dao.db_access')
+local tbl_util = require('lib.utils.table_util')
+local svc_util = require('lib.utils.svc_util')
+local log_util = require('lib.utils.log_util')
+local error_handler = require('lib.utils.error_handler')
+
+local tbl_concat = table.concat
+local tbl_isempty = tbl_util.isempty
+local svc_isactive = svc_util.isactive
+local svc_get_url = svc_util.get_url
+local svc_get_backend_protocol = svc_util.get_backend_protocol
+local svc_use_own_upstream = svc_util.use_own_upstream
+local svc_get_key_prefix = svc_util.get_key_prefix
+local svc_is_api_related_types = svc_util.is_api_related_types
+local error_svc_not_found = error_handler.svc_not_found
+local error_upstream_not_found = error_handler.upstream_not_found
+
+local str_sub = string.sub
+local str_len = string.len
+local str_low = string.lower
+local ngx_var = ngx.var
+local ngx_ctx = ngx.ctx
+local log = log_util.log
+
+---------------------------------------------------------------
+--preCheck:
+-- determine whether it is websocket request
+-- and do internal redirect
+---------------------------------------------------------------
+local http_upgrade = ngx_var.http_upgrade
+if(ngx_var.websocket_internal_redirect == "on") and http_upgrade and str_low(http_upgrade) == "websocket" then
+ --ngx.log(ngx.ERR, "Websocket request and redirect to @commonwebsocket")
+ return ngx.exec("@commonwebsocket");
+end
+
+---------------------------------------------------------------
+--step0:Preparation
+-- svc_info_key
+-- svc_server_keypattern
+---------------------------------------------------------------
+local svc_name = ngx_var.svc_name
+local req_res = ngx_var.req_res
+local svc_type = ngx_var.svc_type
+
+local key_prefix = svc_get_key_prefix()
+
+local svc_key = ""
+if(svc_is_api_related_types(svc_type)) then
+ -- process version info first
+ local version1 = ngx_var.svc_version1
+ local version2 = ngx_var.svc_version2
+ local version = ""
+ -- check version info appearing befor or after
+ if(not version2) then version2 = "" end --convert nil to empty sting avoiding throw error
+ if(not version1 or version1 == "") then
+ version = version2
+ else
+ version = version1
+ req_res = version2..req_res
+ end
+ -- remove the slash in front of the version (e.g. /V1.0)
+ local svc_version=str_sub(version,2,str_len(version))
+ svc_key = tbl_concat({key_prefix,"api",svc_name,svc_version},":")
+else
+ svc_key = tbl_concat({key_prefix,"iui",svc_name},":")
+end
+
+---------------------------------------------------------------
+--step1:query the service info from share memory or back db
+-- svcinfo: the requested service information
+---------------------------------------------------------------
+local svcinfo = dbclient.load_serviceinfo(svc_key)
+if tbl_isempty(svcinfo) then
+ error_svc_not_found("No route found for this request!","common not match. key--"..svc_key)
+end
+if not svc_isactive(svcinfo) then
+ error_svc_not_found("Service is disabled!","common not match. key--"..svc_key)
+end
+
+local svc_url = svc_get_url(svcinfo,svc_type)
+
+---------------------------------------------------------------
+--step2:rewrite the request uri using the svcinfo
+---------------------------------------------------------------
+local rewrited_uri = svc_url..req_res
+--special handling: avoid throws internal error when it is empty
+--if (rewrited_uri == "") then rewrited_uri = "/" end
+if (rewrited_uri == "") then return ngx.redirect(ngx.var.uri.."/") end
+ngx.req.set_uri(rewrited_uri)
+
+log("matched",svc_name)
+--log("rewrited_uri",rewrited_uri)
+---------------------------------------------------------------
+--step2.1:store the svcinfo in the context, plugins may use it
+---------------------------------------------------------------
+ngx_ctx.svcinfo = svcinfo
+
+---------------------------------------------------------------
+--step3:process the proxy upstream part
+-- con1-using consul template:set the upstream name
+-- con2-using msb balancer:query the server list and store in the ctx
+---------------------------------------------------------------
+--set the http_protocol used by proxy_pass directive
+ngx_var.http_protocol = svc_get_backend_protocol(svcinfo)
+
+if svc_use_own_upstream(svcinfo) then
+ local consul_servicename = svcinfo.spec["consulServiceName"]
+ if not consul_servicename or consul_servicename == "" then
+ error_upstream_not_found()
+ end
+ ngx_var.backend = consul_servicename
+ ngx.ctx.use_ownupstream = true
+else
+ local backservers = svcinfo.spec.nodes
+ if tbl_isempty(backservers) then
+ error_svc_not_found("No active backend server found!"," key--"..svc_key)
+ end
+ ngx_ctx.backservers = backservers
+ ngx_ctx.svc_key = svc_key
+end
\ No newline at end of file diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/rewrite/customrewrite.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/rewrite/customrewrite.lua new file mode 100644 index 0000000..89a829d --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/rewrite/customrewrite.lua @@ -0,0 +1,203 @@ +--[[
+
+ Copyright (C) 2016 ZTE, Inc. and others. All rights reserved. (ZTE)
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+--]]
+
+local msbConf = require('conf.msbinit')
+local dbclient = require('dao.db_access')
+local tbl_util = require('lib.utils.table_util')
+local svc_util = require('lib.utils.svc_util')
+local log_util = require('lib.utils.log_util')
+local error_handler = require('lib.utils.error_handler')
+
+local tbl_concat = table.concat
+local tbl_isempty = tbl_util.isempty
+local svc_isactive = svc_util.isactive
+local svc_get_url = svc_util.get_url
+local svc_get_backend_protocol = svc_util.get_backend_protocol
+local svc_use_own_upstream = svc_util.use_own_upstream
+local svc_get_key_prefix = svc_util.get_key_prefix
+local str_low = string.lower
+local ngx_var = ngx.var
+local ngx_ctx = ngx.ctx
+local log = log_util.log
+local error_svc_not_found = error_handler.svc_not_found
+local error_upstream_not_found = error_handler.upstream_not_found
+
+local enablerefercheck = msbConf.systemConf.enablerefercheck
+local useconsultemplate = msbConf.systemConf.useconsultemplate
+
+---------------------------------------------------------------
+--preCheck:
+-- determine whether it is websocket request
+-- and do internal redirect
+---------------------------------------------------------------
+if ngx.var.uri == "/" then
+ local m, err = ngx.re.match(ngx.var.host, "^((25[0-5]|2[0-4]\\d|[01]?\\d\\d?)\\.){3}(25[0-5]|2[0-4]\\d|[01]?\\d\\d?)$","o")
+ if m then
+ return ngx.exec("@defaultpage")
+ end
+end
+
+local http_upgrade = ngx_var.http_upgrade
+if(ngx_var.websocket_internal_redirect == "on") and http_upgrade and str_low(http_upgrade)== "websocket" then
+ --ngx.log(ngx.ERR, "Websocket request and redirect to @customwebsocket")
+ return ngx.exec("@customwebsocket")
+end
+
+---------------------------------------------------------------
+--step0:Preparation
+-- svcnames:service names registered under this port
+---------------------------------------------------------------
+local req_res = ngx_var.uri
+local svc_type = ngx_var.svc_type
+local key_prefix = svc_get_key_prefix()
+
+local custom_svc_keypattern = tbl_concat({key_prefix,"custom","*"},":")
+
+local get_svckey_custom = function(svcname)
+ return tbl_concat({key_prefix,"custom",svcname},":")
+end
+
+local svcnames,err = dbclient.load_customsvcnames(custom_svc_keypattern)
+if not svcnames then
+ error_svc_not_found("Failed to load the route table!","keypattern--"..custom_svc_keypattern)
+end
+
+---------------------------------------------------------------
+--step1:run the match process(check whether the request
+-- match the name in the svcnames one by one)
+-- and return the matched serice info
+---------------------------------------------------------------
+local matchedsvcname
+local svcinfo
+local svc_key = ""
+--add by wangyg:20160418 special handler for refer
+local matched_usingrefer = false
+--end of add by wangyg:20160418 special handler for refer
+for _, svcname in ipairs(svcnames) do
+ if (svcname == "/") then
+ svc_key = get_svckey_custom(svcname)
+ local svc_info,err = dbclient.load_serviceinfo(svc_key)
+ if svc_info and svc_isactive(svc_info)then
+ matchedsvcname = svcname
+ svcinfo = svc_info
+ break
+ end
+ end
+ local from, to, err = ngx.re.find(req_res, "^"..svcname.."(/(.*))?$", "jo")
+ --check whether svcname is the prefix of the req uri
+ if from then
+ svc_key = get_svckey_custom(svcname)
+ local svc_info,err = dbclient.load_serviceinfo(svc_key)
+ if svc_info and svc_isactive(svc_info) then
+ matchedsvcname = svcname
+ svcinfo = svc_info
+ break
+ end
+ else
+ --do nothing
+ end
+end
+---------------------------------------------------------------
+--step1.1:additional process,test against the refer
+-- similar to step1
+---------------------------------------------------------------
+--add by wangyg:20160418 special handler for refer
+if not matchedsvcname and enablerefercheck then
+ local refer = ngx_var.http_referer
+ if(refer and refer~="") then
+ for _, svcname in ipairs(svcnames) do
+ local urlreg ="^(https://|http://|)(([1-9]|([1-9]\\d)|(1\\d\\d)|(2([0-4]\\d|5[0-5])))\\.)(([0-9]|([1-9]\\d)|(1\\d\\d)|(2([0-4]\\d|5[0-5])))\\.){2}([1-9]|([1-9]\\d)|(1\\d\\d)|(2([0-4]\\d|5[0-5])))(:\\d{1,5})?"..svcname.."(/(.*))?$";
+ local from, to, err = ngx.re.find(refer, urlreg, "jo")
+ ----check whether svcname is the prefix of the req refer
+ if from then
+ svc_key = get_svckey_custom(svcname)
+ local svc_info,err = dbclient.load_serviceinfo(svc_key)
+ if svc_info and svc_isactive(svc_info) then
+ matchedsvcname = svcname
+ svcinfo = svc_info
+ matched_usingrefer = true
+ break
+ end
+ end
+ end
+ end
+end
+--end of add by wangyg:20160418 special handler for refer
+
+if not matchedsvcname or tbl_isempty(svcinfo) then
+ error_svc_not_found("No route found for this request!","custom not match")
+end
+
+local svc_url = svc_get_url(svcinfo,svc_type)
+
+---------------------------------------------------------------
+--step2:rewrite the request uri using the svcinfo
+---------------------------------------------------------------
+local rewrited_uri =""
+if (matchedsvcname == "/") then
+ --special handling: if "/" matched, contact directly
+ rewrited_uri = svc_url..req_res
+else
+ --rewrited_uri = ngx.re.sub(req_res, "^"..matchedsvcname.."(.*)", svc_url.."$1", "o")
+ local newuri,n,err = ngx.re.sub(req_res, "^"..matchedsvcname.."(/.*)?", svc_url.."$1", "o")
+ --add by wangyg:20160418 special handler for refer
+ if(n==0 and matched_usingrefer) then newuri = svc_url..req_res end --special handling if matched using refer
+ --end of add by wangyg:20160418 special handler for refer
+ rewrited_uri = newuri
+end
+--if (rewrited_uri == "") then rewrited_uri = "/" end --avoid throws internal error when it is empty
+if (rewrited_uri == "") then return ngx.redirect(ngx.var.uri.."/") end
+ngx.req.set_uri(rewrited_uri)
+
+--set the matched service info,used in the proxy_redirect directive
+ngx_var.svc_name = matchedsvcname
+ngx_var.svc_url = svc_url
+
+--log the route info
+log("matched",matchedsvcname)
+if(matched_usingrefer) then log("matched_usingrefer",true) end
+--log("rewrited_uri",rewrited_uri)
+
+---------------------------------------------------------------
+--step2.1:store the svcinfo in the context, plugins may use it
+---------------------------------------------------------------
+ngx_ctx.svcinfo = svcinfo
+
+---------------------------------------------------------------
+--step3:process the proxy upstream part
+-- con1-using consul template:set the upstream name
+-- con2-using msb balancer:query the backserver list and store in the ctx
+---------------------------------------------------------------
+--set the http_protocol used by proxy_pass directive
+ngx_var.http_protocol = svc_get_backend_protocol(svcinfo)
+
+if svc_use_own_upstream(svcinfo) then
+ local consul_servicename = svcinfo.spec["consulServiceName"]
+ if not consul_servicename or consul_servicename == "" then
+ error_upstream_not_found()
+ end
+ ngx_var.backend = consul_servicename
+ ngx.ctx.use_ownupstream = true
+else
+ local backservers = svcinfo.spec.nodes
+ if tbl_isempty(backservers) then
+ error_svc_not_found("No active backend server found!","key--"..svc_key)
+ end
+ ngx_ctx.backservers = backservers
+ ngx_ctx.svc_key = svc_key
+end
\ No newline at end of file diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/classic.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/classic.lua new file mode 100644 index 0000000..d9402f9 --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/classic.lua @@ -0,0 +1,68 @@ +-- +-- classic, object model. +-- +-- Copyright (c) 2014, rxi +-- +-- This module is free software; you can redistribute it and/or modify it under +-- the terms of the MIT license. See LICENSE for details. +-- +-- Base object model used with Kong, see [classic github repo](https://github.com/rxi/classic) for usage information + +local Object = {} +Object.__index = Object + + +function Object:new() +end + + +function Object:extend() + local cls = {} + for k, v in pairs(self) do + if k:find("__") == 1 then + cls[k] = v + end + end + cls.__index = cls + cls.super = self + setmetatable(cls, self) + return cls +end + + +function Object:implement(...) + for _, cls in pairs({...}) do + for k, v in pairs(cls) do + if self[k] == nil and type(v) == "function" then + self[k] = v + end + end + end +end + + +function Object:is(T) + local mt = getmetatable(self) + while mt do + if mt == T then + return true + end + mt = getmetatable(mt) + end + return false +end + + +function Object:__tostring() + return "Object" +end + + +function Object:__call(...) + local obj = setmetatable({}, self) + obj:new(...) + return obj +end + + +return Object diff --git a/openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/shcache.lua b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/shcache.lua new file mode 100644 index 0000000..32a9b6c --- /dev/null +++ b/openresty-ext/src/assembly/resources/openresty/nginx/luaext/vendor/shcache.lua @@ -0,0 +1,440 @@ +-- Copyright (C) 2013 Matthieu Tourne +-- @author Matthieu Tourne <matthieu@cloudflare.com> + +-- small overlay over shdict, smart cache load mechanism + +local M = {} + +local resty_lock = require("resty.lock") + +local DEBUG = false + +-- defaults in secs +local DEFAULT_POSITIVE_TTL = 10 -- cache for, successful lookup +local DEFAULT_NEGATIVE_TTL = 2 -- cache for, failed lookup +local DEFAULT_ACTUALIZE_TTL = 2 -- stale data, actualize data for + +-- default lock options, in secs +local function _get_default_lock_options() + return { + exptime = 1, -- max wait if failing to call unlock() + timeout = 0.5, -- max waiting time of lock() + max_step = 0.1, -- max sleeping interval + } +end + +local function prequire(m) + local ok, err_or_module = pcall(require, m) + if not ok then + return nil, err_or_module + end + return err_or_module +end + +local conf = prequire("conf") +if conf then + DEFAULT_NEGATIVE_TTL = conf.DEFAULT_NEGATIVE_TTL or DEFAULT_NEGATIVE_TTL + DEFAULT_ACTUALIZE_TTL = conf.DEFAULT_ACTUALIZE_TTL or DEFAULT_ACTUALIZE_TTL +end + +local band = bit.band +local bor = bit.bor +local st_format = string.format + +-- there are only really 5 states total + -- is_stale is_neg is_from_cache +local MISS_STATE = 0 -- 0 0 0 +local HIT_POSITIVE_STATE = 1 -- 0 0 1 +local HIT_NEGATIVE_STATE = 3 -- 0 1 1 +local STALE_POSITIVE_STATE = 5 -- 1 0 1 + +-- stale negative doesn't really make sense, use HIT_NEGATIVE instead +-- local STALE_NEGATIVE_STATE = 7 -- 1 1 1 + +-- xor to set +local NEGATIVE_FLAG = 2 +local STALE_FLAG = 4 + +local STATES = { + [MISS_STATE] = 'MISS', + [HIT_POSITIVE_STATE] = 'HIT', + [HIT_NEGATIVE_STATE] = 'HIT_NEGATIVE', + [STALE_POSITIVE_STATE] = 'STALE', + -- [STALE_NEGATIVE_STATE] = 'STALE_NEGATIVE', +} + +local function get_status(flags) + return STATES[flags] or st_format('UNDEF (0x%x)', flags) +end + +local EMPTY_DATA = '_EMPTY_' + +-- install debug functions +if DEBUG then + local resty_lock_lock = resty_lock.lock + + resty_lock.lock = function (...) + local _, key = unpack({...}) + print("lock key: ", tostring(key)) + return resty_lock_lock(...) + end + + local resty_lock_unlock = resty_lock.unlock + + resty_lock.unlock = function (...) + print("unlock") + return resty_lock_unlock(...) + end +end + + +-- store the object in the context +-- useful for debugging and tracking cache status +local function _store_object(self, name) + if DEBUG then + print('storing shcache: ', name, ' into ngx.ctx') + end + + local ngx_ctx = ngx.ctx + + if not ngx_ctx.shcache then + ngx_ctx.shcache = {} + end + ngx_ctx.shcache[name] = self +end + +local obj_mt = { + __index = M, +} + +-- default function for callbacks.encode / decode. +local function _identity(data) + return data +end + +-- shdict: ngx.shared.DICT, created by the lua_shared_dict directive +-- callbacks: see shcache state machine for user defined functions +-- * callbacks.external_lookup is required +-- * callbacks.encode : optional encoding before saving to shmem +-- * callbacks.decode : optional decoding when retreiving from shmem +-- opts: +-- * opts.positive_ttl : save a valid external loookup for, in seconds +-- * opts.positive_ttl : save a invalid loookup for, in seconds +-- * opts.actualize_ttl : re-actualize a stale record for, in seconds +-- * opts.lock_options : set option to lock see : http://github.com/agentzh/lua-resty-lock +-- for more details. +-- * opts.locks_shdict : specificy the name of the shdict containing the locks +-- (useful if you might have locks key collisions) +-- uses "locks" by default. +-- * opts.name : if shcache object is named, it will automatically +-- register itself in ngx.ctx.shcache (useful for logging). +local function new(self, shdict, callbacks, opts) + if not shdict then + return nil, "shdict does not exist" + end + + -- check that callbacks.external_lookup is set + if not callbacks or not callbacks.external_lookup then + return nil, "no external_lookup function defined" + end + + if not callbacks.encode then + callbacks.encode = _identity + end + + if not callbacks.decode then + callbacks.decode = _identity + end + + local opts = opts or {} + + -- merge default lock options with the ones passed to new() + local lock_options = _get_default_lock_options() + if opts.lock_options then + for k, v in pairs(opts.lock_options) do + lock_options[k] = v + end + end + + local name = opts.name + + local obj = { + shdict = shdict, + callbacks = callbacks, + + positive_ttl = opts.positive_ttl or DEFAULT_POSITIVE_TTL, + negative_ttl = opts.negative_ttl or DEFAULT_NEGATIVE_TTL, + + -- ttl to actualize stale data to + actualize_ttl = opts.actualize_ttl or DEFAULT_ACTUALIZE_TTL, + + lock_options = lock_options, + + locks_shdict = opts.lock_shdict or "locks", + + -- STATUS -- + + from_cache = false, + cache_status = 'UNDEF', + cache_state = MISS_STATE, + lock_status = 'NO_LOCK', + + -- shdict:set() pushed out another value + forcible_set = false, + + -- cache hit on second attempt (post lock) + hit2 = false, + + name = name, + } + + local locks = ngx.shared[obj.locks_shdict] + + -- check for existence, locks is not directly used + if not locks then + ngx.log(ngx.CRIT, 'shared mem locks is missing.\n', + '## add to you lua conf: lua_shared_dict locks 5M; ##') + return nil + end + + local self = setmetatable(obj, obj_mt) + + -- if the shcache object is named + -- keep track of the object in the context + -- (useful for gathering stats at log phase) + if name then + _store_object(self, name) + end + + return self +end +M.new = new + +-- acquire a lock +local function _get_lock(self) + local lock = self.lock + if not lock then + lock = resty_lock:new(self.locks_shdict, self.lock_options) + self.lock = lock + end + return lock +end + +-- remove the lock if there is any +local function _unlock(self) + local lock = self.lock + if lock then + local ok, err = lock:unlock() + if not ok then + ngx.log(ngx.ERR, "failed to unlock :" , err) + end + self.lock = nil + end +end + +local function _return(self, data, flags) + -- make sure we remove the locks if any before returning data + _unlock(self) + + -- set cache status + local cache_status = get_status(self.cache_state) + + if cache_status == 'MISS' and not data then + cache_status = 'NO_DATA' + end + + self.cache_status = cache_status + + return data, self.from_cache +end + +local function _set(self, ...) + if DEBUG then + local key, data, ttl, flags = unpack({...}) + print("saving key: ", key, ", for: ", ttl) + end + + local ok, err, forcible = self.shdict:set(...) + + self.forcible_set = forcible + + if not ok then + local key, data, ttl, flags = unpack({...}) + ngx.log(ngx.ERR, 'failed to set key: ', key, ', err: ', err) + end + + return ok +end + +-- check if the data returned by :get() is considered empty +local function _is_empty(data, flags) + return flags and band(flags, NEGATIVE_FLAG) and data == EMPTY_DATA +end + +-- save positive, encode the data if needed before :set() +local function _save_positive(self, key, data) + if DEBUG then + print("key: ", key, ". save positive, ttl: ", self.positive_ttl) + end + data = self.callbacks.encode(data) + return _set(self, key, data, self.positive_ttl, HIT_POSITIVE_STATE) +end + +-- save negative, no encoding required (no data actually saved) +local function _save_negative(self, key) + if DEBUG then + print("key: ", key, ". save negative, ttl: ", self.negative_ttl) + end + return _set(self, key, EMPTY_DATA, self.negative_ttl, HIT_NEGATIVE_STATE) +end + +-- save actualize, will boost a stale record to a live one +local function _save_actualize(self, key, data, flags) + local new_flags = bor(flags, STALE_FLAG) + + if DEBUG then + print("key: ", key, ". save actualize, ttl: ", self.actualize_ttl, + ". new state: ", get_status(new_flags)) + end + + _set(self, key, data, self.actualize_ttl, new_flags) + return new_flags +end + +local function _process_cached_data(self, data, flags) + if DEBUG then + print("data: ", data, st_format(", flags: %x", flags)) + end + + self.cache_state = flags + self.from_cache = true + + if _is_empty(data, flags) then + -- empty cached data + return nil + else + return self.callbacks.decode(data) + end +end + +-- wrapper to get data from the shdict +local function _get(self, key) + -- always call get_stale() as it does not free element + -- like get does on each call + local data, flags, stale = self.shdict:get_stale(key) + + if data and stale then + if DEBUG then + print("found stale data for key : ", key) + end + + self.stale_data = { data, flags } + + return nil, nil + end + + return data, flags +end + +local function _get_stale(self) + local stale_data = self.stale_data + if stale_data then + return unpack(stale_data) + end + + return nil, nil +end + +local function load(self, key) + -- start: check for existing cache + local data, flags = _get(self, key) + + -- hit: process_cache_hit + if data then + data = _process_cached_data(self, data, flags) + return _return(self, data) + end + + -- miss: set lock + + -- lock: set a lock before performing external lookup + local lock = _get_lock(self) + local elapsed, err = lock:lock(key) + + if not elapsed then + -- failed to acquire lock, still proceed normally to external_lookup + -- unlock() might fail. + ngx.log(ngx.ERR, "failed to acquire the lock: ", err) + self.lock_status = 'ERROR' + -- _unlock won't try to unlock() without a valid lock + self.lock = nil + else + -- lock acquired successfuly + + if elapsed > 0 then + + -- elapsed > 0 => waited lock (other thread might have :set() the data) + -- (more likely to get a HIT on cache_load 2) + self.lock_status = 'WAITED' + + else + + -- elapsed == 0 => immediate lock + -- it is less likely to get a HIT on cache_load 2 + -- but still perform it (race condition cases) + self.lock_status = 'IMMEDIATE' + end + + -- perform cache_load 2 + data, flags = _get(self, key) + if data then + -- hit2 : process cache hit + + self.hit2 = true + + -- unlock before de-serializing cached data + _unlock(self) + data = _process_cached_data(self, data, flags) + return _return(self, data) + end + + -- continue to external lookup + end + + -- perform external lookup + data, err = self.callbacks.external_lookup() + + if data then + -- succ: save positive and return the data + + _save_positive(self, key, data) + return _return(self, data) + else + ngx.log(ngx.WARN, 'external lookup failed: ', err) + end + + -- external lookup failed + -- attempt to load stale data + data, flags = _get_stale(self) + if data and not _is_empty(data, flags) then + -- hit_stale + valid (positive) data + + flags = _save_actualize(self, key, data, flags) + -- unlock before de-serializing data + _unlock(self) + data = _process_cached_data(self, data, flags) + return _return(self, data) + end + + if DEBUG and data then + -- there is data, but it failed _is_empty() => stale negative data + print('STALE_NEGATIVE data => cache as a new HIT_NEGATIVE') + end + + -- nothing has worked, save negative and return empty + _save_negative(self, key) + return _return(self, nil) +end +M.load = load + +return M
\ No newline at end of file |