add header cache, restructure worker

Author: millken

conf/dyn.conf

@@ -1,7 +1,7 @@
 user  root;
 worker_processes 2;
 daemon off;
-error_log stderr info;
+error_log stderr warn;
 
 pid /var/run/nginx.pid;
 
@@ -24,7 +24,14 @@ http {
     lua_shared_dict settings 250m;
     lua_shared_dict cached 100m;
     lua_shared_dict locked 100k;
-    
+
+	proxy_cache pcache;
+	proxy_cache_path  /data/proxy_cache  levels=1:2   keys_zone=pcache:10m inactive=1d max_size=10m use_temp_path=off inactive=180m;
+    proxy_cache_lock on;
+    proxy_cache_min_uses 1;
+    proxy_cache_revalidate on;
+    proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504;   
+
     init_by_lua_block {
 		local config = require "cdn.config"
 		config:set("log.status", true)
@@ -56,14 +63,20 @@ http {
         set $ups "";
 		location / {
             proxy_pass http://$ups;
+			add_header  X-Cache "$upstream_cache_status from DYN";
 			access_by_lua_block {
 				require "cdn.access"
 			}
 
 			rewrite_by_lua_block {
 				local worker = require "cdn.worker"
-				worker:rewrite()
-			}            
+				worker.rewrite()
+			}
+			
+			header_filter_by_lua_block {
+				local header = require "cdn.header"
+				header:bootstrap()
+			}
         }
 
         location @dyup_init {
@@ -95,6 +108,14 @@ http {
 				ngx.say(t:domain("aasdf.asdfsd.com.cn"))
 			}        
         }
+
+		location /t3 {
+	        content_by_lua_block {
+        		local config = require "cdn.config"
+				local cjson = require "cjson"
+				ngx.say(cjson.encode(config:get("hostcfg")))
+			}
+		}
 
         location /loadall {
         	content_by_lua_block {

conf/postgres.conf

@@ -1,16 +1,14 @@
 local cjson = require "cjson"
 local cookie = require "cdn.cookie"
+local config = require "cdn.config"
+local log = require "cdn.log"
 
 local   tostring, ipairs, pairs, type, tonumber, next, unpack =
         tostring, ipairs, pairs, type, tonumber, next, unpack
 
-local ngx_log = ngx.log
 local ngx_var = ngx.var
 local ngx_ctx = ngx.ctx
 local ngx_re_find = ngx.re.find
-local ngx_DEBUG = ngx.DEBUG
-local ngx_ERR = ngx.ERR
-local ngx_INFO = ngx.INFO
 local ngx_now = ngx.now
 local ngx_exit = ngx.exit
 local ngx_md5 = ngx.md5
@@ -83,7 +81,7 @@ if not hd_count then
 end
 
 
-ngx_log(ngx_INFO, "ip total: ", req_iplist:get(ip_key), "(", ip_interval, "s)", 
+log:info("ip total: ", req_iplist:get(ip_key), "(", ip_interval, "s)", 
 "req total: ", req_metrics:get(req_key), "(", req_interval, "s)",
 "http header total: ", req_metrics:get(hd_key), "(", hd_interval, "s)", ngx_time()
 )

lua-nginx-module

@@ -1,6 +1,4 @@
 local ngx = ngx
-local   tostring, ipairs, pairs, type, tonumber, next, unpack =
-        tostring, ipairs, pairs, type, tonumber, next, unpack
 
 local _M = {
     _VERSION = '0.01',

lualib/cdn/access.lua

@@ -1,6 +1,7 @@
 local cjson = require "cjson"
 local config = require "cdn.config"
 local log = require "cdn.log"
+local hosts = require "cdn.hosts"
 
 local redis_mod = require "resty.redis"
 local dyups = require "ngx.dyups"
@@ -116,14 +117,15 @@ _M.states = {
 		if setting_json == nil then
 			return false
 		end
+		hosts.delete_cache(servername)
+		settings:delete(servername)
 		local setting = cmsgpack.unpack(setting_json)
 		local k
 		for k, _ in pairs(setting) do
 			log:debug("delete vhost: ", k)
 			dyups.delete(k)
 			upstream_cached:delete(k)
 		end
-		settings:delete(servername)
 		return true
 	end,
 

lualib/cdn/config.lua

@@ -0,0 +1,54 @@
+local cjson = require "cjson"
+local config = require "cdn.config"
+local log = require "cdn.log"
+local http_cache = require "http_cache"
+
+local ipairs = ipairs
+local str_find = string.find
+
+local ngx = ngx
+local ngx_var = ngx.var
+
+local _M = {
+    _VERSION = 0.01,
+	events = {"cache"},
+}
+
+function _M.get_file_ext(uri)
+    return uri:match(".+%.(%w+)$") or "none"
+end
+
+_M.states = {
+	cache = function()
+		local cache_status = (ngx_var.upstream_cache_status or "")
+		local upsconf = config:get("upsconf")
+		local cache_time = 0
+		log:info("upsconf header: ", cjson.encode(upsconf), "cache: ", cache_status)
+		if not upsconf["cache"] then return end
+		for _, r in ipairs(upsconf["cache"]) do
+			if r["url"] and str_find(ngx_var.uri, r["url"], 1, true) ~= nil then
+				cache_time = r["time"]
+			elseif r["file"] and str_find(r["file"], _M.get_file_ext(ngx_var.uri), 1, true) ~= nil then
+				cache_time = r["time"]
+			end
+		end
+	    if cache_time > 0 and (cache_status == "MISS" or cache_status == "EXPIRED") then
+            local cache_data = http_cache.get_metadata()
+            local new_expire = ngx.time() + cache_time 
+
+            if cache_data and cache_data["valid_sec"] then
+                http_cache.set_metadata({ valid_sec = new_expire,
+                                          fcn = { valid_sec = new_expire,
+                                          expire = new_expire } })
+            end
+        end	
+	end,
+}
+
+function _M.bootstrap()
+	for _, state in ipairs(_M.events) do
+		_M.states[state]()
+	end
+end
+
+return _M

lualib/cdn/events.lua

@@ -0,0 +1,84 @@
+local cmsgpack = require "cmsgpack"
+local tlds = require "cdn.tlds"
+local lrucache_mod = require "resty.lrucache"
+local log = require "cdn.log"
+local lrudsc, err = lrucache_mod.new(500)
+if not lrudsc then 
+	error("failed to create the cache: " .. (err or "unknown"))	
+end
+local lrucache, err = lrucache_mod.new(500)
+if not lrucache then 
+	error("failed to create the cache: " .. (err or "unknown"))	
+end
+
+local ngx = ngx
+local ngx_var = ngx.var
+local ngx_re_find = ngx.re.find
+
+local settings = ngx.shared.settings
+
+local _M = {
+    _VERSION = '0.01',
+}
+
+function _M.get_ups(host)
+	local ups_key, ups_value
+	local setting, err = _M.get_setting(host)
+	if setting == nil then
+		log:info("get_setting : ", err)
+		return nil, nil
+	end
+	if setting[ngx_var.host] == nil then
+		for k, v in pairs(setting) do
+			local i = k:find("%*")
+			if i then 
+				local rek, n, err = ngx.re.gsub(k, "\\*", "(.*?)")
+				local from, to, err = ngx_re_find(ngx_var.host, rek, "isjo")
+				if from and v.ups ~= nil then
+					ups_key = k
+					lrucache:set(ngx_var.host, ups_key)
+					ups_value = v.ups
+					break
+				end
+			end
+		end
+	else
+		local v = setting[ngx_var.host]
+		if v.ups ~= nil then
+			ups_key = ngx_var.host
+			lrucache:set(ngx_var.host, ups_key)
+			ups_value = v.ups
+		end
+	end
+	return ups_key, ups_value
+end
+
+function _M.get_ups_key(host)
+	return lrucache:get(host)
+end
+
+function _M.get_setting(host)
+	local topleveldomain = tlds.domain(host)
+	if topleveldomain == nil then
+		return nil, "failed to get tld: " .. host
+	end
+	local setting = lrudsc:get(topleveldomain)
+	if setting ~= nil then
+		return setting
+	end
+	local setting_json = settings:get(topleveldomain)
+	if setting_json == nil then
+		return nil, "failed to get setting: " .. topleveldomain
+	end
+	local setting = cmsgpack.unpack(setting_json)
+	lrudsc:set(topleveldomain, setting)
+	return setting, nil
+
+end
+
+function _M.delete_cache(host)
+	lrudsc:delete(host)
+end
+
+
+return _M

lualib/cdn/header.lua

@@ -37,7 +37,7 @@ local function split(s, re)
   end
 end
 
-function _M.get_tlds(self)
+function _M.get_tlds()
 	local tlds = cached:get("tldsdb")
 	local tmp = {}
 	if 	not tlds then
@@ -65,12 +65,12 @@ function _M.get_tlds(self)
 	return cmsgpack.unpack(cached:get("tldsdb"))
 end
 
-function _M.domain(self, host)
+function _M.domain(host)
 	local result, state = lrucache:get(host)
 	if result then
 		return result
 	end
-	local node = self.get_tlds() 
+	local node = _M.get_tlds() 
 	local parts = split(host:lower(), "%.")
 	local i1 = 1
 	local re = "%."