diff --git a/plugins/webstats/conf/webstats.conf b/plugins/webstats/conf/webstats.conf index 36852a9a9..ac03647ee 100644 --- a/plugins/webstats/conf/webstats.conf +++ b/plugins/webstats/conf/webstats.conf @@ -1,2 +1,2 @@ -lua_shared_dict mw_total 500m; +lua_shared_dict mw_total 50m; include {$SERVER_APP}/lua/webstats_log.lua; \ No newline at end of file diff --git a/plugins/webstats/lua/webstats_common.lua b/plugins/webstats/lua/webstats_common.lua index 3074006ee..a4a97c95c 100644 --- a/plugins/webstats/lua/webstats_common.lua +++ b/plugins/webstats/lua/webstats_common.lua @@ -18,6 +18,8 @@ local day_column = "day"..number_day local flow_column = "flow"..number_day local spider_column = "spider_flow"..number_day +local log_dir = "{$SERVER_APP}/logs" + function _M.new(self) local self = { total_key = total_key, @@ -42,7 +44,6 @@ end function _M.initDB(self, input_sn) - local log_dir = "{$SERVER_APP}/logs" local path = log_dir .. '/' .. input_sn .. "/logs.db" db, _ = sqlite3.open(path) @@ -70,24 +71,47 @@ end -- 后台任务 function _M.cron(self) + local cron_key = 'cron_every_1s' local timer_every_get_data = function (premature) + if self:is_working(cron_key) then + return + end + + self:lock_working(cron_key) + local llen, _ = ngx.shared.mw_total:llen(total_key) - -- self:D("cron llen:"..tostring(llen)) - for i=1,llen do - local data, _ = ngx.shared.mw_total:lpop(total_key) + local tmp_log = "{$SERVER_APP}/logs/tmp.log" + -- 空闲空间小于10M,立即存盘 + local capacity_bytes = ngx.shared.mw_total:free_space() + -- self:D("capacity_bytes:"..capacity_bytes) + if capacity_bytes <= 10485760 then + local data = "" + for i = 1,llen do + local tmp, _ = ngx.shared.mw_total:lpop(total_key) + data = data .. tmp .. "\n" + end + if data ~= "" then + self:write_file(tmp_log, data, "ab") + end + end + + -- 每秒100条 + local llen, _ = cache:llen(total_key) + for i=1,100 do + local data, _ = cache:lpop(total_key) if data then local info = json.decode(data) local input_sn = info['server_name'] if self:is_working(input_sn) then - ngx.shared.mw_total:rpush(total_key, data) - local capacity_bytes = ngx.shared.mw_total:free_space() - self:D("cron free_capacity:"..tostring(capacity_bytes)) + cache:rpush(total_key, data) os.execute("sleep " .. 0.6) return true end self:store_logs(info) end end + + self:unlock_working(cron_key) end ngx.timer.every(1, timer_every_get_data) @@ -324,21 +348,21 @@ function _M.is_migrating(self,input_sn) return false end -function _M.is_working(self,input_sn) - local work_status = cache:get(input_sn.."_working") +function _M.is_working(self,sign) + local work_status = cache:get(sign.."_working") if work_status ~= nil and work_status == true then return true end return false end -function _M.lock_working(self, input_sn) - local working_key = input_sn.."_working" +function _M.lock_working(self, sign) + local working_key = sign.."_working" cache:set(working_key, true, 60) end -function _M.unlock_working(self, input_sn) - local working_key = input_sn.."_working" +function _M.unlock_working(self, sign) + local working_key = sign.."_working" cache:set(working_key, false) end