From 817b4b36c7d13551dc05086beb1090fa46591740 Mon Sep 17 00:00:00 2001 From: dami Date: Tue, 7 Jul 2026 02:06:03 +0800 Subject: [PATCH 1/3] Update webstats.conf --- plugins/webstats/conf/webstats.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/webstats/conf/webstats.conf b/plugins/webstats/conf/webstats.conf index da8e87dfb..1c3b9626f 100644 --- a/plugins/webstats/conf/webstats.conf +++ b/plugins/webstats/conf/webstats.conf @@ -1,3 +1,4 @@ -lua_shared_dict mw_total 100m; +lua_shared_dict webstats_queue 128m; +lua_shared_dict webstats_cache 512m; lua_need_request_body off; log_by_lua_file {$SERVER_APP}/lua/webstats_log.lua; \ No newline at end of file From 6e2a8e0a12442531da3ab3f26b08ff0ccbb48646 Mon Sep 17 00:00:00 2001 From: dami Date: Tue, 7 Jul 2026 02:12:09 +0800 Subject: [PATCH 2/3] up --- plugins/webstats/lua/webstats_common.lua | 49 ++++++++++++++++++++------------ plugins/webstats/lua/webstats_log.lua | 11 ++++--- 2 files changed, 38 insertions(+), 22 deletions(-) diff --git a/plugins/webstats/lua/webstats_common.lua b/plugins/webstats/lua/webstats_common.lua index bd91f8b15..6f8d25afd 100644 --- a/plugins/webstats/lua/webstats_common.lua +++ b/plugins/webstats/lua/webstats_common.lua @@ -12,12 +12,12 @@ 架构说明： 1. 单例模式：通过 getInstance() 获取全局唯一实例 - 2. 共享内存：使用 ngx.shared.mw_total 作为日志缓存队列 + 2. 共享内存：webstats_queue 日志队列，webstats_cache UV/IP/域名等缓存 3. SQLite 数据库：每个站点独立一个数据库文件 4. 定时任务：每 0.5 秒执行一次日志持久化 主要数据流向： - webstats_log.lua (日志采集) -> ngx.shared.mw_total (缓存队列) -> cron() (定时处理) -> SQLite (持久化) + webstats_log.lua (日志采集) -> webstats_queue (日志队列) -> cron() (定时处理) -> SQLite (持久化) 依赖模块： - cjson: JSON 编解码 @@ -45,8 +45,9 @@ local total_key = "log_kv_total" local unset_server_name = "unset" -- 日志 ID 最大值，超过后重置 local max_log_id = 99999999999999 --- Nginx 共享内存字典实例 -local cache = ngx.shared.mw_total +-- Nginx 共享内存：队列与缓存分离 +local queue = ngx.shared.webstats_queue +local cache = ngx.shared.webstats_cache -- 当前日期（日） local day = os.date("%d") @@ -291,8 +292,7 @@ function _M.get_sn(self, input_sn) end end - -- 未匹配到任何站点 - cache:set(input_sn, unset_server_name, 86400) + -- 未匹配站点不写入缓存，避免扫描流量占满 webstats_cache return unset_server_name end @@ -475,7 +475,22 @@ function _M.cron(self) ]] local timer_every_get_data = function (premature) - local llen = ngx.shared.mw_total:llen(total_key) + local ok, llen = pcall(function() return queue:llen(total_key) end) + if not ok then + local queue_capacity = queue:capacity() + local queue_free = queue:free_space() + local queue_used = queue_capacity > 0 and (100 - math.floor(queue_free / queue_capacity * 100)) or 0 + local cache_capacity = cache:capacity() + local cache_free = cache:free_space() + local cache_used = cache_capacity > 0 and (100 - math.floor(cache_free / cache_capacity * 100)) or 0 + self:D("timer_every_get_data shared memory for " .. total_key .. ": " .. tostring(llen) + .. ", queue_used=" .. tostring(queue_used) .. "%, queue_free=" .. tostring(queue_free) + .. ", cache_used=" .. tostring(cache_used) .. "%, cache_free=" .. tostring(cache_free)) + return true + end + + + if llen == 0 then return true end @@ -560,7 +575,7 @@ function _M.cron(self) end for i = 1, llen do - local data = ngx.shared.mw_total:lpop(total_key) + local data = queue:lpop(total_key) if not data then break end @@ -568,23 +583,23 @@ function _M.cron(self) local decode_ok, info = pcall(function() return json.decode(data) end) if not decode_ok then self:D("json decode failed: " .. tostring(info)) - ngx.shared.mw_total:rpush(total_key, data) + queue:rpush(total_key, data) return end local input_sn = info['server_name'] if self:is_migrating(input_sn) then - ngx.shared.mw_total:rpush(total_key, data) + queue:rpush(total_key, data) else local db = dbs[input_sn] local stmt = stmts[input_sn] if not db or not stmt then - ngx.shared.mw_total:rpush(total_key, data) + queue:rpush(total_key, data) else local insert_ok = self:store_logs_line(db, stmt, input_sn, info) if not insert_ok then self:D("store_logs_line failed for " .. input_sn .. ": " .. tostring(insert_ok)) - ngx.shared.mw_total:rpush(total_key, data) + queue:rpush(total_key, data) rollback_sites[input_sn] = true else local log_kv = info["log_kv"] @@ -734,13 +749,13 @@ function _M.cron(self) function timer_every_get_data_try_debug() ngx.update_time() local start_time = ngx.now() - local start_llen = ngx.shared.mw_total:llen(total_key) + local start_llen = queue:llen(total_key) local presult, err = pcall(function() timer_every_get_data() end) ngx.update_time() local end_time = ngx.now() - local end_llen = ngx.shared.mw_total:llen(total_key) + local end_llen = queue:llen(total_key) local duration = (end_time - start_time) * 1000 local processed = start_llen - end_llen @@ -1036,13 +1051,11 @@ function _M.clean_stats(self, db, input_sn) end function _M.lpop(self) - local cache = ngx.shared.mw_total - return cache:lpop(total_key) + return queue:lpop(total_key) end function _M.rpop(self) - local cache = ngx.shared.mw_total - return cache:rpop(total_key) + return queue:rpop(total_key) end diff --git a/plugins/webstats/lua/webstats_log.lua b/plugins/webstats/lua/webstats_log.lua index 67e2501ed..895a5e798 100644 --- a/plugins/webstats/lua/webstats_log.lua +++ b/plugins/webstats/lua/webstats_log.lua @@ -57,7 +57,8 @@ local function run_app() local x_forwarded_for = ngx.var.http_x_forwarded_for local referer = ngx.var.http_referer - local cache = ngx.shared.mw_total + local cache = ngx.shared.webstats_cache + local queue = ngx.shared.webstats_queue local total_key = "log_kv_total" local status_codes_to_log = { @@ -276,15 +277,17 @@ local function run_app() log_kv = kv, } - local len, err, forcible = cache:rpush(total_key, json.encode(data)) + local len, err, forcible = queue:rpush(total_key, json.encode(data)) if not len then C:D("webstats rpush failed: " .. tostring(err or "unknown") - .. ", forcible=" .. tostring(forcible) .. ", queue=" .. total_key) + .. ", forcible=" .. tostring(forcible) .. ", queue=" .. total_key + .. ", queue_free=" .. tostring(queue:free_space()) + .. ", cache_free=" .. tostring(cache:free_space())) end end -- C:D("webstats_log run_app start, server_name=" .. tostring(server_name)) - -- C:D(tostring(total_key) ..":" .. tostring(ngx.shared.mw_total:llen(total_key))) + -- C:D(tostring(total_key) ..":" .. tostring(ngx.shared.webstats_queue:llen(total_key))) load_global_exclude_ip() load_exclude_ip(server_name) cache_logs(server_name) From 4f4d1e7580e3d11924e29c364dcea98ef6b4148b Mon Sep 17 00:00:00 2001 From: dami Date: Tue, 7 Jul 2026 02:15:57 +0800 Subject: [PATCH 3/3] Update webstats_common.lua --- plugins/webstats/lua/webstats_common.lua | 2 -- 1 file changed, 2 deletions(-) diff --git a/plugins/webstats/lua/webstats_common.lua b/plugins/webstats/lua/webstats_common.lua index 6f8d25afd..a80a7e294 100644 --- a/plugins/webstats/lua/webstats_common.lua +++ b/plugins/webstats/lua/webstats_common.lua @@ -489,8 +489,6 @@ function _M.cron(self) return true end - - if llen == 0 then return true end