From 795cc99faa2b9b87de132767800d41e3f41bc822 Mon Sep 17 00:00:00 2001 From: vibe-code Date: Thu, 13 Aug 2026 08:56:26 +0000 Subject: [PATCH] =?UTF-8?q?feat:=20=E8=BD=AF=E8=B8=A2=E9=A2=84=E8=AD=A6?= =?UTF-8?q?=E7=BA=BF=E6=8F=90=E5=89=8D=E8=87=B3=201.95M=20=E7=94=A8?= =?UTF-8?q?=E9=87=8F=E5=A2=99=E4=B8=8E=2010=E4=B8=87=E4=BD=99=E9=A2=9D?= =?UTF-8?q?=E5=A2=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - should_soft_evict: usage ≥ min(effective_limit, 1.95M) 或 (balance_at_sync < 10万 且 usage>0)——usage>0 防小额初始池(B0<10万) 模型误杀永久死锁 - should_restore_model 恢复对称: usage < min(额度,1.95M) 且 (余额≥10万 或 usage==0),否则踢了立即被同步恢复成抖动 - _check_request_block 对已软踢条目(*-limit-reached)恒 429, 防踢后 1.95M~2M 区间小请求放行→路由原名缺失→400 - startup_audit / restore_blocked_models / restore_model_if_healthy 统一接入判定; SOFT_EVICT_USAGE_LIMIT/SOFT_EVICT_BALANCE_LIMIT env 可调 - docs §7.6 新增软踢预警线小节(两墙条件/恢复对称/前置联动/2.5% 采集代价) - 数字用例 9/9 验证(踢/不踢/死锁防护/防抖动/动态墙取 min) --- docs/volc-quota-notes.md | 24 ++++++++ volc_limit_sync.py | 121 ++++++++++++++++++++++++++++++++------- 2 files changed, 124 insertions(+), 21 deletions(-) diff --git a/docs/volc-quota-notes.md b/docs/volc-quota-notes.md index 7cd0396..4b4544d 100644 --- a/docs/volc-quota-notes.md +++ b/docs/volc-quota-notes.md @@ -386,6 +386,28 @@ reserve = 18输入 + 16声明 = 34，实际消耗 +167/+150（正文 ~16 + 思 修复：输出侧取 `max(声明 max_tokens, THINKING_RESERVE_FLOOR=4096)` 兜底思考开销。 大输入请求仍偏保守（cl100k_base 中文 2-3 token/字高估），方向安全。 +#### 软踢预警线（2026-08-13） + +后置 soft_evict 触发点从"撞有效墙"提前为两个预警线（`volc_limit_sync.py` `should_soft_evict`， +env `SOFT_EVICT_USAGE_LIMIT=1950000` / `SOFT_EVICT_BALANCE_LIMIT=100000` 可调）： + +| 预警线 | 触发条件 | 目的 | +|---|---|---| +| 用量墙 | usage ≥ min(effective_limit, 1.95M) | 避开贴 2M 边的灰色地带（大请求 429/小请求放行 + reserve 低估/并发击穿余量） | +| 余额墙 | balance_at_sync < 10万 且 usage > 0 | 免费额度见底提前摘除；usage>0 防小额初始池(B0<10万)模型误杀永久死锁 | + +**恢复对称（`should_restore_model`）**：usage < min(effective_limit, 1.95M) 且 +(余额 ≥ 10万 或 usage == 0) 才恢复——否则踢了立即被下一轮同步恢复成踢/恢复抖动。 +恢复函数读 balance_at_sync 与 usage 同源（快照口径）。`restore_blocked_models` / +`restore_model_if_healthy` / `startup_audit` 同口径。 + +**前置联动**：`_check_request_block` 对已软踢条目(`*-limit-reached`)恒 429，防踢后 +1.95M~2M 区间小请求放行 → 路由原名缺失 → 400 路由错误。 + +**代价**：用量墙让 1.95M~2M 的 5 万采集放弃（明日返还少 5 万 ≈2.5%），换取贴边期容错； +余额墙对正常大池模型无影响（余额 ≥10万 不触发）。数字用例验证 2026-08-13：9/9 场景 +（踢/不踢/死锁防护/防抖动/动态墙取 min）全部符合预期。 + --- ## 8. 排障速查 @@ -426,6 +448,8 @@ reserve = 18输入 + 16声明 = 34，实际消耗 +167/+150（正文 ~16 + 思 工具（未公开公式），FAQ 只说"图片计入输入 token"。闸门调用未传 use_default_image_token_count → high-detail 图会在 hook 内同步下载（阻塞）。偏差量级 = 每图几十~几百 token，相对 THINKING_RESERVE_FLOOR(4096) 小；方向待实测（Token 计算器对照或发图看 usage），有图流量时关注 +- [x] 软踢预警线（2026-08-13 实施，§7.6）：后置踢出提前为 用量墙 1.95M / 余额墙 10万(usage>0)， + 恢复对称 + 前置 limit-reached 恒 429；9/9 数字用例验证 - [ ] 空窗期(0:00~返还到账)行为观察：v6 下 B0 自动收紧，确认无异常误杀 - [ ] 数据库泄漏：08-12 一天 deepseek-v4 消耗记录量级约 5 万条导致 Redis 内存膨胀；触发监控/清理机制 - [ ] v6 无静态 2M 兜底时的安全：08-13 幸由 B0(2,242,178) > 2M 静态墙兜底；若某日 B0 < 2M， diff --git a/volc_limit_sync.py b/volc_limit_sync.py index 333ac1b..bd7a2e2 100644 --- a/volc_limit_sync.py +++ b/volc_limit_sync.py @@ -173,6 +173,19 @@ HARD_DAILY_LIMITS = { "__DEFAULT__": 2_000_000, } +# ========================================== +# 🚨 Soft Evict Thresholds（软踢预警线） +# ========================================== +# 提前于硬限额/动态墙踢出模型的预警线，留出缓冲： +# 1. 用量墙：usage ≥ SOFT_EVICT_USAGE_LIMIT(默认 1.95M) 即踢——避免贴 2M 边时 +# 大请求 429/小请求放行的灰色地带（reserve 低估 / 并发击穿余量） +# 2. 余额墙：balance_at_sync < SOFT_EVICT_BALANCE_LIMIT(10万) 且 usage > 0 即踢—— +# usage>0 限定防误杀小额初始池模型（B0<10 万未使用时被永久踢出死锁） +# 恢复条件对称（should_restore_model）：usage < min(effective_limit, 用量墙) +# 且 (余额 ≥ 10万 或 usage == 0)，否则踢了立即被恢复成踢/恢复抖动 +SOFT_EVICT_USAGE_LIMIT = float(os.getenv("SOFT_EVICT_USAGE_LIMIT", "1950000")) +SOFT_EVICT_BALANCE_LIMIT = float(os.getenv("SOFT_EVICT_BALANCE_LIMIT", "100000")) + # ========================================== # 🔌 Redis Connection # ========================================== @@ -477,6 +490,67 @@ def get_effective_limit(clean_model, api_base, current_usage=None): # ========================================== # 👻 Handler Logic (DailyLimitHandler) +# ========================================== +# 🚨 Soft Evict / Restore Decision（软踢与恢复对称判定） +# ========================================== +def soft_evict_wall(effective_limit): + """软踢用量墙 = min(有效限额, SOFT_EVICT_USAGE_LIMIT)；None 表示无限额。""" + if effective_limit is None: + return None + return min(float(effective_limit), SOFT_EVICT_USAGE_LIMIT) + + +def should_soft_evict(clean_model, api_base, current_usage, effective_limit): + """ + 软踢判定：任一满足即触发 + - 用量墙：current_usage >= min(effective_limit, SOFT_EVICT_USAGE_LIMIT) + - 余额墙：balance_at_sync < SOFT_EVICT_BALANCE_LIMIT 且 current_usage > 0 + （balance_at_sync 缺失视为充足；usage>0 防小额初始池模型误杀死锁） + 返回 (triggered, reason)。 + """ + wall = soft_evict_wall(effective_limit) + if wall is not None and float(current_usage) >= wall: + return True, f"usage {int(float(current_usage)):,} >= soft wall {int(wall):,}" + try: + bal_key = f"limit:dynamic:volcengine:balance_at_sync:{clean_model}" + bal_val = redis_client.get(bal_key) if redis_client else None + if ( + bal_val is not None + and float(bal_val) < SOFT_EVICT_BALANCE_LIMIT + and float(current_usage) > 0 + ): + return ( + True, + f"balance {int(float(bal_val)):,} < {SOFT_EVICT_BALANCE_LIMIT:.0f} (usage>0)", + ) + except Exception: + pass + return False, "" + + +def should_restore_model(clean_model, api_base, current_usage, effective_limit): + """ + 恢复判定（与 should_soft_evict 对称，防踢/恢复抖动）： + usage < min(effective_limit, SOFT_EVICT_USAGE_LIMIT) 且 (余额 ≥ 10万 或 usage == 0) + effective_limit None 时仅按余额条件判定（原逻辑 None → 可恢复）。 + """ + wall = soft_evict_wall(effective_limit) + if wall is not None and float(current_usage) >= wall: + return False + try: + bal_key = f"limit:dynamic:volcengine:balance_at_sync:{clean_model}" + bal_val = redis_client.get(bal_key) if redis_client else None + if ( + bal_val is not None + and float(bal_val) < SOFT_EVICT_BALANCE_LIMIT + and float(current_usage) > 0 + ): + return False + except Exception: + pass + return True + + # ========================================== def soft_evict_from_router(target_model, target_api_base): """ @@ -596,14 +670,11 @@ def restore_blocked_models(): # get_effective_limit handles fetching usage if None passed. effective_limit = get_effective_limit(clean_model, original_api_base) - should_restore = False - if effective_limit is None: - should_restore = True - else: - usage_key = get_usage_key(clean_model, original_api_base) - current_val = float(redis_client.get(usage_key) or 0) - if current_val < float(effective_limit): - should_restore = True + usage_key = get_usage_key(clean_model, original_api_base) + current_val = float(redis_client.get(usage_key) or 0) + should_restore = should_restore_model( + clean_model, original_api_base, current_val, effective_limit + ) if should_restore: dprint(f"♻️ [RESTORE] Restoring {backup['model_name']} (Quota OK)") @@ -655,14 +726,11 @@ def restore_model_if_healthy(clean_model, allowed_hashes=None): continue effective_limit = get_effective_limit(clean_model, m_base) - should_restore = False - if effective_limit is None: - should_restore = True - else: - usage_key = get_usage_key(clean_model, m_base) - current_val = float(redis_client.get(usage_key) or 0) - if current_val < float(effective_limit): - should_restore = True + usage_key = get_usage_key(clean_model, m_base) + current_val = float(redis_client.get(usage_key) or 0) + should_restore = should_restore_model( + clean_model, m_base, current_val, effective_limit + ) if should_restore: dprint(f"♻️ [RESTORE] Restoring {backup['model_name']} (Quota OK)") @@ -732,11 +800,14 @@ def startup_audit(): continue usage_key = get_usage_key(clean_model, api_base) - current_val = redis_client.get(usage_key) + current_val = float(redis_client.get(usage_key) or 0) - if current_val and float(current_val) >= float(effective_limit): + evict, evict_reason = should_soft_evict( + clean_model, api_base, current_val, effective_limit + ) + if evict: dprint( - f"🚫 [STARTUP BLOCK] Found {raw_model} over limit ({int(float(current_val))}/{int(effective_limit)}). Evicting now." + f"🚫 [STARTUP BLOCK] {raw_model} soft-evict: {evict_reason}. Evicting now." ) soft_evict_from_router(raw_model, api_base) blocked_count += 1 @@ -798,6 +869,11 @@ def _check_request_block(model_name, reserve=0.0): val = redis_client.get(usage_key) usage = float(val or 0) + # 已被软踢的条目恒 429(无论 reserve)，避免踢后灰色区间(如 1.95M~2M) + # 小请求被放行 → 路由原名不存在 → 400 路由错误 + if entry_name == f"{model_name}-limit-reached" and "_daily_limit_backup" in entry: + return (clean, usage, effective_limit) + if usage + float(reserve) >= float(effective_limit): return (clean, usage, effective_limit) @@ -964,9 +1040,12 @@ class DailyLimitHandler(CustomLogger): level="INFO", ) - if effective_limit is not None and float(current_total) >= float(effective_limit): + evict, evict_reason = should_soft_evict( + clean_model, api_base, current_total, effective_limit + ) + if evict: log( - f"🚫 [BLOCK] {clean_model} Limit Hit: {current_total}/{effective_limit}", + f"🚫 [BLOCK] {clean_model} soft-evict: {evict_reason} (limit: {effective_limit})", level="WARNING", ) soft_evict_from_router(raw_model, api_base)