From 505a55987fd8d5f6c078b48a59b6e3613f9309b0 Mon Sep 17 00:00:00 2001 From: vibe-code Date: Wed, 12 Aug 2026 03:40:44 +0000 Subject: [PATCH] =?UTF-8?q?fix:=20=E9=99=90=E9=A2=9D=E6=81=A2=E5=A4=8D?= =?UTF-8?q?=E5=8D=B3=E6=97=B6=E8=A7=A3=E9=94=81=20+=20=E5=89=8D=E7=BD=AE?= =?UTF-8?q?=E9=A2=84=E7=95=99=E9=97=B8=E9=97=A8=EF=BC=8C=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E6=9C=89=E9=A2=9D=E5=BA=A6=E5=8D=B4=E8=A2=AB=E7=A6=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 同步守护写健康限额后立即恢复被软踢模型（restore_model_if_healthy）， 不等 restore_blocked_models 的 30min/4h 闸门，修复日返还额度到账后 模型带额度空转被禁（no healthy deployments） - 前置闸门改为 usage + reserve >= limit 即拦截，预留优先取请求 max_tokens（零采集损失），堵住并发请求集体撞线的超限窗口 --- volc_limit_sync.py | 90 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 86 insertions(+), 4 deletions(-) diff --git a/volc_limit_sync.py b/volc_limit_sync.py index 991ce0b..c485672 100644 --- a/volc_limit_sync.py +++ b/volc_limit_sync.py @@ -97,6 +97,11 @@ BUFFER = 300000 # User requested 30w buffer CHECK_INTERVAL = 10 THRESHOLD = 300000 +# 前置闸门预留量：请求开始判定 usage + reserve >= limit 即 429。 +# reserve 优先取本次请求自带 max_tokens（真实上限，零采集损失）， +# 未设置时 fallback 本常量。堵住"并发请求穿过闸门后集体撞线"的窗口。 +PREBLOCK_RESERVE_TOKENS = int(os.getenv("PREBLOCK_RESERVE_TOKENS", "50000")) + # stable_limit 缺失时（首次同步前/新增模型/TTL 过期）的保守兜底限额。 # 免费模型余额常态 < 2M（昨日采多少返多少），若 fallback 裸奔到采集上限 2M， # 余额不足的模型会被烧穿扣费。宁可短时收紧，不可裸奔。 @@ -597,6 +602,65 @@ def restore_blocked_models(): log(f"❌ Restore check failed: {e}", level="ERROR") +def restore_model_if_healthy(clean_model, allowed_hashes=None): + """ + 若指定模型已被软踢（_daily_limit_backup）且当前限额已恢复（usage < effective limit）， + 立即恢复其路由条目。供同步守护在写健康限额后调用，不等 restore_blocked_models 的 + 30min/4h 闸门；否则额度到账后模型会带额度空转被禁（有额度但 no healthy deployments）。 + allowed_hashes: 限定匹配的 api_base hash 集合（None = 不限，按 clean 名匹配）。 + 返回恢复的条目数；任何异常 fail-safe 返回 0。 + """ + if not LITELLM_AVAILABLE or not redis_client: + return 0 + try: + from litellm.proxy.proxy_server import llm_router + + if not llm_router or not llm_router.model_list: + return 0 + + restored = 0 + for i, model_entry in enumerate(llm_router.model_list): + backup = model_entry.get("_daily_limit_backup") + if not backup: + continue + + params = model_entry.get("litellm_params", {}) + m_model = params.get("model", "") + if clean_model_name(m_model) != clean_model: + continue + + m_base = params.get("api_base", "") + if allowed_hashes is not None and generate_api_base_hash( + m_base + ) not in allowed_hashes: + continue + + effective_limit = get_effective_limit(clean_model, m_base) + should_restore = False + if effective_limit is None: + should_restore = True + else: + usage_key = get_usage_key(clean_model, m_base) + current_val = float(redis_client.get(usage_key) or 0) + if current_val < float(effective_limit): + should_restore = True + + if should_restore: + dprint(f"♻️ [RESTORE] Restoring {backup['model_name']} (Quota OK)") + llm_router.model_list[i]["model_name"] = backup["model_name"] + llm_router.model_list[i]["litellm_params"]["api_base"] = m_base + del llm_router.model_list[i]["_daily_limit_backup"] + restored += 1 + + if restored > 0: + log(f"✅ Restored {restored} entry/entries of {clean_model}.", level="INFO") + return restored + + except Exception as e: + log(f"❌ Restore failed for {clean_model}: {e}", level="ERROR") + return 0 + + def startup_audit(): """ 🚀 Startup Audit: @@ -661,10 +725,12 @@ def startup_audit(): log(f"❌ Startup audit failed: {e}", level="ERROR") -def _check_request_block(model_name): +def _check_request_block(model_name, reserve=0.0): """ 请求前闸门：对外模型名 → 判定是否超限。 只拦截 MONITORED_MODEL_PAIRS 内的火山监控条目；newapi 等非监控渠道天然放行。 + reserve: 本次请求的预估消耗（max_tokens 或常量兜底），usage + reserve >= limit 即拦截， + 避免并发请求在闸门检查后集体撞线（限额是低频快照，用量是实时累计）。 返回 (clean_model, usage, limit) 表示超限，None 表示放行。任何异常 fail-open。 """ if not redis_client or not LITELLM_AVAILABLE: @@ -705,7 +771,7 @@ def _check_request_block(model_name): val = redis_client.get(usage_key) usage = float(val or 0) - if usage >= float(effective_limit): + if usage + float(reserve) >= float(effective_limit): return (clean, usage, effective_limit) return None @@ -744,13 +810,24 @@ class DailyLimitHandler(CustomLogger): if not data or not data.get("model"): return None - blocked = _check_request_block(data["model"]) + # 预留本次请求的预估消耗：优先真实 max_tokens（零采集损失）， + # 未设置/非法时 fallback 常量，避免并发请求集体撞线超限。 + try: + reserve = float( + data.get("max_tokens") + or data.get("max_completion_tokens") + or PREBLOCK_RESERVE_TOKENS + ) + except (TypeError, ValueError): + reserve = float(PREBLOCK_RESERVE_TOKENS) + + blocked = _check_request_block(data["model"], reserve=reserve) if not blocked: return None clean, usage, limit = blocked log( - f"🚫 [PRE-BLOCK] {data['model']} daily limit reached ({int(usage):,}/{int(limit):,})", + f"🚫 [PRE-BLOCK] {data['model']} daily limit reached ({int(usage):,}+{int(reserve):,}/{int(limit):,})", level="WARNING", ) raise RateLimitError( @@ -1435,6 +1512,11 @@ def active_monitor_loop(): flush=True, ) + # 即时恢复：同步发现限额已恢复（usage < effective limit）时立刻解除软踢， + # 不等 restore_blocked_models 的 30min/4h 闸门 —— 否则日返还额度到账后 + # 模型仍被禁（有额度但 no healthy deployments），直到下一次恢复检查。 + restore_model_if_healthy(target, target_hashes) + # Track min remaining for frequency control configured_models_found = True