fix: 限额恢复即时解锁 + 前置预留闸门，修复有额度却被禁
- 同步守护写健康限额后立即恢复被软踢模型（restore_model_if_healthy）， 不等 restore_blocked_models 的 30min/4h 闸门，修复日返还额度到账后 模型带额度空转被禁（no healthy deployments） - 前置闸门改为 usage + reserve >= limit 即拦截，预留优先取请求 max_tokens（零采集损失），堵住并发请求集体撞线的超限窗口
This commit is contained in:
vibe-code
+86
-4
@@ -97,6 +97,11 @@ BUFFER = 300000 # User requested 30w buffer
|
||||
CHECK_INTERVAL = 10
|
||||
THRESHOLD = 300000
|
||||
|
||||
# 前置闸门预留量：请求开始判定 usage + reserve >= limit 即 429。
|
||||
# reserve 优先取本次请求自带 max_tokens（真实上限，零采集损失），
|
||||
# 未设置时 fallback 本常量。堵住"并发请求穿过闸门后集体撞线"的窗口。
|
||||
PREBLOCK_RESERVE_TOKENS = int(os.getenv("PREBLOCK_RESERVE_TOKENS", "50000"))
|
||||
|
||||
# stable_limit 缺失时（首次同步前/新增模型/TTL 过期）的保守兜底限额。
|
||||
# 免费模型余额常态 < 2M（昨日采多少返多少），若 fallback 裸奔到采集上限 2M，
|
||||
# 余额不足的模型会被烧穿扣费。宁可短时收紧，不可裸奔。
|
||||
@@ -597,6 +602,65 @@ def restore_blocked_models():
|
||||
log(f"❌ Restore check failed: {e}", level="ERROR")
|
||||
|
||||
|
||||
def restore_model_if_healthy(clean_model, allowed_hashes=None):
|
||||
"""
|
||||
若指定模型已被软踢（_daily_limit_backup）且当前限额已恢复（usage < effective limit），
|
||||
立即恢复其路由条目。供同步守护在写健康限额后调用，不等 restore_blocked_models 的
|
||||
30min/4h 闸门；否则额度到账后模型会带额度空转被禁（有额度但 no healthy deployments）。
|
||||
allowed_hashes: 限定匹配的 api_base hash 集合（None = 不限，按 clean 名匹配）。
|
||||
返回恢复的条目数；任何异常 fail-safe 返回 0。
|
||||
"""
|
||||
if not LITELLM_AVAILABLE or not redis_client:
|
||||
return 0
|
||||
try:
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
if not llm_router or not llm_router.model_list:
|
||||
return 0
|
||||
|
||||
restored = 0
|
||||
for i, model_entry in enumerate(llm_router.model_list):
|
||||
backup = model_entry.get("_daily_limit_backup")
|
||||
if not backup:
|
||||
continue
|
||||
|
||||
params = model_entry.get("litellm_params", {})
|
||||
m_model = params.get("model", "")
|
||||
if clean_model_name(m_model) != clean_model:
|
||||
continue
|
||||
|
||||
m_base = params.get("api_base", "")
|
||||
if allowed_hashes is not None and generate_api_base_hash(
|
||||
m_base
|
||||
) not in allowed_hashes:
|
||||
continue
|
||||
|
||||
effective_limit = get_effective_limit(clean_model, m_base)
|
||||
should_restore = False
|
||||
if effective_limit is None:
|
||||
should_restore = True
|
||||
else:
|
||||
usage_key = get_usage_key(clean_model, m_base)
|
||||
current_val = float(redis_client.get(usage_key) or 0)
|
||||
if current_val < float(effective_limit):
|
||||
should_restore = True
|
||||
|
||||
if should_restore:
|
||||
dprint(f"♻️ [RESTORE] Restoring {backup['model_name']} (Quota OK)")
|
||||
llm_router.model_list[i]["model_name"] = backup["model_name"]
|
||||
llm_router.model_list[i]["litellm_params"]["api_base"] = m_base
|
||||
del llm_router.model_list[i]["_daily_limit_backup"]
|
||||
restored += 1
|
||||
|
||||
if restored > 0:
|
||||
log(f"✅ Restored {restored} entry/entries of {clean_model}.", level="INFO")
|
||||
return restored
|
||||
|
||||
except Exception as e:
|
||||
log(f"❌ Restore failed for {clean_model}: {e}", level="ERROR")
|
||||
return 0
|
||||
|
||||
|
||||
def startup_audit():
|
||||
"""
|
||||
🚀 Startup Audit:
|
||||
@@ -661,10 +725,12 @@ def startup_audit():
|
||||
log(f"❌ Startup audit failed: {e}", level="ERROR")
|
||||
|
||||
|
||||
def _check_request_block(model_name):
|
||||
def _check_request_block(model_name, reserve=0.0):
|
||||
"""
|
||||
请求前闸门：对外模型名 → 判定是否超限。
|
||||
只拦截 MONITORED_MODEL_PAIRS 内的火山监控条目；newapi 等非监控渠道天然放行。
|
||||
reserve: 本次请求的预估消耗（max_tokens 或常量兜底），usage + reserve >= limit 即拦截，
|
||||
避免并发请求在闸门检查后集体撞线（限额是低频快照，用量是实时累计）。
|
||||
返回 (clean_model, usage, limit) 表示超限，None 表示放行。任何异常 fail-open。
|
||||
"""
|
||||
if not redis_client or not LITELLM_AVAILABLE:
|
||||
@@ -705,7 +771,7 @@ def _check_request_block(model_name):
|
||||
val = redis_client.get(usage_key)
|
||||
usage = float(val or 0)
|
||||
|
||||
if usage >= float(effective_limit):
|
||||
if usage + float(reserve) >= float(effective_limit):
|
||||
return (clean, usage, effective_limit)
|
||||
|
||||
return None
|
||||
@@ -744,13 +810,24 @@ class DailyLimitHandler(CustomLogger):
|
||||
if not data or not data.get("model"):
|
||||
return None
|
||||
|
||||
blocked = _check_request_block(data["model"])
|
||||
# 预留本次请求的预估消耗：优先真实 max_tokens（零采集损失），
|
||||
# 未设置/非法时 fallback 常量，避免并发请求集体撞线超限。
|
||||
try:
|
||||
reserve = float(
|
||||
data.get("max_tokens")
|
||||
or data.get("max_completion_tokens")
|
||||
or PREBLOCK_RESERVE_TOKENS
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
reserve = float(PREBLOCK_RESERVE_TOKENS)
|
||||
|
||||
blocked = _check_request_block(data["model"], reserve=reserve)
|
||||
if not blocked:
|
||||
return None
|
||||
|
||||
clean, usage, limit = blocked
|
||||
log(
|
||||
f"🚫 [PRE-BLOCK] {data['model']} daily limit reached ({int(usage):,}/{int(limit):,})",
|
||||
f"🚫 [PRE-BLOCK] {data['model']} daily limit reached ({int(usage):,}+{int(reserve):,}/{int(limit):,})",
|
||||
level="WARNING",
|
||||
)
|
||||
raise RateLimitError(
|
||||
@@ -1435,6 +1512,11 @@ def active_monitor_loop():
|
||||
flush=True,
|
||||
)
|
||||
|
||||
# 即时恢复：同步发现限额已恢复（usage < effective limit）时立刻解除软踢，
|
||||
# 不等 restore_blocked_models 的 30min/4h 闸门 —— 否则日返还额度到账后
|
||||
# 模型仍被禁（有额度但 no healthy deployments），直到下一次恢复检查。
|
||||
restore_model_if_healthy(target, target_hashes)
|
||||
|
||||
# Track min remaining for frequency control
|
||||
configured_models_found = True
|
||||
|
||||
|
||||
Reference in New Issue
Block a user