fix: 限额恢复即时解锁 + 前置预留闸门,修复有额度却被禁
- 同步守护写健康限额后立即恢复被软踢模型(restore_model_if_healthy), 不等 restore_blocked_models 的 30min/4h 闸门,修复日返还额度到账后 模型带额度空转被禁(no healthy deployments) - 前置闸门改为 usage + reserve >= limit 即拦截,预留优先取请求 max_tokens(零采集损失),堵住并发请求集体撞线的超限窗口
This commit is contained in:
+86
-4
@@ -97,6 +97,11 @@ BUFFER = 300000 # User requested 30w buffer
|
|||||||
CHECK_INTERVAL = 10
|
CHECK_INTERVAL = 10
|
||||||
THRESHOLD = 300000
|
THRESHOLD = 300000
|
||||||
|
|
||||||
|
# 前置闸门预留量:请求开始判定 usage + reserve >= limit 即 429。
|
||||||
|
# reserve 优先取本次请求自带 max_tokens(真实上限,零采集损失),
|
||||||
|
# 未设置时 fallback 本常量。堵住"并发请求穿过闸门后集体撞线"的窗口。
|
||||||
|
PREBLOCK_RESERVE_TOKENS = int(os.getenv("PREBLOCK_RESERVE_TOKENS", "50000"))
|
||||||
|
|
||||||
# stable_limit 缺失时(首次同步前/新增模型/TTL 过期)的保守兜底限额。
|
# stable_limit 缺失时(首次同步前/新增模型/TTL 过期)的保守兜底限额。
|
||||||
# 免费模型余额常态 < 2M(昨日采多少返多少),若 fallback 裸奔到采集上限 2M,
|
# 免费模型余额常态 < 2M(昨日采多少返多少),若 fallback 裸奔到采集上限 2M,
|
||||||
# 余额不足的模型会被烧穿扣费。宁可短时收紧,不可裸奔。
|
# 余额不足的模型会被烧穿扣费。宁可短时收紧,不可裸奔。
|
||||||
@@ -597,6 +602,65 @@ def restore_blocked_models():
|
|||||||
log(f"❌ Restore check failed: {e}", level="ERROR")
|
log(f"❌ Restore check failed: {e}", level="ERROR")
|
||||||
|
|
||||||
|
|
||||||
|
def restore_model_if_healthy(clean_model, allowed_hashes=None):
|
||||||
|
"""
|
||||||
|
若指定模型已被软踢(_daily_limit_backup)且当前限额已恢复(usage < effective limit),
|
||||||
|
立即恢复其路由条目。供同步守护在写健康限额后调用,不等 restore_blocked_models 的
|
||||||
|
30min/4h 闸门;否则额度到账后模型会带额度空转被禁(有额度但 no healthy deployments)。
|
||||||
|
allowed_hashes: 限定匹配的 api_base hash 集合(None = 不限,按 clean 名匹配)。
|
||||||
|
返回恢复的条目数;任何异常 fail-safe 返回 0。
|
||||||
|
"""
|
||||||
|
if not LITELLM_AVAILABLE or not redis_client:
|
||||||
|
return 0
|
||||||
|
try:
|
||||||
|
from litellm.proxy.proxy_server import llm_router
|
||||||
|
|
||||||
|
if not llm_router or not llm_router.model_list:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
restored = 0
|
||||||
|
for i, model_entry in enumerate(llm_router.model_list):
|
||||||
|
backup = model_entry.get("_daily_limit_backup")
|
||||||
|
if not backup:
|
||||||
|
continue
|
||||||
|
|
||||||
|
params = model_entry.get("litellm_params", {})
|
||||||
|
m_model = params.get("model", "")
|
||||||
|
if clean_model_name(m_model) != clean_model:
|
||||||
|
continue
|
||||||
|
|
||||||
|
m_base = params.get("api_base", "")
|
||||||
|
if allowed_hashes is not None and generate_api_base_hash(
|
||||||
|
m_base
|
||||||
|
) not in allowed_hashes:
|
||||||
|
continue
|
||||||
|
|
||||||
|
effective_limit = get_effective_limit(clean_model, m_base)
|
||||||
|
should_restore = False
|
||||||
|
if effective_limit is None:
|
||||||
|
should_restore = True
|
||||||
|
else:
|
||||||
|
usage_key = get_usage_key(clean_model, m_base)
|
||||||
|
current_val = float(redis_client.get(usage_key) or 0)
|
||||||
|
if current_val < float(effective_limit):
|
||||||
|
should_restore = True
|
||||||
|
|
||||||
|
if should_restore:
|
||||||
|
dprint(f"♻️ [RESTORE] Restoring {backup['model_name']} (Quota OK)")
|
||||||
|
llm_router.model_list[i]["model_name"] = backup["model_name"]
|
||||||
|
llm_router.model_list[i]["litellm_params"]["api_base"] = m_base
|
||||||
|
del llm_router.model_list[i]["_daily_limit_backup"]
|
||||||
|
restored += 1
|
||||||
|
|
||||||
|
if restored > 0:
|
||||||
|
log(f"✅ Restored {restored} entry/entries of {clean_model}.", level="INFO")
|
||||||
|
return restored
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
log(f"❌ Restore failed for {clean_model}: {e}", level="ERROR")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
def startup_audit():
|
def startup_audit():
|
||||||
"""
|
"""
|
||||||
🚀 Startup Audit:
|
🚀 Startup Audit:
|
||||||
@@ -661,10 +725,12 @@ def startup_audit():
|
|||||||
log(f"❌ Startup audit failed: {e}", level="ERROR")
|
log(f"❌ Startup audit failed: {e}", level="ERROR")
|
||||||
|
|
||||||
|
|
||||||
def _check_request_block(model_name):
|
def _check_request_block(model_name, reserve=0.0):
|
||||||
"""
|
"""
|
||||||
请求前闸门:对外模型名 → 判定是否超限。
|
请求前闸门:对外模型名 → 判定是否超限。
|
||||||
只拦截 MONITORED_MODEL_PAIRS 内的火山监控条目;newapi 等非监控渠道天然放行。
|
只拦截 MONITORED_MODEL_PAIRS 内的火山监控条目;newapi 等非监控渠道天然放行。
|
||||||
|
reserve: 本次请求的预估消耗(max_tokens 或常量兜底),usage + reserve >= limit 即拦截,
|
||||||
|
避免并发请求在闸门检查后集体撞线(限额是低频快照,用量是实时累计)。
|
||||||
返回 (clean_model, usage, limit) 表示超限,None 表示放行。任何异常 fail-open。
|
返回 (clean_model, usage, limit) 表示超限,None 表示放行。任何异常 fail-open。
|
||||||
"""
|
"""
|
||||||
if not redis_client or not LITELLM_AVAILABLE:
|
if not redis_client or not LITELLM_AVAILABLE:
|
||||||
@@ -705,7 +771,7 @@ def _check_request_block(model_name):
|
|||||||
val = redis_client.get(usage_key)
|
val = redis_client.get(usage_key)
|
||||||
usage = float(val or 0)
|
usage = float(val or 0)
|
||||||
|
|
||||||
if usage >= float(effective_limit):
|
if usage + float(reserve) >= float(effective_limit):
|
||||||
return (clean, usage, effective_limit)
|
return (clean, usage, effective_limit)
|
||||||
|
|
||||||
return None
|
return None
|
||||||
@@ -744,13 +810,24 @@ class DailyLimitHandler(CustomLogger):
|
|||||||
if not data or not data.get("model"):
|
if not data or not data.get("model"):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
blocked = _check_request_block(data["model"])
|
# 预留本次请求的预估消耗:优先真实 max_tokens(零采集损失),
|
||||||
|
# 未设置/非法时 fallback 常量,避免并发请求集体撞线超限。
|
||||||
|
try:
|
||||||
|
reserve = float(
|
||||||
|
data.get("max_tokens")
|
||||||
|
or data.get("max_completion_tokens")
|
||||||
|
or PREBLOCK_RESERVE_TOKENS
|
||||||
|
)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
reserve = float(PREBLOCK_RESERVE_TOKENS)
|
||||||
|
|
||||||
|
blocked = _check_request_block(data["model"], reserve=reserve)
|
||||||
if not blocked:
|
if not blocked:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
clean, usage, limit = blocked
|
clean, usage, limit = blocked
|
||||||
log(
|
log(
|
||||||
f"🚫 [PRE-BLOCK] {data['model']} daily limit reached ({int(usage):,}/{int(limit):,})",
|
f"🚫 [PRE-BLOCK] {data['model']} daily limit reached ({int(usage):,}+{int(reserve):,}/{int(limit):,})",
|
||||||
level="WARNING",
|
level="WARNING",
|
||||||
)
|
)
|
||||||
raise RateLimitError(
|
raise RateLimitError(
|
||||||
@@ -1435,6 +1512,11 @@ def active_monitor_loop():
|
|||||||
flush=True,
|
flush=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# 即时恢复:同步发现限额已恢复(usage < effective limit)时立刻解除软踢,
|
||||||
|
# 不等 restore_blocked_models 的 30min/4h 闸门 —— 否则日返还额度到账后
|
||||||
|
# 模型仍被禁(有额度但 no healthy deployments),直到下一次恢复检查。
|
||||||
|
restore_model_if_healthy(target, target_hashes)
|
||||||
|
|
||||||
# Track min remaining for frequency control
|
# Track min remaining for frequency control
|
||||||
configured_models_found = True
|
configured_models_found = True
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user