perf: 已软踢模型短路跳过 token_counter 预估
- _soft_evicted_block: 扫描 router 中 {model}-limit-reached 条目返回
(usage, limit), 供 async_pre_call_hook 直接 429
- 省去注定被拒请求的 token_counter 调用(含多模态图片 URL 下载等昂贵路径)
- 与 _check_request_block 的 limit-reached 恒 429 双保险
- stub 验证: 已踢/未踢/不存在三场景符合预期
This commit is contained in:
@@ -403,6 +403,9 @@ env `SOFT_EVICT_USAGE_LIMIT=1950000` / `SOFT_EVICT_BALANCE_LIMIT=100000` 可调
|
|||||||
|
|
||||||
**前置联动**:`_check_request_block` 对已软踢条目(`*-limit-reached`)恒 429,防踢后
|
**前置联动**:`_check_request_block` 对已软踢条目(`*-limit-reached`)恒 429,防踢后
|
||||||
1.95M~2M 区间小请求放行 → 路由原名缺失 → 400 路由错误。
|
1.95M~2M 区间小请求放行 → 路由原名缺失 → 400 路由错误。
|
||||||
|
**短路优化**:`async_pre_call_hook` 在 `token_counter` 之前先查 `_soft_evicted_block`——
|
||||||
|
已软踢模型直接 429,跳过 token 预估(含多模态图片 URL 下载等昂贵路径),与
|
||||||
|
`_check_request_block` 的 limit-reached 恒 429 双保险。
|
||||||
|
|
||||||
**代价**:用量墙让 1.95M~2M 的 5 万采集放弃(明日返还少 5 万 ≈2.5%),换取贴边期容错;
|
**代价**:用量墙让 1.95M~2M 的 5 万采集放弃(明日返还少 5 万 ≈2.5%),换取贴边期容错;
|
||||||
余额墙对正常大池模型无影响(余额 ≥10万 不触发)。数字用例验证 2026-08-13:9/9 场景
|
余额墙对正常大池模型无影响(余额 ≥10万 不触发)。数字用例验证 2026-08-13:9/9 场景
|
||||||
|
|||||||
@@ -528,6 +528,34 @@ def should_soft_evict(clean_model, api_base, current_usage, effective_limit):
|
|||||||
return False, ""
|
return False, ""
|
||||||
|
|
||||||
|
|
||||||
|
def _soft_evicted_block(model_name):
|
||||||
|
"""
|
||||||
|
若模型已被软踢(router 中存在 {model_name}-limit-reached 条目),返回
|
||||||
|
(usage, effective_limit) 供直接 429;否则 None。用于 async_pre_call_hook 短路:
|
||||||
|
已踢模型注定被拒,跳过 litellm.token_counter 精确预估(含多模态图片下载等昂贵路径)。
|
||||||
|
"""
|
||||||
|
if not LITELLM_AVAILABLE or not redis_client:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
from litellm.proxy.proxy_server import llm_router
|
||||||
|
|
||||||
|
if not llm_router or not llm_router.model_list:
|
||||||
|
return None
|
||||||
|
for entry in llm_router.model_list:
|
||||||
|
entry_name = entry.get("model_name") or ""
|
||||||
|
if entry_name == f"{model_name}-limit-reached" and "_daily_limit_backup" in entry:
|
||||||
|
params = entry.get("litellm_params", {})
|
||||||
|
clean = clean_model_name(params.get("model", ""))
|
||||||
|
api_base = params.get("api_base", "")
|
||||||
|
usage_key = get_usage_key(clean, api_base)
|
||||||
|
usage = float(redis_client.get(usage_key) or 0)
|
||||||
|
limit = get_effective_limit(clean, api_base)
|
||||||
|
return (usage, limit)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def should_restore_model(clean_model, api_base, current_usage, effective_limit):
|
def should_restore_model(clean_model, api_base, current_usage, effective_limit):
|
||||||
"""
|
"""
|
||||||
恢复判定(与 should_soft_evict 对称,防踢/恢复抖动):
|
恢复判定(与 should_soft_evict 对称,防踢/恢复抖动):
|
||||||
@@ -913,6 +941,23 @@ class DailyLimitHandler(CustomLogger):
|
|||||||
if not data or not data.get("model"):
|
if not data or not data.get("model"):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
# 已被软踢的模型直接 429,跳过 token_counter 精确预估(注定被拒,无需计数,
|
||||||
|
# 也避免多模态图片 URL 下载等昂贵路径)。注:与 _check_request_block 的
|
||||||
|
# limit-reached 恒 429 双保险——这里短路省 token_counter,那里兜底路由阶段。
|
||||||
|
evicted = _soft_evicted_block(data.get("model"))
|
||||||
|
if evicted is not None:
|
||||||
|
usage, ev_limit = evicted
|
||||||
|
limit_disp = int(ev_limit) if ev_limit is not None else 0
|
||||||
|
log(
|
||||||
|
f"🚫 [PRE-BLOCK] {data['model']} soft-evicted, 429 (usage {int(usage):,}/{limit_disp:,})",
|
||||||
|
level="WARNING",
|
||||||
|
)
|
||||||
|
raise RateLimitError(
|
||||||
|
message=f"Daily limit reached for model '{data['model']}' ({int(usage):,}/{limit_disp:,}) (soft-evicted)",
|
||||||
|
llm_provider="volcengine",
|
||||||
|
model=data["model"],
|
||||||
|
)
|
||||||
|
|
||||||
# 预留本次请求的预估消耗 = 输入 token(精确) + 输出上限(max_tokens)。
|
# 预留本次请求的预估消耗 = 输入 token(精确) + 输出上限(max_tokens)。
|
||||||
# 输入用 litellm.token_counter 精确计数(未知模型 fallback cl100k_base,
|
# 输入用 litellm.token_counter 精确计数(未知模型 fallback cl100k_base,
|
||||||
# 对中文偏高更安全)。
|
# 对中文偏高更安全)。
|
||||||
|
|||||||
Reference in New Issue
Block a user