fix: 前置闸门 reserve 改用 token_counter 精确估算输入 token
reserve 从单凭 max_tokens fallback 50k 改为: reserve = litellm.token_counter(messages) + max_tokens(声明值) - 输入 token 精确计数(未知模型 fallback cl100k_base,对中文偏高更安全) - 输出用请求声明的 max_tokens,未声明时 fallback PREBLOCK_RESERVE_TOKENS - 修复 2M 采集上限被单个大请求(176k output)击穿的问题
This commit is contained in:
+18
-4
@@ -827,16 +827,30 @@ class DailyLimitHandler(CustomLogger):
|
||||
if not data or not data.get("model"):
|
||||
return None
|
||||
|
||||
# 预留本次请求的预估消耗:优先真实 max_tokens(零采集损失),
|
||||
# 未设置/非法时 fallback 常量,避免并发请求集体撞线超限。
|
||||
# 预留本次请求的预估消耗 = 输入 token(精确) + 输出上限(max_tokens)
|
||||
# 输入用 litellm.token_counter 精确计数(未知模型 fallback cl100k_base,
|
||||
# 对中文偏高更安全);输出用请求声明的 max_tokens,未声明时 fallback 常量。
|
||||
try:
|
||||
reserve = float(
|
||||
max_out = float(
|
||||
data.get("max_tokens")
|
||||
or data.get("max_completion_tokens")
|
||||
or PREBLOCK_RESERVE_TOKENS
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
reserve = float(PREBLOCK_RESERVE_TOKENS)
|
||||
max_out = float(PREBLOCK_RESERVE_TOKENS)
|
||||
|
||||
input_tokens = 0
|
||||
try:
|
||||
from litellm import token_counter as _tc
|
||||
input_tokens = _tc(
|
||||
model=data.get("model", ""),
|
||||
messages=data.get("messages", []),
|
||||
)
|
||||
except Exception:
|
||||
# token_counter 出错时保守:用 PREBLOCK_RESERVE_TOKENS 兜底输入
|
||||
input_tokens = PREBLOCK_RESERVE_TOKENS
|
||||
|
||||
reserve = input_tokens + max_out
|
||||
|
||||
blocked = _check_request_block(data["model"], reserve=reserve)
|
||||
if not blocked:
|
||||
|
||||
Reference in New Issue
Block a user