fix: 前置闸门 reserve 改用 token_counter 精确估算输入 token
reserve 从单凭 max_tokens fallback 50k 改为: reserve = litellm.token_counter(messages) + max_tokens(声明值) - 输入 token 精确计数(未知模型 fallback cl100k_base,对中文偏高更安全) - 输出用请求声明的 max_tokens,未声明时 fallback PREBLOCK_RESERVE_TOKENS - 修复 2M 采集上限被单个大请求(176k output)击穿的问题
This commit is contained in:
+18
-4
@@ -827,16 +827,30 @@ class DailyLimitHandler(CustomLogger):
|
|||||||
if not data or not data.get("model"):
|
if not data or not data.get("model"):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# 预留本次请求的预估消耗:优先真实 max_tokens(零采集损失),
|
# 预留本次请求的预估消耗 = 输入 token(精确) + 输出上限(max_tokens)
|
||||||
# 未设置/非法时 fallback 常量,避免并发请求集体撞线超限。
|
# 输入用 litellm.token_counter 精确计数(未知模型 fallback cl100k_base,
|
||||||
|
# 对中文偏高更安全);输出用请求声明的 max_tokens,未声明时 fallback 常量。
|
||||||
try:
|
try:
|
||||||
reserve = float(
|
max_out = float(
|
||||||
data.get("max_tokens")
|
data.get("max_tokens")
|
||||||
or data.get("max_completion_tokens")
|
or data.get("max_completion_tokens")
|
||||||
or PREBLOCK_RESERVE_TOKENS
|
or PREBLOCK_RESERVE_TOKENS
|
||||||
)
|
)
|
||||||
except (TypeError, ValueError):
|
except (TypeError, ValueError):
|
||||||
reserve = float(PREBLOCK_RESERVE_TOKENS)
|
max_out = float(PREBLOCK_RESERVE_TOKENS)
|
||||||
|
|
||||||
|
input_tokens = 0
|
||||||
|
try:
|
||||||
|
from litellm import token_counter as _tc
|
||||||
|
input_tokens = _tc(
|
||||||
|
model=data.get("model", ""),
|
||||||
|
messages=data.get("messages", []),
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
# token_counter 出错时保守:用 PREBLOCK_RESERVE_TOKENS 兜底输入
|
||||||
|
input_tokens = PREBLOCK_RESERVE_TOKENS
|
||||||
|
|
||||||
|
reserve = input_tokens + max_out
|
||||||
|
|
||||||
blocked = _check_request_block(data["model"], reserve=reserve)
|
blocked = _check_request_block(data["model"], reserve=reserve)
|
||||||
if not blocked:
|
if not blocked:
|
||||||
|
|||||||
Reference in New Issue
Block a user