Compare commits
10 Commits
9423adeea6
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 10d3c7f88c | |||
| fafcf6e732 | |||
| f0aa78c904 | |||
| 6dfaa3f34c | |||
| 84828646bf | |||
| 5869d311c9 | |||
| 69d144ba34 | |||
| 2502a6b507 | |||
| 4479e179bb | |||
| 9eaafdec6b |
+68
-212
@@ -36,40 +36,6 @@ model_list:
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
###
|
|
||||||
|
|
||||||
|
|
||||||
#日常使用ds v3.2 ,强制think
|
|
||||||
# - model_name: deepseek-v3.2
|
|
||||||
# litellm_params:
|
|
||||||
# <<: *token_pricing
|
|
||||||
# model: volcengine/deepseek-v3-2-251201
|
|
||||||
# api_base: os.environ/HUOSHAN_BASE_URL
|
|
||||||
# api_key: os.environ/HUOSHAN_API_KEY
|
|
||||||
# 说明中建议设置30分钟(1800秒)以上,避免深度思考过程中断连接
|
|
||||||
# timeout: 1800
|
|
||||||
# 关键配置2:启用思考模式
|
|
||||||
# extra_body 中的参数会被直接合并到请求的 JSON body 中
|
|
||||||
# extra_headers:
|
|
||||||
# Connection: "close"
|
|
||||||
# extra_body:
|
|
||||||
# thinking:
|
|
||||||
# type: "enabled"
|
|
||||||
# model_info:
|
|
||||||
# max_input_tokens: 128000 # 允许的最大输入长度(单位:token)
|
|
||||||
# max_context_length: 128000 # 上下文窗口上限
|
|
||||||
|
|
||||||
# - model_name: deepseek-v3.2-nothink
|
|
||||||
# litellm_params:
|
|
||||||
# <<: *token_pricing
|
|
||||||
# model: volcengine/deepseek-v3-2-251201
|
|
||||||
# api_base: os.environ/HUOSHAN_BASE_URL
|
|
||||||
# api_key: os.environ/HUOSHAN_API_KEY
|
|
||||||
# extra_headers:
|
|
||||||
# Connection: "close"
|
|
||||||
# extra_body:
|
|
||||||
# thinking:
|
|
||||||
# type: "disabled"
|
|
||||||
|
|
||||||
|
|
||||||
- model_name: huoshan/deepseek-v4-flash
|
- model_name: huoshan/deepseek-v4-flash
|
||||||
@@ -78,102 +44,20 @@ model_list:
|
|||||||
model: volcengine/deepseek-v4-flash-ga-260731
|
model: volcengine/deepseek-v4-flash-ga-260731
|
||||||
api_base: os.environ/HUOSHAN_BASE_URL
|
api_base: os.environ/HUOSHAN_BASE_URL
|
||||||
api_key: os.environ/HUOSHAN_API_KEY
|
api_key: os.environ/HUOSHAN_API_KEY
|
||||||
|
allowed_openai_params: ["reasoning_effort"]
|
||||||
drop_params: true
|
drop_params: true
|
||||||
|
|
||||||
|
- model_name: huoshan/deepseek-v4-pro
|
||||||
|
litellm_params:
|
||||||
|
<<: *token_pricing
|
||||||
|
model: volcengine/deepseek-v4-pro-ga-260813
|
||||||
|
api_base: os.environ/HUOSHAN_BASE_URL
|
||||||
|
api_key: os.environ/HUOSHAN_API_KEY
|
||||||
|
allowed_openai_params: ["reasoning_effort"]
|
||||||
|
drop_params: true
|
||||||
|
|
||||||
|
|
||||||
- model_name: doubao-seed-1-8
|
|
||||||
litellm_params:
|
|
||||||
<<: *token_pricing
|
|
||||||
model: volcengine/doubao-seed-1-8-251228
|
|
||||||
api_base: os.environ/HUOSHAN_BASE_URL
|
|
||||||
api_key: os.environ/HUOSHAN_API_KEY
|
|
||||||
drop_params: true
|
|
||||||
extra_headers:
|
|
||||||
Connection: "close"
|
|
||||||
|
|
||||||
- model_name: doubao-seed-1-8-nothink
|
|
||||||
litellm_params:
|
|
||||||
<<: *token_pricing
|
|
||||||
model: volcengine/doubao-seed-1-8-251228
|
|
||||||
api_base: os.environ/HUOSHAN_BASE_URL
|
|
||||||
api_key: os.environ/HUOSHAN_API_KEY
|
|
||||||
extra_headers:
|
|
||||||
Connection: "close"
|
|
||||||
drop_params: true
|
|
||||||
extra_body:
|
|
||||||
reasoning_effort: "minimal"
|
|
||||||
|
|
||||||
|
|
||||||
# - model_name: doubao-seed-2-0-pro
|
|
||||||
# litellm_params:
|
|
||||||
# <<: *token_pricing
|
|
||||||
# model: volcengine/doubao-seed-2-0-pro-260215
|
|
||||||
# api_base: os.environ/HUOSHAN_BASE_URL
|
|
||||||
# api_key: os.environ/HUOSHAN_API_KEY
|
|
||||||
# timeout: 1800
|
|
||||||
# drop_params: true
|
|
||||||
# extra_headers:
|
|
||||||
# Connection: "close"
|
|
||||||
|
|
||||||
|
|
||||||
# - model_name: doubao-seed-2-0-lite
|
|
||||||
# litellm_params:
|
|
||||||
# <<: *token_pricing
|
|
||||||
# model: volcengine/doubao-seed-2-0-lite-260215
|
|
||||||
# api_base: os.environ/HUOSHAN_BASE_URL
|
|
||||||
# api_key: os.environ/HUOSHAN_API_KEY
|
|
||||||
# drop_params: true
|
|
||||||
# extra_headers:
|
|
||||||
# Connection: "close"
|
|
||||||
|
|
||||||
- model_name: doubao-seed-2-0-code
|
|
||||||
litellm_params:
|
|
||||||
<<: *token_pricing
|
|
||||||
model: volcengine/doubao-seed-2-0-code-preview-260215
|
|
||||||
api_base: os.environ/HUOSHAN_BASE_URL
|
|
||||||
api_key: os.environ/HUOSHAN_API_KEY
|
|
||||||
drop_params: true
|
|
||||||
extra_headers:
|
|
||||||
Connection: "close"
|
|
||||||
model_info:
|
|
||||||
max_input_tokens: 224000 # 允许的最大输入长度(单位:token)
|
|
||||||
max_context_length: 256000 # 上下文窗口上限
|
|
||||||
|
|
||||||
#禁用思考,用于标题等
|
|
||||||
- model_name: doubao-seed-2-0-code-nothink
|
|
||||||
litellm_params:
|
|
||||||
<<: *token_pricing
|
|
||||||
model: volcengine/doubao-seed-2-0-code-preview-260215
|
|
||||||
api_base: os.environ/HUOSHAN_BASE_URL
|
|
||||||
api_key: os.environ/HUOSHAN_API_KEY
|
|
||||||
drop_params: true
|
|
||||||
extra_headers:
|
|
||||||
Connection: "close"
|
|
||||||
extra_body:
|
|
||||||
# thinking:
|
|
||||||
# type: "disable"
|
|
||||||
reasoning_effort: "minimal"
|
|
||||||
model_info:
|
|
||||||
max_input_tokens: 224000 # 允许的最大输入长度(单位:token)
|
|
||||||
max_context_length: 256000 # 上下文窗口上限
|
|
||||||
|
|
||||||
# doubao-seed-2-0-code 强制中等思考,用作回退
|
|
||||||
- model_name: doubao-seed-2-0-code-medium
|
|
||||||
litellm_params:
|
|
||||||
<<: *token_pricing
|
|
||||||
model: volcengine/doubao-seed-2-0-code-preview-260215
|
|
||||||
api_base: os.environ/HUOSHAN_BASE_URL
|
|
||||||
api_key: os.environ/HUOSHAN_API_KEY
|
|
||||||
timeout: 1800
|
|
||||||
drop_params: true
|
|
||||||
extra_headers:
|
|
||||||
Connection: "close"
|
|
||||||
extra_body:
|
|
||||||
reasoning_effort: "medium"
|
|
||||||
model_info:
|
|
||||||
max_input_tokens: 224000 # 允许的最大输入长度(单位:token)
|
|
||||||
max_context_length: 256000 # 上下文窗口上限
|
|
||||||
|
|
||||||
- model_name: doubao-seed-2-1-turbo
|
- model_name: doubao-seed-2-1-turbo
|
||||||
litellm_params:
|
litellm_params:
|
||||||
@@ -181,6 +65,7 @@ model_list:
|
|||||||
model: volcengine/doubao-seed-2-1-turbo-260628
|
model: volcengine/doubao-seed-2-1-turbo-260628
|
||||||
api_base: os.environ/HUOSHAN_BASE_URL
|
api_base: os.environ/HUOSHAN_BASE_URL
|
||||||
api_key: os.environ/HUOSHAN_API_KEY
|
api_key: os.environ/HUOSHAN_API_KEY
|
||||||
|
allowed_openai_params: ["reasoning_effort"]
|
||||||
drop_params: true
|
drop_params: true
|
||||||
extra_headers:
|
extra_headers:
|
||||||
Connection: "close"
|
Connection: "close"
|
||||||
@@ -194,6 +79,7 @@ model_list:
|
|||||||
model: volcengine/doubao-seed-2-1-pro-260628
|
model: volcengine/doubao-seed-2-1-pro-260628
|
||||||
api_base: os.environ/HUOSHAN_BASE_URL
|
api_base: os.environ/HUOSHAN_BASE_URL
|
||||||
api_key: os.environ/HUOSHAN_API_KEY
|
api_key: os.environ/HUOSHAN_API_KEY
|
||||||
|
allowed_openai_params: ["reasoning_effort"]
|
||||||
drop_params: true
|
drop_params: true
|
||||||
extra_headers:
|
extra_headers:
|
||||||
Connection: "close"
|
Connection: "close"
|
||||||
@@ -208,6 +94,17 @@ model_list:
|
|||||||
model: volcengine/glm-5-2-260617
|
model: volcengine/glm-5-2-260617
|
||||||
api_base: os.environ/HUOSHAN_BASE_URL
|
api_base: os.environ/HUOSHAN_BASE_URL
|
||||||
api_key: os.environ/HUOSHAN_API_KEY
|
api_key: os.environ/HUOSHAN_API_KEY
|
||||||
|
# 方舟 glm-5-2-260617 支持 reasoning_effort(none/minimal/high/max,默认 high;
|
||||||
|
# 官方参数页 + 实测 curl 均确认),但 LiteLLM 的 VolcEngineChatConfig
|
||||||
|
# get_supported_openai_params() 白名单里没有它 —— 它是 openai-like 的固定
|
||||||
|
# 参数表,不含 reasoning_effort —— 客户端一传就抛 UnsupportedParamsError。
|
||||||
|
# allowed_openai_params 是官方逃生口:get_optional_params 用它在校验前
|
||||||
|
# extend supported_params,随后 _apply_openai_param_overrides 把该参数从
|
||||||
|
# non_default_params 原样写回 optional_params,最终透传到请求体。
|
||||||
|
# 不用 drop_params:那是把该模型所有不支持参数静默丢弃,会掩盖其它真错。
|
||||||
|
# 注意 thinking.type 无需此项,VolcEngineChatConfig 已原生白名单。
|
||||||
|
allowed_openai_params:
|
||||||
|
- reasoning_effort
|
||||||
extra_headers:
|
extra_headers:
|
||||||
Connection: "close"
|
Connection: "close"
|
||||||
model_info:
|
model_info:
|
||||||
@@ -215,18 +112,22 @@ model_list:
|
|||||||
max_context_length: 1024000 # 上下文窗口上限
|
max_context_length: 1024000 # 上下文窗口上限
|
||||||
|
|
||||||
#=======================豆包汇总nothink
|
#=======================豆包汇总nothink
|
||||||
|
# 【思考档位别名组】doubao-seed-nothink / -minimal / -low / -medium / -high / fanyi
|
||||||
|
# 语义:别名后缀即【默认】档位,非硬锁。
|
||||||
|
# - 客户端不传 reasoning_effort:按 extra_body 钉的档位发给方舟。
|
||||||
|
# - 客户端显式传 reasoning_effort:以客户端为准(覆盖默认档位)。
|
||||||
|
# 为什么钉不成硬锁(实测,本地 echo 抓上游 body):
|
||||||
|
# 请求带 reasoning_effort 时,Router._deployment_params_with_request_reasoning_override
|
||||||
|
# 会【无条件】剥掉部署侧 extra_body.reasoning_effort / thinking;
|
||||||
|
# 此时若没有 allowed_openai_params 放行,drop_params 又把客户端这个参数丢掉,
|
||||||
|
# 两者同归于尽 ⇒ 上游 body 两者都没有 ⇒ 方舟回落默认 high(静默失效)。
|
||||||
|
# 故必须加 allowed_openai_params:让客户端值能通过校验,使结果至少是
|
||||||
|
# 「客户端说了算」而不是「悄悄变回 high」。
|
||||||
|
# LiteLLM 当前无纯配置手段做到「不论客户端传什么一律覆盖」;要硬锁只能加
|
||||||
|
# pre-call hook(本仓库当前不采用)。
|
||||||
|
# 注:方舟 doubao 上 minimal 等价于关闭思考(实测 reasoning_tokens=0),
|
||||||
|
# 故 nothink/minimal/fanyi 三个别名默认档位统一用 minimal。
|
||||||
|
|
||||||
- model_name: doubao-seed-nothink
|
|
||||||
litellm_params:
|
|
||||||
<<: *token_pricing
|
|
||||||
model: volcengine/doubao-seed-2-0-code-preview-260215
|
|
||||||
api_base: os.environ/HUOSHAN_BASE_URL
|
|
||||||
api_key: os.environ/HUOSHAN_API_KEY
|
|
||||||
drop_params: true
|
|
||||||
extra_headers:
|
|
||||||
Connection: "close"
|
|
||||||
extra_body:
|
|
||||||
reasoning_effort: "minimal"
|
|
||||||
|
|
||||||
- model_name: doubao-seed-nothink
|
- model_name: doubao-seed-nothink
|
||||||
litellm_params:
|
litellm_params:
|
||||||
@@ -234,6 +135,7 @@ model_list:
|
|||||||
model: volcengine/doubao-seed-2-1-pro-260628
|
model: volcengine/doubao-seed-2-1-pro-260628
|
||||||
api_base: os.environ/HUOSHAN_BASE_URL
|
api_base: os.environ/HUOSHAN_BASE_URL
|
||||||
api_key: os.environ/HUOSHAN_API_KEY
|
api_key: os.environ/HUOSHAN_API_KEY
|
||||||
|
allowed_openai_params: ["reasoning_effort"]
|
||||||
drop_params: true
|
drop_params: true
|
||||||
extra_headers:
|
extra_headers:
|
||||||
Connection: "close"
|
Connection: "close"
|
||||||
@@ -246,6 +148,7 @@ model_list:
|
|||||||
model: volcengine/doubao-seed-2-1-turbo-260628
|
model: volcengine/doubao-seed-2-1-turbo-260628
|
||||||
api_base: os.environ/HUOSHAN_BASE_URL
|
api_base: os.environ/HUOSHAN_BASE_URL
|
||||||
api_key: os.environ/HUOSHAN_API_KEY
|
api_key: os.environ/HUOSHAN_API_KEY
|
||||||
|
allowed_openai_params: ["reasoning_effort"]
|
||||||
drop_params: true
|
drop_params: true
|
||||||
extra_headers:
|
extra_headers:
|
||||||
Connection: "close"
|
Connection: "close"
|
||||||
@@ -256,17 +159,6 @@ model_list:
|
|||||||
|
|
||||||
#=======================豆包汇总思考等级minimal
|
#=======================豆包汇总思考等级minimal
|
||||||
|
|
||||||
- model_name: doubao-seed-minimal
|
|
||||||
litellm_params:
|
|
||||||
<<: *token_pricing
|
|
||||||
model: volcengine/doubao-seed-2-0-code-preview-260215
|
|
||||||
api_base: os.environ/HUOSHAN_BASE_URL
|
|
||||||
api_key: os.environ/HUOSHAN_API_KEY
|
|
||||||
drop_params: true
|
|
||||||
extra_headers:
|
|
||||||
Connection: "close"
|
|
||||||
extra_body:
|
|
||||||
reasoning_effort: "minimal"
|
|
||||||
|
|
||||||
- model_name: doubao-seed-minimal
|
- model_name: doubao-seed-minimal
|
||||||
litellm_params:
|
litellm_params:
|
||||||
@@ -274,6 +166,7 @@ model_list:
|
|||||||
model: volcengine/doubao-seed-2-1-pro-260628
|
model: volcengine/doubao-seed-2-1-pro-260628
|
||||||
api_base: os.environ/HUOSHAN_BASE_URL
|
api_base: os.environ/HUOSHAN_BASE_URL
|
||||||
api_key: os.environ/HUOSHAN_API_KEY
|
api_key: os.environ/HUOSHAN_API_KEY
|
||||||
|
allowed_openai_params: ["reasoning_effort"]
|
||||||
drop_params: true
|
drop_params: true
|
||||||
extra_headers:
|
extra_headers:
|
||||||
Connection: "close"
|
Connection: "close"
|
||||||
@@ -286,6 +179,7 @@ model_list:
|
|||||||
model: volcengine/doubao-seed-2-1-turbo-260628
|
model: volcengine/doubao-seed-2-1-turbo-260628
|
||||||
api_base: os.environ/HUOSHAN_BASE_URL
|
api_base: os.environ/HUOSHAN_BASE_URL
|
||||||
api_key: os.environ/HUOSHAN_API_KEY
|
api_key: os.environ/HUOSHAN_API_KEY
|
||||||
|
allowed_openai_params: ["reasoning_effort"]
|
||||||
drop_params: true
|
drop_params: true
|
||||||
extra_headers:
|
extra_headers:
|
||||||
Connection: "close"
|
Connection: "close"
|
||||||
@@ -295,29 +189,7 @@ model_list:
|
|||||||
|
|
||||||
#=======================豆包汇总思考等级low
|
#=======================豆包汇总思考等级low
|
||||||
|
|
||||||
- model_name: doubao-seed-1-8
|
|
||||||
litellm_params:
|
|
||||||
<<: *token_pricing
|
|
||||||
model: volcengine/doubao-seed-1-8-251228
|
|
||||||
api_base: os.environ/HUOSHAN_BASE_URL
|
|
||||||
api_key: os.environ/HUOSHAN_API_KEY
|
|
||||||
drop_params: true
|
|
||||||
extra_headers:
|
|
||||||
Connection: "close"
|
|
||||||
extra_body:
|
|
||||||
reasoning_effort: "high"
|
|
||||||
|
|
||||||
- model_name: doubao-seed-low
|
|
||||||
litellm_params:
|
|
||||||
<<: *token_pricing
|
|
||||||
model: volcengine/doubao-seed-2-0-code-preview-260215
|
|
||||||
api_base: os.environ/HUOSHAN_BASE_URL
|
|
||||||
api_key: os.environ/HUOSHAN_API_KEY
|
|
||||||
drop_params: true
|
|
||||||
extra_headers:
|
|
||||||
Connection: "close"
|
|
||||||
extra_body:
|
|
||||||
reasoning_effort: "low"
|
|
||||||
|
|
||||||
- model_name: doubao-seed-low
|
- model_name: doubao-seed-low
|
||||||
litellm_params:
|
litellm_params:
|
||||||
@@ -325,6 +197,7 @@ model_list:
|
|||||||
model: volcengine/doubao-seed-2-1-pro-260628
|
model: volcengine/doubao-seed-2-1-pro-260628
|
||||||
api_base: os.environ/HUOSHAN_BASE_URL
|
api_base: os.environ/HUOSHAN_BASE_URL
|
||||||
api_key: os.environ/HUOSHAN_API_KEY
|
api_key: os.environ/HUOSHAN_API_KEY
|
||||||
|
allowed_openai_params: ["reasoning_effort"]
|
||||||
drop_params: true
|
drop_params: true
|
||||||
extra_headers:
|
extra_headers:
|
||||||
Connection: "close"
|
Connection: "close"
|
||||||
@@ -337,6 +210,7 @@ model_list:
|
|||||||
model: volcengine/doubao-seed-2-1-turbo-260628
|
model: volcengine/doubao-seed-2-1-turbo-260628
|
||||||
api_base: os.environ/HUOSHAN_BASE_URL
|
api_base: os.environ/HUOSHAN_BASE_URL
|
||||||
api_key: os.environ/HUOSHAN_API_KEY
|
api_key: os.environ/HUOSHAN_API_KEY
|
||||||
|
allowed_openai_params: ["reasoning_effort"]
|
||||||
drop_params: true
|
drop_params: true
|
||||||
extra_headers:
|
extra_headers:
|
||||||
Connection: "close"
|
Connection: "close"
|
||||||
@@ -346,17 +220,6 @@ model_list:
|
|||||||
|
|
||||||
#=======================豆包汇总思考等级medium
|
#=======================豆包汇总思考等级medium
|
||||||
|
|
||||||
- model_name: doubao-seed-medium
|
|
||||||
litellm_params:
|
|
||||||
<<: *token_pricing
|
|
||||||
model: volcengine/doubao-seed-2-0-code-preview-260215
|
|
||||||
api_base: os.environ/HUOSHAN_BASE_URL
|
|
||||||
api_key: os.environ/HUOSHAN_API_KEY
|
|
||||||
drop_params: true
|
|
||||||
extra_headers:
|
|
||||||
Connection: "close"
|
|
||||||
extra_body:
|
|
||||||
reasoning_effort: "medium"
|
|
||||||
|
|
||||||
- model_name: doubao-seed-medium
|
- model_name: doubao-seed-medium
|
||||||
litellm_params:
|
litellm_params:
|
||||||
@@ -364,6 +227,7 @@ model_list:
|
|||||||
model: volcengine/doubao-seed-2-1-pro-260628
|
model: volcengine/doubao-seed-2-1-pro-260628
|
||||||
api_base: os.environ/HUOSHAN_BASE_URL
|
api_base: os.environ/HUOSHAN_BASE_URL
|
||||||
api_key: os.environ/HUOSHAN_API_KEY
|
api_key: os.environ/HUOSHAN_API_KEY
|
||||||
|
allowed_openai_params: ["reasoning_effort"]
|
||||||
drop_params: true
|
drop_params: true
|
||||||
extra_headers:
|
extra_headers:
|
||||||
Connection: "close"
|
Connection: "close"
|
||||||
@@ -376,6 +240,7 @@ model_list:
|
|||||||
model: volcengine/doubao-seed-2-1-turbo-260628
|
model: volcengine/doubao-seed-2-1-turbo-260628
|
||||||
api_base: os.environ/HUOSHAN_BASE_URL
|
api_base: os.environ/HUOSHAN_BASE_URL
|
||||||
api_key: os.environ/HUOSHAN_API_KEY
|
api_key: os.environ/HUOSHAN_API_KEY
|
||||||
|
allowed_openai_params: ["reasoning_effort"]
|
||||||
drop_params: true
|
drop_params: true
|
||||||
extra_headers:
|
extra_headers:
|
||||||
Connection: "close"
|
Connection: "close"
|
||||||
@@ -385,17 +250,7 @@ model_list:
|
|||||||
|
|
||||||
#=======================豆包汇总思考等级high
|
#=======================豆包汇总思考等级high
|
||||||
|
|
||||||
- model_name: doubao-seed-high
|
|
||||||
litellm_params:
|
|
||||||
<<: *token_pricing
|
|
||||||
model: volcengine/doubao-seed-2-0-code-preview-260215
|
|
||||||
api_base: os.environ/HUOSHAN_BASE_URL
|
|
||||||
api_key: os.environ/HUOSHAN_API_KEY
|
|
||||||
drop_params: true
|
|
||||||
extra_headers:
|
|
||||||
Connection: "close"
|
|
||||||
extra_body:
|
|
||||||
reasoning_effort: "high"
|
|
||||||
|
|
||||||
- model_name: doubao-seed-high
|
- model_name: doubao-seed-high
|
||||||
litellm_params:
|
litellm_params:
|
||||||
@@ -403,6 +258,7 @@ model_list:
|
|||||||
model: volcengine/doubao-seed-2-1-pro-260628
|
model: volcengine/doubao-seed-2-1-pro-260628
|
||||||
api_base: os.environ/HUOSHAN_BASE_URL
|
api_base: os.environ/HUOSHAN_BASE_URL
|
||||||
api_key: os.environ/HUOSHAN_API_KEY
|
api_key: os.environ/HUOSHAN_API_KEY
|
||||||
|
allowed_openai_params: ["reasoning_effort"]
|
||||||
drop_params: true
|
drop_params: true
|
||||||
extra_headers:
|
extra_headers:
|
||||||
Connection: "close"
|
Connection: "close"
|
||||||
@@ -415,6 +271,7 @@ model_list:
|
|||||||
model: volcengine/doubao-seed-2-1-turbo-260628
|
model: volcengine/doubao-seed-2-1-turbo-260628
|
||||||
api_base: os.environ/HUOSHAN_BASE_URL
|
api_base: os.environ/HUOSHAN_BASE_URL
|
||||||
api_key: os.environ/HUOSHAN_API_KEY
|
api_key: os.environ/HUOSHAN_API_KEY
|
||||||
|
allowed_openai_params: ["reasoning_effort"]
|
||||||
drop_params: true
|
drop_params: true
|
||||||
extra_headers:
|
extra_headers:
|
||||||
Connection: "close"
|
Connection: "close"
|
||||||
@@ -427,30 +284,29 @@ model_list:
|
|||||||
- model_name: fanyi
|
- model_name: fanyi
|
||||||
litellm_params:
|
litellm_params:
|
||||||
<<: *token_pricing
|
<<: *token_pricing
|
||||||
model: volcengine/doubao-seed-1-8-251228
|
model: volcengine/doubao-seed-2-1-pro-260628
|
||||||
api_base: os.environ/HUOSHAN_BASE_URL
|
api_base: os.environ/HUOSHAN_BASE_URL
|
||||||
api_key: os.environ/HUOSHAN_API_KEY
|
api_key: os.environ/HUOSHAN_API_KEY
|
||||||
timeout: 120
|
allowed_openai_params: ["reasoning_effort"]
|
||||||
stream: false
|
|
||||||
drop_params: true
|
drop_params: true
|
||||||
extra_headers:
|
extra_headers:
|
||||||
Connection: "close"
|
Connection: "close"
|
||||||
extra_body:
|
extra_body:
|
||||||
reasoning_effort: "minimal"
|
reasoning_effort: "minimal"
|
||||||
|
|
||||||
|
- model_name: fanyi
|
||||||
|
litellm_params:
|
||||||
|
<<: *token_pricing
|
||||||
|
model: volcengine/doubao-seed-2-1-turbo-260628
|
||||||
|
api_base: os.environ/HUOSHAN_BASE_URL
|
||||||
|
api_key: os.environ/HUOSHAN_API_KEY
|
||||||
|
allowed_openai_params: ["reasoning_effort"]
|
||||||
|
drop_params: true
|
||||||
|
extra_headers:
|
||||||
|
Connection: "close"
|
||||||
|
extra_body:
|
||||||
|
reasoning_effort: "minimal"
|
||||||
|
|
||||||
# - model_name: fanyi
|
|
||||||
# litellm_params:
|
|
||||||
# <<: *token_pricing
|
|
||||||
# model: volcengine/doubao-seed-2-0-lite-260215
|
|
||||||
# api_base: os.environ/HUOSHAN_BASE_URL
|
|
||||||
# api_key: os.environ/HUOSHAN_API_KEY
|
|
||||||
# timeout: 120
|
|
||||||
# stream: false
|
|
||||||
# drop_params: true
|
|
||||||
# extra_headers:
|
|
||||||
# Connection: "close"
|
|
||||||
# extra_body:
|
|
||||||
# reasoning_effort: "minimal"
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@@ -462,7 +318,7 @@ litellm_settings:
|
|||||||
#drop_params: true #启用drop_params,这样LiteLLM会忽略不支持的参数,但参数仍然会传递给后端API
|
#drop_params: true #启用drop_params,这样LiteLLM会忽略不支持的参数,但参数仍然会传递给后端API
|
||||||
num_retries: 3 # retry call 3 times on each model_name (e.g. zephyr-beta)
|
num_retries: 3 # retry call 3 times on each model_name (e.g. zephyr-beta)
|
||||||
fallbacks: [
|
fallbacks: [
|
||||||
{"doubao-seed-high": ["deepseek-v4-pro"]}
|
#{"doubao-seed-high": ["deepseek-v4-pro"]}
|
||||||
]
|
]
|
||||||
#default_fallbacks: ["deepseek-v4-flash"]
|
#default_fallbacks: ["deepseek-v4-flash"]
|
||||||
request_timeout: 120 # raise Timeout error if call takes longer than 10s. Sets litellm.request_timeout
|
request_timeout: 120 # raise Timeout error if call takes longer than 10s. Sets litellm.request_timeout
|
||||||
|
|||||||
+21
-1
@@ -21,8 +21,28 @@ services:
|
|||||||
TZ: Asia/Shanghai
|
TZ: Asia/Shanghai
|
||||||
# ⚠️ 必须设置这个环境变量,否则 Python 找不到你的脚本
|
# ⚠️ 必须设置这个环境变量,否则 Python 找不到你的脚本
|
||||||
PYTHONPATH: "/app"
|
PYTHONPATH: "/app"
|
||||||
DATABASE_URL: "postgresql://litellm_app:shriek-sixfold-ethanol@192.168.1.34:5432/litellm"
|
# ⚠️ 必须保留 client_encoding=UTF8。
|
||||||
|
# 本库 litellm 是 SQL_ASCII 编码(server/client_encoding 均为 SQL_ASCII,
|
||||||
|
# datcollate=C)。psycopg 对 SQL_ASCII 连接的 TextLoader 直接返回 bytes
|
||||||
|
# 而非 str(psycopg/types/string.py:110-120),于是 litellm_proxy_extras
|
||||||
|
# 里 _MIGRATION_TS_RE.match(migration_name)、
|
||||||
|
# _TRANSIENT_INDEX_SUFFIX_RE.search(index.name) 等对 bytes 用字符串正则,
|
||||||
|
# 抛 TypeError: cannot use a string pattern on a bytes-like object。
|
||||||
|
# 该参数经 libpq 作为启动参数下发(fe-protocol3.c:2503),使连接按 UTF8
|
||||||
|
# 协商;psycopg 读服务端确认值决定解码(pq_ctypes.py:229-232),
|
||||||
|
# 从而正常返回 str。Prisma 侧无影响:quaint 丢弃未知参数
|
||||||
|
# (prisma-engines quaint/src/connector/postgres/url.rs:459),
|
||||||
|
# 且其驱动 tokio-postgres 本就固定发送 client_encoding=UTF8。
|
||||||
|
# 已全库校验:450 个文本列 / 43548 行 / 14.5MB 全部是合法 UTF-8,
|
||||||
|
# 不存在解码失败风险。
|
||||||
|
DATABASE_URL: "postgresql://litellm_app:shriek-sixfold-ethanol@192.168.1.34:5432/litellm?client_encoding=UTF8"
|
||||||
STORE_MODEL_IN_DB: "True"
|
STORE_MODEL_IN_DB: "True"
|
||||||
|
# v2 migration resolver 的额外兜底(可去掉以启用 v2):
|
||||||
|
# main-latest 自 2026-09-20 起默认启用 v2(上游 commit fac518dbf7),
|
||||||
|
# v2 会在启动时调用 _warn_if_db_ahead_of_head() —— 即上面那个 TypeError
|
||||||
|
# 的触发点。上面的 client_encoding 修复已从根因上解除该崩溃,
|
||||||
|
# 但 v2 还引入迁移协调锁等新机制;保守起见先沿用更新前的 v1 路径。
|
||||||
|
USE_V2_MIGRATION_RESOLVER: "false"
|
||||||
# --- Modified: 添加 Redis 配置 ---
|
# --- Modified: 添加 Redis 配置 ---
|
||||||
REDIS_HOST: "redis" # 对应下方 redis 服务的名字
|
REDIS_HOST: "redis" # 对应下方 redis 服务的名字
|
||||||
REDIS_PORT: "6379"
|
REDIS_PORT: "6379"
|
||||||
|
|||||||
+165
-11
@@ -109,6 +109,53 @@ PREBLOCK_RESERVE_TOKENS = int(os.getenv("PREBLOCK_RESERVE_TOKENS", "50000"))
|
|||||||
# max(声明值, 本下限) 兜住思考开销;2M 限额下 4096 偏差仅 0.2%。
|
# max(声明值, 本下限) 兜住思考开销;2M 限额下 4096 偏差仅 0.2%。
|
||||||
THINKING_RESERVE_FLOOR = int(os.getenv("THINKING_RESERVE_FLOOR", "4096"))
|
THINKING_RESERVE_FLOOR = int(os.getenv("THINKING_RESERVE_FLOOR", "4096"))
|
||||||
|
|
||||||
|
# 在途预留(inflight)TTL:pre-hook 放行时预扣的 reserve 记入独立
|
||||||
|
# daily_usage_inflight key,正常请求在结算回调中配对释放;若请求
|
||||||
|
# 失败/进程崩溃/回调丢失,预留量由本 TTL 自动归零,防止额度被永久
|
||||||
|
# 卡死。1 小时远大于正常结算延迟(秒级),兜底释放足够及时。
|
||||||
|
PREBLOCK_INFLIGHT_TTL = int(os.getenv("PREBLOCK_INFLIGHT_TTL", "3600"))
|
||||||
|
|
||||||
|
# pre-hook 写入请求 metadata 的预留量键名,结算回调据此配对释放 inflight。
|
||||||
|
# 以 "_" 前缀避免与用户/代理自定义 metadata 冲突。
|
||||||
|
PREBLOCK_RESERVE_META_KEY = "_volc_preblock_reserve"
|
||||||
|
|
||||||
|
# Lua 原子闸门:把"读 usage + 读 inflight + 判 limit + 预扣"合并为
|
||||||
|
# 一个 Redis 原子操作,堵住并发请求在同一 usage 快照上集体撞线的
|
||||||
|
# TOCTOU 窗口(见 docs/volc-quota-notes.md 并发击穿分析)。
|
||||||
|
# KEYS[1]=usage_key, KEYS[2]=inflight_key
|
||||||
|
# ARGV[1]=effective_limit, ARGV[2]=reserve, ARGV[3]=inflight TTL
|
||||||
|
# 返回: 1=放行(已预扣), 0=拦截(未预扣)
|
||||||
|
GATE_SCRIPT = """
|
||||||
|
local usage = tonumber(redis.call('GET', KEYS[1]) or '0')
|
||||||
|
local inflight = tonumber(redis.call('GET', KEYS[2]) or '0')
|
||||||
|
local limit = tonumber(ARGV[1])
|
||||||
|
local reserve = tonumber(ARGV[2])
|
||||||
|
if usage + inflight + reserve >= limit then
|
||||||
|
return 0
|
||||||
|
end
|
||||||
|
redis.call('INCRBY', KEYS[2], reserve)
|
||||||
|
redis.call('EXPIRE', KEYS[2], tonumber(ARGV[3]))
|
||||||
|
return 1
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Lua 原子释放:结算时配对扣减 inflight,floor 0 防止迟到结算的幽灵释放
|
||||||
|
# 误删别的请求的预留(TTL 过期后迟到 DECRBY 导致负数 → DEL 全部)。
|
||||||
|
# SET 后补 EXPIRE,保留剩余预留的 TTL(不因 SET 重置为永久)。
|
||||||
|
# KEYS[1]=inflight_key, ARGV[1]=reserve, ARGV[2]=inflight TTL
|
||||||
|
# 返回: 扣减后剩余 inflight 值(>=0)
|
||||||
|
RELEASE_SCRIPT = """
|
||||||
|
local cur = tonumber(redis.call('GET', KEYS[1]) or '0')
|
||||||
|
local reserve = tonumber(ARGV[1])
|
||||||
|
local new_val = cur - reserve
|
||||||
|
if new_val <= 0 then
|
||||||
|
redis.call('DEL', KEYS[1])
|
||||||
|
return 0
|
||||||
|
end
|
||||||
|
redis.call('SET', KEYS[1], new_val)
|
||||||
|
redis.call('EXPIRE', KEYS[1], tonumber(ARGV[2]))
|
||||||
|
return new_val
|
||||||
|
"""
|
||||||
|
|
||||||
# stable_limit 缺失时(首次同步前/新增模型/TTL 过期/Redis 数据丢失)的保守兜底限额。
|
# stable_limit 缺失时(首次同步前/新增模型/TTL 过期/Redis 数据丢失)的保守兜底限额。
|
||||||
# 免费池常态 ~2M(首开赠送/昨日采多少返多少),若 fallback 裸奔到采集上限 2M,
|
# 免费池常态 ~2M(首开赠送/昨日采多少返多少),若 fallback 裸奔到采集上限 2M,
|
||||||
# 余额不足或已耗尽的模型会被烧穿扣费。900k 对 0 余额模型仍是 9 倍优惠,
|
# 余额不足或已耗尽的模型会被烧穿扣费。900k 对 0 余额模型仍是 9 倍优惠,
|
||||||
@@ -157,9 +204,11 @@ MANUAL_MONITORED_MODEL_ALIASES = {
|
|||||||
"doubao-seed-code": "doubao-seed-code-preview-251028",
|
"doubao-seed-code": "doubao-seed-code-preview-251028",
|
||||||
"doubao-seed-translation": "doubao-seed-translation-250915",
|
"doubao-seed-translation": "doubao-seed-translation-250915",
|
||||||
"doubao-seed-2-0-code": "doubao-seed-2-0-code-preview-260215",
|
"doubao-seed-2-0-code": "doubao-seed-2-0-code-preview-260215",
|
||||||
# 火山资源包名用 MMDD(deepseek-v4-flash-0731),上游模型名用 -ga-YYMMDD(-ga-260731),
|
# 火山资源包名用 MMDD(deepseek-v4-flash-0731 / deepseek-v4-pro-0813),
|
||||||
|
# 上游模型名用 -ga-YYMMDD(-ga-260731 / -ga-260813),
|
||||||
# 两者不一致,必须手工映射,否则资源包会被 "Not in monitored models" 跳过。
|
# 两者不一致,必须手工映射,否则资源包会被 "Not in monitored models" 跳过。
|
||||||
"deepseek-v4-flash-0731": "deepseek-v4-flash-ga-260731",
|
"deepseek-v4-flash-0731": "deepseek-v4-flash-ga-260731",
|
||||||
|
"deepseek-v4-pro-0813": "deepseek-v4-pro-ga-260813",
|
||||||
}
|
}
|
||||||
|
|
||||||
# Auto-discovered monitored upstream models and endpoint pairs from config.yaml
|
# Auto-discovered monitored upstream models and endpoint pairs from config.yaml
|
||||||
@@ -186,13 +235,13 @@ HARD_DAILY_LIMITS = {
|
|||||||
# 🚨 Soft Evict Thresholds(软踢预警线)
|
# 🚨 Soft Evict Thresholds(软踢预警线)
|
||||||
# ==========================================
|
# ==========================================
|
||||||
# 提前于硬限额/动态墙踢出模型的预警线,留出缓冲:
|
# 提前于硬限额/动态墙踢出模型的预警线,留出缓冲:
|
||||||
# 1. 用量墙:usage ≥ SOFT_EVICT_USAGE_LIMIT(默认 1.95M) 即踢——避免贴 2M 边时
|
# 1. 用量墙:usage ≥ SOFT_EVICT_USAGE_LIMIT(默认 1.9M) 即踢——避免贴 2M 边时
|
||||||
# 大请求 429/小请求放行的灰色地带(reserve 低估 / 并发击穿余量)
|
# 大请求 429/小请求放行的灰色地带(reserve 低估 / 并发击穿余量)
|
||||||
# 2. 余额墙:balance_at_sync < SOFT_EVICT_BALANCE_LIMIT(10万) 且 usage > 0 即踢——
|
# 2. 余额墙:balance_at_sync < SOFT_EVICT_BALANCE_LIMIT(10万) 且 usage > 0 即踢——
|
||||||
# usage>0 限定防误杀小额初始池模型(B0<10 万未使用时被永久踢出死锁)
|
# usage>0 限定防误杀小额初始池模型(B0<10 万未使用时被永久踢出死锁)
|
||||||
# 恢复条件对称(should_restore_model):usage < min(effective_limit, 用量墙)
|
# 恢复条件对称(should_restore_model):usage < min(effective_limit, 用量墙)
|
||||||
# 且 (余额 ≥ 10万 或 usage == 0),否则踢了立即被恢复成踢/恢复抖动
|
# 且 (余额 ≥ 10万 或 usage == 0),否则踢了立即被恢复成踢/恢复抖动
|
||||||
SOFT_EVICT_USAGE_LIMIT = float(os.getenv("SOFT_EVICT_USAGE_LIMIT", "1950000"))
|
SOFT_EVICT_USAGE_LIMIT = float(os.getenv("SOFT_EVICT_USAGE_LIMIT", "1900000"))
|
||||||
SOFT_EVICT_BALANCE_LIMIT = float(os.getenv("SOFT_EVICT_BALANCE_LIMIT", "100000"))
|
SOFT_EVICT_BALANCE_LIMIT = float(os.getenv("SOFT_EVICT_BALANCE_LIMIT", "100000"))
|
||||||
|
|
||||||
# ==========================================
|
# ==========================================
|
||||||
@@ -373,6 +422,16 @@ def get_usage_key(clean_model, api_base=None):
|
|||||||
return f"daily_usage:{date_str}:{clean_model}{suffix}"
|
return f"daily_usage:{date_str}:{clean_model}{suffix}"
|
||||||
|
|
||||||
|
|
||||||
|
def get_inflight_key(clean_model, api_base=None):
|
||||||
|
"""在途预留 key:与 usage key 同日期轮换,独立账户,结算配对释放。
|
||||||
|
|
||||||
|
pre-hook 放行时 INCRBY reserve(Lua 原子闸门内完成),请求成功结算时
|
||||||
|
按该请求的 reserve DECRBY 释放,usage 只计实际消耗;失败/丢失的预留
|
||||||
|
由 PREBLOCK_INFLIGHT_TTL 自动归零。
|
||||||
|
"""
|
||||||
|
return get_usage_key(clean_model, api_base) + ":inflight"
|
||||||
|
|
||||||
|
|
||||||
def get_static_limit_for_hash(clean_model, current_hash=None):
|
def get_static_limit_for_hash(clean_model, current_hash=None):
|
||||||
"""Check static configuration in HARD_DAILY_LIMITS using an optional endpoint hash."""
|
"""Check static configuration in HARD_DAILY_LIMITS using an optional endpoint hash."""
|
||||||
if current_hash:
|
if current_hash:
|
||||||
@@ -860,13 +919,58 @@ def startup_audit():
|
|||||||
log(f"❌ Startup audit failed: {e}", level="ERROR")
|
log(f"❌ Startup audit failed: {e}", level="ERROR")
|
||||||
|
|
||||||
|
|
||||||
|
def _atomic_reserve(clean_model, api_base, effective_limit, reserve):
|
||||||
|
"""Lua 原子闸门:usage + inflight + reserve >= limit 则拦截,否则预扣并放行。
|
||||||
|
|
||||||
|
返回 (granted, usage_total)。granted=False 时 usage_total = usage + inflight
|
||||||
|
(供日志展示);granted=True 时预扣已入 inflight key。
|
||||||
|
Redis 不可用/脚本失败时抛异常由调用方 fail-closed 处理。
|
||||||
|
"""
|
||||||
|
usage_key = get_usage_key(clean_model, api_base)
|
||||||
|
inflight_key = get_inflight_key(clean_model, api_base)
|
||||||
|
granted = redis_client.eval(
|
||||||
|
GATE_SCRIPT,
|
||||||
|
2,
|
||||||
|
usage_key,
|
||||||
|
inflight_key,
|
||||||
|
int(float(effective_limit)),
|
||||||
|
int(reserve),
|
||||||
|
int(PREBLOCK_INFLIGHT_TTL),
|
||||||
|
)
|
||||||
|
usage = float(redis_client.get(usage_key) or 0)
|
||||||
|
inflight = float(redis_client.get(inflight_key) or 0)
|
||||||
|
return bool(granted), usage + inflight
|
||||||
|
|
||||||
|
|
||||||
|
def _release_inflight(clean_model, api_base, reserve):
|
||||||
|
"""结算配对释放:请求完成后把 pre-hook 预扣的 reserve 从 inflight 账户扣除。
|
||||||
|
|
||||||
|
用 Lua 原子脚本做 DECRBY + floor 0,防止 TTL 过期后迟到结算的幽灵释放
|
||||||
|
把 DECRBY 打到负数 → 误删别的请求的预留。reserve<=0 或脚本失败时无操作,
|
||||||
|
inflight 残留由 TTL 兜底归零。
|
||||||
|
"""
|
||||||
|
if not redis_client or not reserve:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
inflight_key = get_inflight_key(clean_model, api_base)
|
||||||
|
redis_client.eval(
|
||||||
|
RELEASE_SCRIPT, 1, inflight_key, int(reserve),
|
||||||
|
int(PREBLOCK_INFLIGHT_TTL),
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
log(f"❌ _release_inflight failed: {e}", level="ERROR")
|
||||||
|
|
||||||
|
|
||||||
def _check_request_block(model_name, reserve=0.0):
|
def _check_request_block(model_name, reserve=0.0):
|
||||||
"""
|
"""
|
||||||
请求前闸门:对外模型名 → 判定是否超限。
|
请求前闸门:对外模型名 → 判定是否超限。
|
||||||
只拦截 MONITORED_MODEL_PAIRS 内的火山监控条目;newapi 等非监控渠道天然放行。
|
只拦截 MONITORED_MODEL_PAIRS 内的火山监控条目;newapi 等非监控渠道天然放行。
|
||||||
reserve: 本次请求的预估消耗(max_tokens 或常量兜底),usage + reserve >= limit 即拦截,
|
reserve: 本次请求的预估消耗(max_tokens 或常量兜底)。
|
||||||
避免并发请求在闸门检查后集体撞线(限额是低频快照,用量是实时累计)。
|
判定用 Lua 原子闸门:usage + inflight(在途预留) + reserve >= limit 即拦截,
|
||||||
返回 (clean_model, usage, limit) 表示超限,None 表示放行。
|
inflight 账户在 Redis 内原子预扣,并发请求串行看到彼此的在途量,堵死
|
||||||
|
"同一 usage 快照集体撞线"的 TOCTOU 击穿。放行时 reserve 已预扣入 inflight,
|
||||||
|
由结算回调配对释放(见 _release_inflight)。
|
||||||
|
返回 (clean_model, usage, limit) 表示超限,None 表示放行(已预扣)。
|
||||||
fail-closed:Redis 不可用 / 限额获取失败 / 读取异常时,对监控条目返回
|
fail-closed:Redis 不可用 / 限额获取失败 / 读取异常时,对监控条目返回
|
||||||
(clean_model, -1, -1) 标记拒绝(宁可错杀,不可漏放);非监控条目不受影响。
|
(clean_model, -1, -1) 标记拒绝(宁可错杀,不可漏放);非监控条目不受影响。
|
||||||
手动软踢:Redis key manual_evict:{config model 字段全名} 存在时返回
|
手动软踢:Redis key manual_evict:{config model 字段全名} 存在时返回
|
||||||
@@ -973,16 +1077,24 @@ def _check_request_block(model_name, reserve=0.0):
|
|||||||
return (clean, -1, -1)
|
return (clean, -1, -1)
|
||||||
|
|
||||||
usage_key = get_usage_key(clean, api_base)
|
usage_key = get_usage_key(clean, api_base)
|
||||||
val = redis_client.get(usage_key)
|
|
||||||
usage = float(val or 0)
|
|
||||||
|
|
||||||
# 已被软踢的条目恒 429(无论 reserve),避免踢后灰色区间(如 1.95M~2M)
|
# 已被软踢的条目恒 429(无论 reserve),避免踢后灰色区间(如 1.95M~2M)
|
||||||
# 小请求被放行 → 路由原名不存在 → 400 路由错误
|
# 小请求被放行 → 路由原名不存在 → 400 路由错误
|
||||||
if entry_name == f"{model_name}-limit-reached" and "_daily_limit_backup" in entry:
|
if entry_name == f"{model_name}-limit-reached" and "_daily_limit_backup" in entry:
|
||||||
|
usage = float(redis_client.get(usage_key) or 0)
|
||||||
return (clean, usage, effective_limit)
|
return (clean, usage, effective_limit)
|
||||||
|
|
||||||
if usage + float(reserve) >= float(effective_limit):
|
# Lua 原子闸门:usage + inflight + reserve >= limit 即拒绝(不预扣),
|
||||||
return (clean, usage, effective_limit)
|
# 否则把 reserve 预扣进 inflight 账户并放行。判定与预扣在 Redis 内
|
||||||
|
# 原子完成,并发请求串行看到彼此的在途预留,堵死"同一 usage 快照上
|
||||||
|
# 集体撞线"的 TOCTOU 击穿(08-18 13:14:41-50 根因,见诊断)。
|
||||||
|
granted, _inflight_total = _atomic_reserve(
|
||||||
|
clean, api_base, effective_limit, reserve
|
||||||
|
)
|
||||||
|
if not granted:
|
||||||
|
return (clean, _inflight_total, effective_limit)
|
||||||
|
# 放行:预扣已入 inflight,结算时配对释放
|
||||||
|
return None
|
||||||
|
|
||||||
return None
|
return None
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@@ -1046,6 +1158,22 @@ class DailyLimitHandler(CustomLogger):
|
|||||||
# 输出侧:思考模型 usage 含 reasoning tokens 且不受 max_tokens 限制,
|
# 输出侧:思考模型 usage 含 reasoning tokens 且不受 max_tokens 限制,
|
||||||
# 故取 max(请求声明 max_tokens, THINKING_RESERVE_FLOOR) 兜底思考开销,
|
# 故取 max(请求声明 max_tokens, THINKING_RESERVE_FLOOR) 兜底思考开销,
|
||||||
# 防短文本 + 小 max_tokens 请求 reserve 低估(实测 34 vs 实际 150+)。
|
# 防短文本 + 小 max_tokens 请求 reserve 低估(实测 34 vs 实际 150+)。
|
||||||
|
# 幂等防重:litellm 对同一请求可能调用两次 async_pre_call_hook
|
||||||
|
# (proxy 级 + deployment 级),第二次预扣会重复占 inflight 且
|
||||||
|
# 结算只释放一次 → 残留累积。已有 reserve 标记时跳过预扣(含
|
||||||
|
# token_counter 精确计数等昂贵路径)直接放行。
|
||||||
|
# 假设:deployment-level hook 收到的是同一 data dict 引用(litellm
|
||||||
|
# pre_call_hook 循环体 data=result 赋值,就地修改会带过去)。若
|
||||||
|
# litellm 升级为深拷贝 data 传给第二次 hook,此检查失效,需改用
|
||||||
|
# 请求级唯一 ID(如 litellm_call_id)做去重。部署后留意日志确认
|
||||||
|
# 第二次 pre-hook 是否被跳过(Pre-reserve 只应打印一次)。
|
||||||
|
try:
|
||||||
|
_existing_meta = data.get("metadata") or {}
|
||||||
|
if isinstance(_existing_meta, dict) and PREBLOCK_RESERVE_META_KEY in _existing_meta:
|
||||||
|
return data
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
try:
|
try:
|
||||||
declared = (
|
declared = (
|
||||||
data.get("max_tokens")
|
data.get("max_tokens")
|
||||||
@@ -1079,7 +1207,19 @@ class DailyLimitHandler(CustomLogger):
|
|||||||
|
|
||||||
blocked = _check_request_block(data["model"], reserve=reserve)
|
blocked = _check_request_block(data["model"], reserve=reserve)
|
||||||
if not blocked:
|
if not blocked:
|
||||||
return None
|
# 放行:Lua 原子闸门已把 reserve 预扣进 inflight 账户。
|
||||||
|
# 把本次预留量记入请求 metadata,随 litellm 流转到 log_success_event
|
||||||
|
# 的 kwargs["litellm_params"]["metadata"],结算时配对释放(否则
|
||||||
|
# inflight 只会增不减,额度被已放行请求永久占死)。
|
||||||
|
try:
|
||||||
|
meta = data.get("metadata") or {}
|
||||||
|
if not isinstance(meta, dict):
|
||||||
|
meta = {}
|
||||||
|
meta[PREBLOCK_RESERVE_META_KEY] = int(reserve)
|
||||||
|
data["metadata"] = meta
|
||||||
|
except Exception:
|
||||||
|
pass # 写 metadata 失败仅影响结算释放,TTL 兜底仍会回收
|
||||||
|
return data
|
||||||
|
|
||||||
clean, usage, limit = blocked
|
clean, usage, limit = blocked
|
||||||
if usage < 0 or limit < 0:
|
if usage < 0 or limit < 0:
|
||||||
@@ -1155,6 +1295,20 @@ class DailyLimitHandler(CustomLogger):
|
|||||||
|
|
||||||
if clean_model and total_tokens > 0:
|
if clean_model and total_tokens > 0:
|
||||||
key = get_usage_key(clean_model, api_base)
|
key = get_usage_key(clean_model, api_base)
|
||||||
|
|
||||||
|
# 结算配对释放:读回 pre-hook 预扣的 reserve(经 metadata 透传),
|
||||||
|
# 从 inflight 账户扣除。实际消耗 total_tokens 照常记入 usage;
|
||||||
|
# 预扣量 = 预估值,与实耗差额天然由 usage 侧的"只计实际"修正,
|
||||||
|
# inflight 只代表"已放行未结算"的在途量,结算即清零。
|
||||||
|
released = 0
|
||||||
|
try:
|
||||||
|
meta = litellm_params.get("metadata") or {}
|
||||||
|
if isinstance(meta, dict):
|
||||||
|
released = int(meta.get(PREBLOCK_RESERVE_META_KEY, 0) or 0)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
released = 0
|
||||||
|
_release_inflight(clean_model, api_base, released)
|
||||||
|
|
||||||
current_total = redis_client.incrby(key, int(total_tokens))
|
current_total = redis_client.incrby(key, int(total_tokens))
|
||||||
redis_client.expire(key, 172800)
|
redis_client.expire(key, 172800)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user