kim/litellm-config
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: streamline monitored model maintenance

Browse Source
This commit is contained in:
kim
2026-04-01 13:29:10 +08:00
parent 63658dadd2
commit 2f1c2642e5
7 changed files with 565 additions and 259 deletions
Download Patch File Download Diff File Expand all files Collapse all files
volc_limit_sync.py
+88 -42
View File
@@ -91,24 +91,18 @@ SUFFIXES_TO_REMOVE = [
"-free-infer-res-pack", # Matches: DeepSeek-V3-free-infer-res-pack
]
# Volcengine Name -> LiteLLM Model Name Map
# Key is the normalized (lowercase, hyphens) name from Volcengine API
VOLC_MODEL_MAP = {
# Manual package-name aliases for monitored models.
# 常规版本号模型（如 xxx-260215）会根据 config.yaml 自动推导成 xxx，
# 这里只保留特殊别名或一对多映射，减少新增/删除监控模型时需要手改的地方。
MANUAL_MONITORED_MODEL_ALIASES = {
"doubao-seed-1-6": ["doubao-seed-1-6-250615", "doubao-seed-1-6-251015"],
"doubao-seed-1-8": "doubao-seed-1-8-251228",
"doubao-seed-code": "doubao-seed-code-preview-251028",
"doubao-seed-translation": "doubao-seed-translation-250915",
"deepseek-v3": "deepseek-v3-250324",
"deepseek-v3-1": "deepseek-v3-1-terminus",
"deepseek-v3-2": "deepseek-v3-2-251201",
"deepseek-r1": "deepseek-r1-250528",
"kimi-k2": "kimi-k2-thinking-251104",
"glm-4-7": "glm-4-7-251222",
}
# Whitelist of (model_name, api_base_hash) managed by VolcSync
# Populated by active_monitor_loop scanning config.yaml
VOLC_MANAGED_PAIRS = set()
# Auto-discovered monitored upstream models and endpoint pairs from config.yaml
MONITORED_MODEL_PAIRS = set()
MONITORED_MODEL_ALIASES = {}
# ==========================================
# 🛑 Hard Daily Limits (Local Override / 本地每日硬限额)
@@ -122,6 +116,9 @@ VOLC_MANAGED_PAIRS = set()
HARD_DAILY_LIMITS = {
# Specific Limit for Doubao Code
"doubao-seed-code-preview-251028": 4_800_000,
# Specific Limits for Doubao Seed 2.0 models on HUOSHAN_BASE_URL
"doubao-seed-2-0-pro-260215:ba0383d3": 1_900_000,
"doubao-seed-2-0-lite-260215:ba0383d3": 1_900_000,
# Default Limit for all other models (Universal Fallback)
"__DEFAULT__": 1_800_000,
}
@@ -189,8 +186,53 @@ def clean_model_name(name):
)
def strip_model_version_suffix(model_name):
"""Convert foo-bar-260215 -> foo-bar for common versioned upstream names."""
if not model_name:
return ""
parts = model_name.rsplit("-", 1)
if len(parts) == 2 and len(parts[1]) == 6 and parts[1].isdigit():
return parts[0]
return model_name
def build_monitored_model_aliases(monitored_models):
"""
Build alias -> monitored upstream model(s) mapping.
Common versioned names are derived automatically from config.yaml.
"""
alias_map = {}
def add_alias(alias_name, target_model):
clean_alias = clean_model_name(alias_name)
clean_target = clean_model_name(target_model)
if not clean_alias or not clean_target:
return
alias_map.setdefault(clean_alias, [])
if clean_target not in alias_map[clean_alias]:
alias_map[clean_alias].append(clean_target)
for target_model in sorted(monitored_models):
add_alias(target_model, target_model)
base_alias = strip_model_version_suffix(target_model)
if base_alias != target_model:
add_alias(base_alias, target_model)
for alias_name, targets in MANUAL_MONITORED_MODEL_ALIASES.items():
if isinstance(targets, list):
for target_model in targets:
add_alias(alias_name, target_model)
else:
add_alias(alias_name, targets)
return alias_map
def normalize_volc_name(raw_name):
"""Volcengine API response cleaning."""
"""Normalize Volcengine package names before monitored-model resolution."""
if not raw_name:
return "unknown"
clean_name = raw_name
@@ -200,7 +242,7 @@ def normalize_volc_name(raw_name):
break
clean_name = clean_name.replace("_", "-").replace(".", "-")
clean_name = clean_name.lower().strip()
return VOLC_MODEL_MAP.get(clean_name, clean_name)
return clean_name
def sign(key, msg):
@@ -573,8 +615,8 @@ class DailyLimitHandler(CustomLogger):
# This prevents tracking NewAPI aliases that point to other providers
current_hash = generate_api_base_hash(api_base)
if (
VOLC_MANAGED_PAIRS
and (clean_model, current_hash) not in VOLC_MANAGED_PAIRS
MONITORED_MODEL_PAIRS
and (clean_model, current_hash) not in MONITORED_MODEL_PAIRS
):
return
@@ -726,12 +768,9 @@ def process_packages_breakdown(packages):
available = pkg.get("AvailableAmount", 0)
unit = pkg.get("Unit", "token")
# 2. Handle 1-to-many mapping (e.g. doubao-seed-1-6 -> [v1, v2])
target_models = []
if isinstance(norm_result, list):
target_models = norm_result
else:
target_models = [norm_result]
# 2. Resolve to monitored upstream models.
# 普通版本号模型会自动从 config.yaml 推导；特殊别名走手工覆盖。
target_models = MONITORED_MODEL_ALIASES.get(norm_result, [norm_result])
# Debug log
if (
@@ -740,7 +779,7 @@ def process_packages_breakdown(packages):
or "glm" in str(target_models)
):
print(
f"DEBUG: Pkg Raw='{target_raw}' -> Norm='{norm_result}', Avail={int(float(available)):,}",
f"DEBUG: Pkg Raw='{target_raw}' -> Norm='{norm_result}' -> Targets={target_models}, Avail={int(float(available)):,}",
flush=True,
)
@@ -768,9 +807,10 @@ def process_packages_breakdown(packages):
def active_monitor_loop():
# Config Discovery
global VOLC_MANAGED_PAIRS
VOLC_MANAGED_PAIRS.clear()
INTERESTED_MODELS = set()
global MONITORED_MODEL_PAIRS, MONITORED_MODEL_ALIASES
MONITORED_MODEL_PAIRS.clear()
MONITORED_MODEL_ALIASES.clear()
MONITORED_MODELS = set()
try:
if os.path.exists("config.yaml"):
with open("config.yaml", "r", encoding="utf-8") as f:
@@ -778,7 +818,7 @@ def active_monitor_loop():
lines = content.split("\n")
print(
"Auto-discovering Volcengine models from config.yaml / 正在从 config.yaml 自动发现火山引擎模型...",
"Auto-discovering monitored Volcengine models from config.yaml / 正在从 config.yaml 自动发现纳入监控的火山模型...",
flush=True,
)
@@ -800,12 +840,12 @@ def active_monitor_loop():
and current_entry.get("upstream_model")
and current_entry.get("api_base_val")
):
INTERESTED_MODELS.add(current_entry["upstream_model"])
MONITORED_MODELS.add(current_entry["upstream_model"])
# Compute Hash
h = generate_api_base_hash(current_entry["api_base_val"])
VOLC_MANAGED_PAIRS.add((current_entry["upstream_model"], h))
MONITORED_MODEL_PAIRS.add((current_entry["upstream_model"], h))
print(
f"✅ [Config] Added Managed Pair: {current_entry['upstream_model']} | Hash: {h} | Base: {str(current_entry['api_base_val'])[:30]}...",
f"✅ [Config] Added monitored pair: {current_entry['upstream_model']} | Hash: {h} | Base: {str(current_entry['api_base_val'])[:30]}...",
flush=True,
)
@@ -866,22 +906,28 @@ def active_monitor_loop():
and current_entry.get("upstream_model")
and current_entry.get("api_base_val")
):
INTERESTED_MODELS.add(current_entry["upstream_model"])
MONITORED_MODELS.add(current_entry["upstream_model"])
h = generate_api_base_hash(current_entry["api_base_val"])
VOLC_MANAGED_PAIRS.add((current_entry["upstream_model"], h))
MONITORED_MODEL_PAIRS.add((current_entry["upstream_model"], h))
if DEBUG_MODE:
print(
f"DEBUG: Added Managed Pair: {current_entry['upstream_model']} @ {h}"
f"DEBUG: Added monitored pair: {current_entry['upstream_model']} @ {h}"
)
MONITORED_MODEL_ALIASES = build_monitored_model_aliases(MONITORED_MODELS)
print(
f"Auto-discovered / 已自动发现 {len(INTERESTED_MODELS)} Active Volcengine models: {INTERESTED_MODELS}",
f"Auto-discovered / 已自动发现 {len(MONITORED_MODELS)} 个纳入监控的火山上游模型: {MONITORED_MODELS}",
flush=True,
)
print(
f"Managed Pairs Hashes: {[p[1] for p in VOLC_MANAGED_PAIRS]}",
f"Monitored model pair hashes / 监控模型端点哈希: {[p[1] for p in MONITORED_MODEL_PAIRS]}",
flush=True,
)
if DEBUG_MODE:
print(
f"DEBUG: Monitored model aliases / 监控模型别名: {MONITORED_MODEL_ALIASES}",
flush=True,
)
except Exception as e:
print(f"Config Auto-discovery failed / 配置自动发现失败: {e}", flush=True)
@@ -986,7 +1032,7 @@ def active_monitor_loop():
burst_triggered = False
burst_reason = ""
for model in INTERESTED_MODELS:
for model in MONITORED_MODELS:
# 1. Get Current Usage
curr_usage_key = f"limit:dynamic:volcengine:usage:{model}"
curr_usage_val = redis_client.get(curr_usage_key)
@@ -1062,19 +1108,19 @@ def active_monitor_loop():
for target in targets:
# STRICT FILTER: Only sync models explicitly configured with VolcEngine Base URL
# User Requirement: Do NOT sync or limit models that share names but use other providers (e.g. NewAPI)
if target not in INTERESTED_MODELS:
if target not in MONITORED_MODELS:
if DEBUG_MODE:
print(
f"DEBUG: Skipping {target} (Not in INTERESTED_MODELS / 未在白名单)",
f"DEBUG: Skipping {target} (Not in monitored models / 未纳入监控模型)",
flush=True,
)
continue
# Fetch Current Usage from Redis (to calculate Total Pool)
# We need to find the correct hash for this model from VOLC_MANAGED_PAIRS
# We need to find the correct hash for this model from MONITORED_MODEL_PAIRS
current_usage = 0
target_hashes = [
p[1] for p in VOLC_MANAGED_PAIRS if p[0] == target
p[1] for p in MONITORED_MODEL_PAIRS if p[0] == target
]
# Sum usage across all configured endpoints for this model (usually just one)
@@ -1222,7 +1268,7 @@ def active_monitor_loop():
# Update snapshots for next iteration delta calculation
current_usage_global_snapshot = 0
for m in INTERESTED_MODELS:
for m in MONITORED_MODELS:
uk = f"limit:dynamic:volcengine:usage:{m}"
uv = redis_client.get(uk)
u_val = float(uv) if uv else 0.0