kim/litellm-config
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

optimize: adjust restore window to 07:00-13:00 Beijing time and optimize sleep logic for dead models

Browse Source
This commit is contained in:
kim
2026-01-29 20:27:32 +08:00
parent e21a510f0f
commit e6110cd9c1
1 changed files with 61 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
volc_limit_sync.py
+61 -67
View File
@@ -60,7 +60,7 @@ HOST = "open.volcengineapi.com"
REGION = "cn-beijing"
# Daemon Config
BUFFER = 300000 # User requested 30w buffer
BUFFER = 800000 # User requested 30w buffer
SYNC_INTERVAL = 60
CHECK_INTERVAL = 10
THRESHOLD = 50000
@@ -311,61 +311,44 @@ def get_effective_limit(clean_model, api_base, current_usage=None):
# ==========================================
# 👻 Handler Logic (DailyLimitHandler)
# ==========================================
def soft_evict_from_router(target_upstream_model, target_api_base=None):
try:
from litellm.proxy.proxy_server import llm_router
if not llm_router or not llm_router.model_list:
return
evicted = False
target_upstream_clean = clean_model_name(target_upstream_model)
for i, model_entry in enumerate(llm_router.model_list):
params = model_entry.get("litellm_params", {})
upstream_model = params.get("model", "")
current_api_base = params.get("api_base", "")
if target_upstream_clean not in clean_model_name(upstream_model):
continue
if target_api_base and current_api_base != target_api_base:
continue
current_alias = model_entry.get("model_name")
if current_alias and "blocked" not in current_alias:
log(
f"👻 [ROUTER HIDE] Blocking '{current_alias}' for {target_upstream_model}",
level="INFO",
)
model_entry["_daily_limit_backup"] = {
"model_name": current_alias,
"api_base": current_api_base,
}
llm_router.model_list[i]["model_name"] = f"{current_alias}-blocked"
llm_router.model_list[i]["litellm_params"]["api_base"] = (
"http://0.0.0.0"
)
evicted = True
if evicted:
dprint("✅ Node effectively removed from routing pool.")
except Exception as e:
print(f"❌ Router eviction failed: {e}", flush=True)
# ... (eviction logic omitted for brevity) ...
last_restore_check = 0
RESTORE_CHECK_INTERVAL = 60
def restore_blocked_models():
"""
Periodically check if blocked models have recovered quota.
Optimized for Volcengine:
- Quota usually resets in Beijing Time mornings (uncertain time).
- Logic:
- During Beijing Morning (00:00 - 14:00): Check every 30 mins.
- Other times: Check every 4 hours (low probability of reset).
"""
global last_restore_check
now = time.time()
if now - last_restore_check < RESTORE_CHECK_INTERVAL:
# Calculate Dynamic Interval based on Beijing Time
# UTC+8 Logic
utc_now_struct = time.gmtime(now)
beijing_hour = (utc_now_struct.tm_hour + 8) % 24
# Morning Window: 07:00 - 13:00
if 7 <= beijing_hour < 13:
interval = 1800 # 30 minutes
mode_str = "Morning/Window (30m)"
else:
interval = 14400 # 4 hours
mode_str = "Off-Peak (4h)"
if now - last_restore_check < interval:
return
last_restore_check = now
try:
from litellm.proxy.proxy_server import llm_router
# ... (rest of the function) ...
if not llm_router or not llm_router.model_list:
return
@@ -448,26 +431,6 @@ class DailyLimitHandler(CustomLogger):
# FILTER: Only track models explicitly discovered as VolcEngine in config.yaml
# This prevents tracking NewAPI aliases that point to other providers
current_hash = generate_api_base_hash(api_base)
# DEBUG: Trace why NewAPI models might be slipping through
if DEBUG_MODE or True: # Force print for debugging
is_managed = (clean_model, current_hash) in VOLC_MANAGED_PAIRS
if not is_managed and VOLC_MANAGED_PAIRS:
print(
f"DEBUG: Skipping {clean_model} (Hash: {current_hash}) - Not in Managed Pairs",
flush=True,
)
elif is_managed:
print(
f"DEBUG: Tracking {clean_model} (Hash: {current_hash}) - Found in Managed Pairs",
flush=True,
)
else:
print(
f"DEBUG: Permitting {clean_model} (Hash: {current_hash}) - Managed Pairs Empty!",
flush=True,
)
if (
VOLC_MANAGED_PAIRS
and (clean_model, current_hash) not in VOLC_MANAGED_PAIRS
@@ -920,16 +883,47 @@ def active_monitor_loop():
# Track min remaining for frequency control
configured_models_found = True
if remaining < temp_min_remaining:
# Only track POSITIVE remaining for frequency control
# If a model is 0 (dead), we shouldn't panic-poll the healthy ones.
# We rely on the Restore Loop to check dead models.
if remaining > 0 and remaining < temp_min_remaining:
temp_min_remaining = remaining
global_min_remaining = temp_min_remaining if model_balances else 0
# If all models are dead (temp_min_remaining is still inf), set to 0 for logic but handle gracefully
# If we found models but all were <= 0, then temp_min_remaining is inf.
# Logic:
# 1. No models found at all -> Default to Safe (300k)
# 2. Models found, but all 0 -> "All Dead Mode"
# 3. Models found, some > 0 -> Use min of >0 models
if not configured_models_found:
global_min_remaining = 300000 # Default
elif temp_min_remaining == float("inf"):
# All active models are <= 0
# Check time of day to decide if we should panic
utc_now_struct = time.gmtime(now)
beijing_hour = (utc_now_struct.tm_hour + 8) % 24
if 7 <= beijing_hour < 13:
# Morning Reset Window: Check relatively frequently
global_min_remaining = (
200000 # Pretend we have some quota to get ~5min interval
)
else:
# Afternoon/Night: Unlikely to reset
global_min_remaining = (
1000000 # Pretend high quota to sleep 60m
)
else:
global_min_remaining = temp_min_remaining
last_usage_map["global"] = current_usage_global
last_sync_time = now
if global_min_remaining > 800000:
print(
f"🌊 High Tide Mode: Sleeping for 30m (Min Rem: {int(global_min_remaining):,}) / 高水位模式：休眠 60 分钟",
f"🌊 High Tide Mode: Sleeping for 60m (Effective Min Rem: {int(global_min_remaining):,}) / 高水位模式：休眠 60 分钟",
flush=True,
)
elif global_min_remaining > 500000: