optimize: adjust restore window to 07:00-13:00 Beijing time and optimize sleep logic for dead models

volc_limit_sync.py

@@ -60,7 +60,7 @@ HOST = "open.volcengineapi.com"
 REGION = "cn-beijing"
 
 # Daemon Config
-BUFFER = 300000  # User requested 30w buffer
+BUFFER = 800000  # User requested 30w buffer
 SYNC_INTERVAL = 60
 CHECK_INTERVAL = 10
 THRESHOLD = 50000
@@ -311,61 +311,44 @@ def get_effective_limit(clean_model, api_base, current_usage=None):
 # ==========================================
 # 👻 Handler Logic (DailyLimitHandler)
 # ==========================================
-def soft_evict_from_router(target_upstream_model, target_api_base=None):
+# ... (eviction logic omitted for brevity) ...
-    try:
-        from litellm.proxy.proxy_server import llm_router
-
-        if not llm_router or not llm_router.model_list:
-            return
-
-        evicted = False
-        target_upstream_clean = clean_model_name(target_upstream_model)
-
-        for i, model_entry in enumerate(llm_router.model_list):
-            params = model_entry.get("litellm_params", {})
-            upstream_model = params.get("model", "")
-            current_api_base = params.get("api_base", "")
-
-            if target_upstream_clean not in clean_model_name(upstream_model):
-                continue
-            if target_api_base and current_api_base != target_api_base:
-                continue
-
-            current_alias = model_entry.get("model_name")
-            if current_alias and "blocked" not in current_alias:
-                log(
-                    f"👻 [ROUTER HIDE] Blocking '{current_alias}' for {target_upstream_model}",
-                    level="INFO",
-                )
-                model_entry["_daily_limit_backup"] = {
-                    "model_name": current_alias,
-                    "api_base": current_api_base,
-                }
-                llm_router.model_list[i]["model_name"] = f"{current_alias}-blocked"
-                llm_router.model_list[i]["litellm_params"]["api_base"] = (
-                    "http://0.0.0.0"
-                )
-                evicted = True
-
-        if evicted:
-            dprint("✅ Node effectively removed from routing pool.")
-    except Exception as e:
-        print(f"❌ Router eviction failed: {e}", flush=True)
-
 
 last_restore_check = 0
-RESTORE_CHECK_INTERVAL = 60
 
 
 def restore_blocked_models():
+    """
+    Periodically check if blocked models have recovered quota.
+    Optimized for Volcengine:
+    - Quota usually resets in Beijing Time mornings (uncertain time).
+    - Logic:
+      - During Beijing Morning (00:00 - 14:00): Check every 30 mins.
+      - Other times: Check every 4 hours (low probability of reset).
+    """
     global last_restore_check
     now = time.time()
-    if now - last_restore_check < RESTORE_CHECK_INTERVAL:
+
+    # Calculate Dynamic Interval based on Beijing Time
+    # UTC+8 Logic
+    utc_now_struct = time.gmtime(now)
+    beijing_hour = (utc_now_struct.tm_hour + 8) % 24
+
+    # Morning Window: 07:00 - 13:00
+    if 7 <= beijing_hour < 13:
+        interval = 1800  # 30 minutes
+        mode_str = "Morning/Window (30m)"
+    else:
+        interval = 14400  # 4 hours
+        mode_str = "Off-Peak (4h)"
+
+    if now - last_restore_check < interval:
         return
+
     last_restore_check = now
 
     try:
         from litellm.proxy.proxy_server import llm_router
+        # ... (rest of the function) ...
 
         if not llm_router or not llm_router.model_list:
             return
@@ -448,26 +431,6 @@ class DailyLimitHandler(CustomLogger):
             # FILTER: Only track models explicitly discovered as VolcEngine in config.yaml
             # This prevents tracking NewAPI aliases that point to other providers
             current_hash = generate_api_base_hash(api_base)
-
-            # DEBUG: Trace why NewAPI models might be slipping through
-            if DEBUG_MODE or True:  # Force print for debugging
-                is_managed = (clean_model, current_hash) in VOLC_MANAGED_PAIRS
-                if not is_managed and VOLC_MANAGED_PAIRS:
-                    print(
-                        f"DEBUG: Skipping {clean_model} (Hash: {current_hash}) - Not in Managed Pairs",
-                        flush=True,
-                    )
-                elif is_managed:
-                    print(
-                        f"DEBUG: Tracking {clean_model} (Hash: {current_hash}) - Found in Managed Pairs",
-                        flush=True,
-                    )
-                else:
-                    print(
-                        f"DEBUG: Permitting {clean_model} (Hash: {current_hash}) - Managed Pairs Empty!",
-                        flush=True,
-                    )
-
             if (
                 VOLC_MANAGED_PAIRS
                 and (clean_model, current_hash) not in VOLC_MANAGED_PAIRS
@@ -920,16 +883,47 @@ def active_monitor_loop():
 
                         # Track min remaining for frequency control
                         configured_models_found = True
-                        if remaining < temp_min_remaining:
+
+                        # Only track POSITIVE remaining for frequency control
+                        # If a model is 0 (dead), we shouldn't panic-poll the healthy ones.
+                        # We rely on the Restore Loop to check dead models.
+                        if remaining > 0 and remaining < temp_min_remaining:
                             temp_min_remaining = remaining
 
-                global_min_remaining = temp_min_remaining if model_balances else 0
+                # If all models are dead (temp_min_remaining is still inf), set to 0 for logic but handle gracefully
+                # If we found models but all were <= 0, then temp_min_remaining is inf.
+
+                # Logic:
+                # 1. No models found at all -> Default to Safe (300k)
+                # 2. Models found, but all 0 -> "All Dead Mode"
+                # 3. Models found, some > 0 -> Use min of >0 models
+
+                if not configured_models_found:
+                    global_min_remaining = 300000  # Default
+                elif temp_min_remaining == float("inf"):
+                    # All active models are <= 0
+                    # Check time of day to decide if we should panic
+                    utc_now_struct = time.gmtime(now)
+                    beijing_hour = (utc_now_struct.tm_hour + 8) % 24
+                    if 7 <= beijing_hour < 13:
+                        # Morning Reset Window: Check relatively frequently
+                        global_min_remaining = (
+                            200000  # Pretend we have some quota to get ~5min interval
+                        )
+                    else:
+                        # Afternoon/Night: Unlikely to reset
+                        global_min_remaining = (
+                            1000000  # Pretend high quota to sleep 60m
+                        )
+                else:
+                    global_min_remaining = temp_min_remaining
+
                 last_usage_map["global"] = current_usage_global
                 last_sync_time = now
 
                 if global_min_remaining > 800000:
                     print(
-                        f"🌊 High Tide Mode: Sleeping for 30m (Min Rem: {int(global_min_remaining):,}) / 高水位模式：休眠 60 分钟",
+                        f"🌊 High Tide Mode: Sleeping for 60m (Effective Min Rem: {int(global_min_remaining):,}) / 高水位模式：休眠 60 分钟",
                         flush=True,
                     )
                 elif global_min_remaining > 500000: