fix: upgrade quota logic to track Total Pool (Usage + Rem) to prevent mid-day blocking
This commit is contained in:
+41
-11
@@ -60,7 +60,7 @@ HOST = "open.volcengineapi.com"
|
|||||||
REGION = "cn-beijing"
|
REGION = "cn-beijing"
|
||||||
|
|
||||||
# Daemon Config
|
# Daemon Config
|
||||||
BUFFER = 800000 # User requested 30w buffer
|
BUFFER = 300000 # User requested 30w buffer
|
||||||
SYNC_INTERVAL = 60
|
SYNC_INTERVAL = 60
|
||||||
CHECK_INTERVAL = 10
|
CHECK_INTERVAL = 10
|
||||||
THRESHOLD = 300000
|
THRESHOLD = 300000
|
||||||
@@ -973,6 +973,27 @@ def active_monitor_loop():
|
|||||||
)
|
)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
# Fetch Current Usage from Redis (to calculate Total Pool)
|
||||||
|
# We need to find the correct hash for this model from VOLC_MANAGED_PAIRS
|
||||||
|
current_usage = 0
|
||||||
|
target_hashes = [
|
||||||
|
p[1] for p in VOLC_MANAGED_PAIRS if p[0] == target
|
||||||
|
]
|
||||||
|
|
||||||
|
# Sum usage across all configured endpoints for this model (usually just one)
|
||||||
|
for h in target_hashes:
|
||||||
|
# Replicate get_usage_key logic
|
||||||
|
# UTC+1 split logic from get_usage_key
|
||||||
|
now_ts = time.time()
|
||||||
|
adjusted_time = now_ts - 3600
|
||||||
|
usage_date = time.strftime(
|
||||||
|
"%Y-%m-%d", time.gmtime(adjusted_time)
|
||||||
|
)
|
||||||
|
u_key = f"daily_usage:{usage_date}:{target}:{h}"
|
||||||
|
val = redis_client.get(u_key)
|
||||||
|
if val:
|
||||||
|
current_usage += float(val)
|
||||||
|
|
||||||
# Use normalized key for storage
|
# Use normalized key for storage
|
||||||
# Format: limit:dynamic:volcengine:remaining:{model_name}
|
# Format: limit:dynamic:volcengine:remaining:{model_name}
|
||||||
dynamic_key = f"limit:dynamic:volcengine:remaining:{target}"
|
dynamic_key = f"limit:dynamic:volcengine:remaining:{target}"
|
||||||
@@ -980,28 +1001,37 @@ def active_monitor_loop():
|
|||||||
# ==========================================
|
# ==========================================
|
||||||
# 🏔️ Peak Tracking & Stable Limit
|
# 🏔️ Peak Tracking & Stable Limit
|
||||||
# ==========================================
|
# ==========================================
|
||||||
# Track the highest remaining balance seen today.
|
# Logic Change: Track "Total Daily Budget" instead of just "Remaining".
|
||||||
# Limit = Peak * 0.85 (15% Safety Buffer)
|
# Total Budget = Current Usage + Current Remaining.
|
||||||
# This prevents "Expanding Limit" bugs caused by API latency.
|
# This handles mid-day restarts correctly.
|
||||||
peak_key = f"volc:peak_balance:{today_str}:{target}"
|
|
||||||
|
total_pool = current_usage + balance
|
||||||
|
|
||||||
|
peak_key = f"volc:peak_pool:{today_str}:{target}"
|
||||||
current_peak = float(redis_client.get(peak_key) or 0)
|
current_peak = float(redis_client.get(peak_key) or 0)
|
||||||
|
|
||||||
if balance > current_peak:
|
if total_pool > current_peak:
|
||||||
if DEBUG_MODE:
|
if DEBUG_MODE:
|
||||||
print(
|
print(
|
||||||
f"DEBUG: New Peak for {target}: {int(balance):,}",
|
f"DEBUG: New Peak Pool for {target}: {int(total_pool):,} (Usg:{int(current_usage):,} + Rem:{int(balance):,})",
|
||||||
flush=True,
|
flush=True,
|
||||||
)
|
)
|
||||||
redis_client.setex(
|
redis_client.setex(
|
||||||
peak_key, 172800, int(balance)
|
peak_key, 172800, int(total_pool)
|
||||||
) # 48h retention
|
) # 48h retention
|
||||||
|
current_peak = total_pool
|
||||||
|
|
||||||
|
# Fallback: If peak is somehow less than balance (shouldn't happen with logic above), fix it
|
||||||
|
if current_peak < balance:
|
||||||
current_peak = balance
|
current_peak = balance
|
||||||
|
|
||||||
# Calculate Stable Limit
|
# Calculate Stable Limit
|
||||||
# Logic: Limit = Peak - Buffer
|
# Limit = Peak Pool - Buffer
|
||||||
# Buffer = Max(Peak * 15%, 250,000)
|
# Buffer = Max(Peak * 15%, 250,000)
|
||||||
# This ensures at least 250k safety margin, or 15% for large packages.
|
|
||||||
safety_margin = max(current_peak * 0.15, 250000)
|
safety_margin = max(current_peak * 0.15, 250000)
|
||||||
|
|
||||||
|
# The Limit is the "Max Allowed Cumulative Usage"
|
||||||
|
# Max Usage = Total Pool - Buffer
|
||||||
stable_limit = int(max(0, current_peak - safety_margin))
|
stable_limit = int(max(0, current_peak - safety_margin))
|
||||||
|
|
||||||
stable_limit_key = (
|
stable_limit_key = (
|
||||||
@@ -1013,7 +1043,7 @@ def active_monitor_loop():
|
|||||||
|
|
||||||
if DEBUG_MODE:
|
if DEBUG_MODE:
|
||||||
print(
|
print(
|
||||||
f"DEBUG: {target} Peak={int(current_peak):,} | Buffer={int(safety_margin):,} | Limit={int(stable_limit):,}",
|
f"DEBUG: {target} PeakPool={int(current_peak):,} | Limit={int(stable_limit):,} | Usage={int(current_usage):,}",
|
||||||
flush=True,
|
flush=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user