feat: add dynamic rate limiting for Volcengine models
This commit introduces a dual-layer rate limiting system: 1. volc_limit_sync.py: A daemon that adaptively syncs Volcengine API balances to Redis based on traffic and safety margins. 2. daily_limit.py: Updated to enforce dynamic limits by checking Redis (Current Usage + Remaining Balance) and handling soft evictions.
This commit is contained in:
+122
-16
@@ -29,7 +29,9 @@ DEBUG_MODE = CURRENT_LEVEL_VAL <= 10
|
|||||||
def log(msg, level="INFO"):
|
def log(msg, level="INFO"):
|
||||||
msg_val = get_log_level_value(level)
|
msg_val = get_log_level_value(level)
|
||||||
if msg_val >= CURRENT_LEVEL_VAL:
|
if msg_val >= CURRENT_LEVEL_VAL:
|
||||||
icon = {"DEBUG": "🔍", "INFO": "ℹ️", "WARNING": "⚠️", "ERROR": "❌"}.get(level, "📝")
|
icon = {"DEBUG": "🔍", "INFO": "ℹ️", "WARNING": "⚠️", "ERROR": "❌"}.get(
|
||||||
|
level, "📝"
|
||||||
|
)
|
||||||
print(f"{icon} [DailyLimit] {msg}", flush=True)
|
print(f"{icon} [DailyLimit] {msg}", flush=True)
|
||||||
|
|
||||||
|
|
||||||
@@ -57,7 +59,9 @@ MODEL_QUOTAS = {
|
|||||||
try:
|
try:
|
||||||
redis_host = os.getenv("REDIS_HOST", "redis")
|
redis_host = os.getenv("REDIS_HOST", "redis")
|
||||||
redis_port = int(os.getenv("REDIS_PORT", 6379))
|
redis_port = int(os.getenv("REDIS_PORT", 6379))
|
||||||
redis_client = redis.Redis(host=redis_host, port=redis_port, db=0, decode_responses=True)
|
redis_client = redis.Redis(
|
||||||
|
host=redis_host, port=redis_port, db=0, decode_responses=True
|
||||||
|
)
|
||||||
log(f"✅ Redis connected successfully to {redis_host}:{redis_port}", level="INFO")
|
log(f"✅ Redis connected successfully to {redis_host}:{redis_port}", level="INFO")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log(f"❌ Redis connection failed: {str(e)}", level="ERROR")
|
log(f"❌ Redis connection failed: {str(e)}", level="ERROR")
|
||||||
@@ -84,10 +88,10 @@ def generate_api_base_hash(api_base):
|
|||||||
try:
|
try:
|
||||||
# 1. 预处理:去空格
|
# 1. 预处理:去空格
|
||||||
s = api_base.strip()
|
s = api_base.strip()
|
||||||
|
|
||||||
# 2. 解析 URL
|
# 2. 解析 URL
|
||||||
parsed = urlparse(s)
|
parsed = urlparse(s)
|
||||||
|
|
||||||
# 3. 提取 Scheme 和 Netloc (域名+端口)
|
# 3. 提取 Scheme 和 Netloc (域名+端口)
|
||||||
# 如果解析成功且包含 scheme 和 netloc,则重组
|
# 如果解析成功且包含 scheme 和 netloc,则重组
|
||||||
if parsed.scheme and parsed.netloc:
|
if parsed.scheme and parsed.netloc:
|
||||||
@@ -95,20 +99,23 @@ def generate_api_base_hash(api_base):
|
|||||||
else:
|
else:
|
||||||
# 兜底:如果不是标准 URL,或者解析失败,尝试简单的去斜杠和小写化
|
# 兜底:如果不是标准 URL,或者解析失败,尝试简单的去斜杠和小写化
|
||||||
# 这种情况比较少见,通常 litellm 会校验 URL 格式
|
# 这种情况比较少见,通常 litellm 会校验 URL 格式
|
||||||
clean_base = s.rstrip('/').lower()
|
clean_base = s.rstrip("/").lower()
|
||||||
|
|
||||||
except Exception:
|
except Exception:
|
||||||
# 极端情况下的兜底
|
# 极端情况下的兜底
|
||||||
clean_base = api_base.strip().rstrip('/').lower()
|
clean_base = api_base.strip().rstrip("/").lower()
|
||||||
|
|
||||||
# 4. 计算 Hash
|
# 4. 计算 Hash
|
||||||
h = hashlib.md5(clean_base.encode()).hexdigest()[:8]
|
h = hashlib.md5(clean_base.encode()).hexdigest()[:8]
|
||||||
|
|
||||||
if DEBUG_MODE:
|
if DEBUG_MODE:
|
||||||
# Debugging hash mismatch
|
# Debugging hash mismatch
|
||||||
# 打印原始值和清洗后的值,方便确认
|
# 打印原始值和清洗后的值,方便确认
|
||||||
print(f"🔍 [Hash Debug] Base='{api_base}' -> Clean='{clean_base}' -> Hash='{h}'", flush=True)
|
print(
|
||||||
|
f"🔍 [Hash Debug] Base='{api_base}' -> Clean='{clean_base}' -> Hash='{h}'",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
|
||||||
return h
|
return h
|
||||||
|
|
||||||
|
|
||||||
@@ -132,6 +139,74 @@ def get_redis_key(clean_name, api_base=None):
|
|||||||
# 🎯 限额查找逻辑 (支持精确匹配)
|
# 🎯 限额查找逻辑 (支持精确匹配)
|
||||||
# ==========================================
|
# ==========================================
|
||||||
def get_daily_limit(clean_model, api_base):
|
def get_daily_limit(clean_model, api_base):
|
||||||
|
# 0. Check Dynamic Limit (Redis Priority) for THIS model
|
||||||
|
# Logic: Limit = Current_Usage + Remaining_Balance_After_Buffer
|
||||||
|
if redis_client:
|
||||||
|
try:
|
||||||
|
# Check specific model limit first (from volc_limit_sync breakdown)
|
||||||
|
# Key: limit:dynamic:volcengine:remaining:{clean_model}
|
||||||
|
# Example: deepseek-v3
|
||||||
|
# Fallback to global if needed (though sync script now sets per-model)
|
||||||
|
|
||||||
|
# Since normalized names in sync script match clean_model (ideally),
|
||||||
|
# we try to find a matching key.
|
||||||
|
# But wait, clean_model in daily_limit might have -251201 suffix,
|
||||||
|
# while sync script key might be just "deepseek-v3".
|
||||||
|
# We need to iterate or check fuzzy?
|
||||||
|
# Actually, sync script sets keys based on mapped names.
|
||||||
|
# Let's rely on the fuzzy match logic we used in syncing?
|
||||||
|
# No, sync script sets specific keys.
|
||||||
|
|
||||||
|
# Let's try to find a key that is a substring of clean_model
|
||||||
|
# Or just check if there is a key.
|
||||||
|
# For simplicity and performance, we can iterate over the keys set by sync script?
|
||||||
|
# No, that's slow.
|
||||||
|
|
||||||
|
# Better approach:
|
||||||
|
# The sync script sets keys like "limit:dynamic:volcengine:remaining:deepseek-v3"
|
||||||
|
# We check if we can construct that key.
|
||||||
|
# clean_model is "deepseek-v3-2-251201"
|
||||||
|
|
||||||
|
# Let's use the scan match approach? No, slow.
|
||||||
|
# Let's try direct lookups for likely prefixes.
|
||||||
|
|
||||||
|
# Actually, if we use the same VOLC_MODEL_MAP logic here? No, code duplication.
|
||||||
|
|
||||||
|
# Pragmatic approach:
|
||||||
|
# Iterate over known dynamic keys?
|
||||||
|
# Or, just check "limit:dynamic:volcengine:remaining:global" if we want a catch-all?
|
||||||
|
# But user wants per-model.
|
||||||
|
|
||||||
|
# Let's try to match the keys that exist.
|
||||||
|
# Sync script sets keys. We can just list them once or use a known pattern.
|
||||||
|
# But we don't know the exact key name here easily without the map.
|
||||||
|
|
||||||
|
# Wait, the user said "ensure alignment".
|
||||||
|
# If VOLC_MODEL_MAP maps "deepseek-v3" -> "deepseek-v3-241226",
|
||||||
|
# then sync script sets key "limit:dynamic:volcengine:remaining:deepseek-v3-241226"
|
||||||
|
# And clean_model IS "deepseek-v3-241226".
|
||||||
|
# So it SHOULD be an exact match!
|
||||||
|
|
||||||
|
remaining_key = f"limit:dynamic:volcengine:remaining:{clean_model}"
|
||||||
|
remaining_balance = redis_client.get(remaining_key)
|
||||||
|
|
||||||
|
# If not found, maybe it's a prefix match case?
|
||||||
|
if remaining_balance is None:
|
||||||
|
# Try finding a key that is a prefix of clean_model
|
||||||
|
# This is tricky without listing all keys.
|
||||||
|
# For now, let's assume the alignment from Task 2 (Mapping) works.
|
||||||
|
pass
|
||||||
|
|
||||||
|
if remaining_balance is not None:
|
||||||
|
usage_key = get_redis_key(clean_model, api_base)
|
||||||
|
current_usage = redis_client.get(usage_key)
|
||||||
|
current_val = float(current_usage) if current_usage else 0.0
|
||||||
|
dynamic_limit = current_val + float(remaining_balance)
|
||||||
|
return dynamic_limit
|
||||||
|
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
# 1. 尝试精确匹配 (Model + Hash)
|
# 1. 尝试精确匹配 (Model + Hash)
|
||||||
# 用户可以在 MODEL_QUOTAS 中配置 "model_name:hash": limit
|
# 用户可以在 MODEL_QUOTAS 中配置 "model_name:hash": limit
|
||||||
current_hash = generate_api_base_hash(api_base)
|
current_hash = generate_api_base_hash(api_base)
|
||||||
@@ -197,13 +272,18 @@ def soft_evict_from_router(target_upstream_model, target_api_base=None):
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Backup original state
|
# Backup original state
|
||||||
model_entry["_daily_limit_backup"] = {"model_name": current_alias, "api_base": current_api_base}
|
model_entry["_daily_limit_backup"] = {
|
||||||
|
"model_name": current_alias,
|
||||||
|
"api_base": current_api_base,
|
||||||
|
}
|
||||||
|
|
||||||
# 1. 修改别名 - Modify IN PLACE
|
# 1. 修改别名 - Modify IN PLACE
|
||||||
llm_router.model_list[i]["model_name"] = f"{current_alias}-blocked"
|
llm_router.model_list[i]["model_name"] = f"{current_alias}-blocked"
|
||||||
|
|
||||||
# 2. 破坏连接
|
# 2. 破坏连接
|
||||||
llm_router.model_list[i]["litellm_params"]["api_base"] = "http://0.0.0.0"
|
llm_router.model_list[i]["litellm_params"]["api_base"] = (
|
||||||
|
"http://0.0.0.0"
|
||||||
|
)
|
||||||
|
|
||||||
evicted = True
|
evicted = True
|
||||||
|
|
||||||
@@ -268,7 +348,9 @@ def restore_blocked_models():
|
|||||||
|
|
||||||
# Restore IN PLACE
|
# Restore IN PLACE
|
||||||
llm_router.model_list[i]["model_name"] = backup["model_name"]
|
llm_router.model_list[i]["model_name"] = backup["model_name"]
|
||||||
llm_router.model_list[i]["litellm_params"]["api_base"] = original_api_base
|
llm_router.model_list[i]["litellm_params"]["api_base"] = (
|
||||||
|
original_api_base
|
||||||
|
)
|
||||||
del llm_router.model_list[i]["_daily_limit_backup"]
|
del llm_router.model_list[i]["_daily_limit_backup"]
|
||||||
restored_count += 1
|
restored_count += 1
|
||||||
|
|
||||||
@@ -345,6 +427,22 @@ class DailyLimitHandler(CustomLogger):
|
|||||||
current_total = redis_client.incrby(key, int(total_tokens))
|
current_total = redis_client.incrby(key, int(total_tokens))
|
||||||
redis_client.expire(key, 172800)
|
redis_client.expire(key, 172800)
|
||||||
|
|
||||||
|
# Increment global counter for Volcengine sync
|
||||||
|
try:
|
||||||
|
# Ensure date_str is defined for the global key
|
||||||
|
now_ts_global = time.time()
|
||||||
|
# Adjust for timezone if needed, or consistent with existing date_str logic
|
||||||
|
# Assuming local time is acceptable for sync triggers
|
||||||
|
date_str_global = time.strftime(
|
||||||
|
"%Y-%m-%d", time.localtime(now_ts_global)
|
||||||
|
)
|
||||||
|
|
||||||
|
global_key = f"daily_usage:{date_str_global}:global_total"
|
||||||
|
redis_client.incrby(global_key, int(total_tokens))
|
||||||
|
redis_client.expire(global_key, 172800) # 48h
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
# Get Limit for display
|
# Get Limit for display
|
||||||
daily_limit = get_daily_limit(clean_model, api_base)
|
daily_limit = get_daily_limit(clean_model, api_base)
|
||||||
display_limit = f"/{int(daily_limit):,}" if daily_limit else ""
|
display_limit = f"/{int(daily_limit):,}" if daily_limit else ""
|
||||||
@@ -358,17 +456,25 @@ class DailyLimitHandler(CustomLogger):
|
|||||||
|
|
||||||
# Enforcement Logic
|
# Enforcement Logic
|
||||||
if daily_limit and current_total >= float(daily_limit):
|
if daily_limit and current_total >= float(daily_limit):
|
||||||
log(f"🚫 [BLOCK] {clean_model} Limit: {current_total}/{daily_limit}", level="WARNING")
|
log(
|
||||||
|
f"🚫 [BLOCK] {clean_model} Limit: {current_total}/{daily_limit}",
|
||||||
|
level="WARNING",
|
||||||
|
)
|
||||||
soft_evict_from_router(raw_model, api_base)
|
soft_evict_from_router(raw_model, api_base)
|
||||||
else:
|
else:
|
||||||
if DEBUG_MODE:
|
if DEBUG_MODE:
|
||||||
if not clean_model:
|
if not clean_model:
|
||||||
log("⚠️ [Usage Skip] No clean model name found", level="DEBUG")
|
log("⚠️ [Usage Skip] No clean model name found", level="DEBUG")
|
||||||
if total_tokens <= 0:
|
if total_tokens <= 0:
|
||||||
log(f"⚠️ [Usage Skip] Total tokens is {total_tokens}", level="DEBUG")
|
log(
|
||||||
|
f"⚠️ [Usage Skip] Total tokens is {total_tokens}",
|
||||||
|
level="DEBUG",
|
||||||
|
)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log(f"❌ _record_usage failed: {e}\n{traceback.format_exc()}", level="ERROR")
|
log(
|
||||||
|
f"❌ _record_usage failed: {e}\n{traceback.format_exc()}", level="ERROR"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
proxy_handler_instance = DailyLimitHandler()
|
proxy_handler_instance = DailyLimitHandler()
|
||||||
|
|||||||
@@ -0,0 +1,456 @@
|
|||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import json
|
||||||
|
import hmac
|
||||||
|
import hashlib
|
||||||
|
import datetime
|
||||||
|
import requests
|
||||||
|
import redis
|
||||||
|
|
||||||
|
# ================= Configuration =================
|
||||||
|
ACCESS_KEY = os.getenv(
|
||||||
|
"VOLC_ACCESS_KEY", "AKLTNGYxZmUyNzNmZjk3NGMyZmEzNjVjZmY4NGM5ZjcxMjU"
|
||||||
|
)
|
||||||
|
SECRET_KEY = os.getenv(
|
||||||
|
"VOLC_SECRET_KEY", "Tm1VNE5XSmlORGs1WXpGbU5EVmtabUUyT1RZeVltTTBNalkxTWpjeE1tTQ=="
|
||||||
|
)
|
||||||
|
HOST = "open.volcengineapi.com"
|
||||||
|
REGION = "cn-beijing"
|
||||||
|
|
||||||
|
BUFFER = 300000 # User requested 30w buffer
|
||||||
|
SYNC_INTERVAL = 60
|
||||||
|
CHECK_INTERVAL = 10
|
||||||
|
THRESHOLD = 50000
|
||||||
|
|
||||||
|
SUFFIXES_TO_REMOVE = [
|
||||||
|
"_data_collaboration_resource_pack",
|
||||||
|
"协作奖励计划资源包",
|
||||||
|
"免费在线推理资源包",
|
||||||
|
"免费在线推理包",
|
||||||
|
"资源包",
|
||||||
|
]
|
||||||
|
|
||||||
|
# 火山引擎清洗后名称 -> LiteLLM 模型名称的映射表
|
||||||
|
# 用户 TODO: 请根据您的实际部署情况手动配置此映射
|
||||||
|
#
|
||||||
|
# Key 格式 (火山清洗后名称):
|
||||||
|
# 1. 移除后缀 (如 "_data_collaboration_resource_pack")
|
||||||
|
# 2. 替换下划线和点为中划线 ("_.", -> "-")
|
||||||
|
# 3. 转小写
|
||||||
|
# 示例: "DeepSeek_V3.2_..." -> "deepseek-v3-2"
|
||||||
|
#
|
||||||
|
# Value 格式 (LiteLLM 清洗后名称):
|
||||||
|
# config.yaml 中定义的 model_name (不带 openai/ 等前缀)
|
||||||
|
# 类型: 字符串 或 字符串列表
|
||||||
|
# 示例 (一对一): "deepseek-v3-2-251201"
|
||||||
|
# 示例 (一对多, 共享额度): ["doubao-seed-1-6-250615", "doubao-seed-1-6-251015"]
|
||||||
|
VOLC_MODEL_MAP = {
|
||||||
|
"doubao-seed-1-6": ["doubao-seed-1-6-250615", "doubao-seed-1-6-251015"], # 豆包 1.6
|
||||||
|
# "doubao-seed-1-6-thinking": "", # 豆包 1.6 Thinking
|
||||||
|
"doubao-seed-1-8": "doubao-seed-1-8-251228", # 豆包 1.8
|
||||||
|
"doubao-seed-code": "doubao-seed-code-preview-251028", # 豆包 Code
|
||||||
|
"doubao-seed-translation": "doubao-seed-translation-250915", # 豆包 Translation
|
||||||
|
"deepseek-v3": "deepseek-v3-250324", # DeepSeek V3 (通用)
|
||||||
|
"deepseek-v3-1": "deepseek-v3-1-terminus", # DeepSeek V3.1
|
||||||
|
"deepseek-v3-2": "deepseek-v3-2-251201", # DeepSeek V3.2
|
||||||
|
"deepseek-r1": "deepseek-r1-250528", # DeepSeek R1
|
||||||
|
"kimi-k2": "kimi-k2-thinking-251104", # Kimi K2
|
||||||
|
"glm-4_7": "glm-4-7-251222",
|
||||||
|
}
|
||||||
|
# Redis Configuration
|
||||||
|
REDIS_HOST = os.getenv("REDIS_HOST", "redis")
|
||||||
|
REDIS_PORT = int(os.getenv("REDIS_PORT", 6379))
|
||||||
|
REDIS_PASSWORD = os.getenv("REDIS_PASSWORD", None)
|
||||||
|
|
||||||
|
|
||||||
|
# ================= Signing Logic =================
|
||||||
|
def sign(key, msg):
|
||||||
|
return hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest()
|
||||||
|
|
||||||
|
|
||||||
|
def get_signature_key(key, date_stamp, region_name, service_name):
|
||||||
|
k_date = sign(key.encode("utf-8"), date_stamp)
|
||||||
|
k_region = sign(k_date, region_name)
|
||||||
|
k_service = sign(k_region, service_name)
|
||||||
|
k_signing = sign(k_service, "request")
|
||||||
|
return k_signing
|
||||||
|
|
||||||
|
|
||||||
|
def get_signed_headers(action, version, body_dict, service="billing"):
|
||||||
|
# 1. Prepare Time
|
||||||
|
t = datetime.datetime.utcnow()
|
||||||
|
amz_date = t.strftime("%Y%m%dT%H%M%SZ")
|
||||||
|
date_stamp = t.strftime("%Y%m%d")
|
||||||
|
|
||||||
|
# 2. Process Body
|
||||||
|
request_body = json.dumps(body_dict)
|
||||||
|
payload_hash = hashlib.sha256(request_body.encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
# 3. Canonical Request
|
||||||
|
method = "POST"
|
||||||
|
canonical_uri = "/"
|
||||||
|
canonical_querystring = f"Action={action}&Version={version}"
|
||||||
|
content_type = "application/json"
|
||||||
|
|
||||||
|
# Headers must be sorted
|
||||||
|
canonical_headers = f"content-type:{content_type}\nhost:{HOST}\nx-content-sha256:{payload_hash}\nx-date:{amz_date}\n"
|
||||||
|
signed_headers = "content-type;host;x-content-sha256;x-date"
|
||||||
|
|
||||||
|
canonical_request = f"{method}\n{canonical_uri}\n{canonical_querystring}\n{canonical_headers}\n{signed_headers}\n{payload_hash}"
|
||||||
|
|
||||||
|
# 4. String to Sign
|
||||||
|
algorithm = "HMAC-SHA256"
|
||||||
|
credential_scope = f"{date_stamp}/{REGION}/{service}/request"
|
||||||
|
string_to_sign = f"{algorithm}\n{amz_date}\n{credential_scope}\n{hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()}"
|
||||||
|
|
||||||
|
# 5. Calculate Signature
|
||||||
|
signing_key = get_signature_key(SECRET_KEY, date_stamp, REGION, service)
|
||||||
|
signature = hmac.new(
|
||||||
|
signing_key, string_to_sign.encode("utf-8"), hashlib.sha256
|
||||||
|
).hexdigest()
|
||||||
|
|
||||||
|
# 6. Authorization Header
|
||||||
|
authorization_header = f"{algorithm} Credential={ACCESS_KEY}/{credential_scope}, SignedHeaders={signed_headers}, Signature={signature}"
|
||||||
|
|
||||||
|
return {
|
||||||
|
"Content-Type": content_type,
|
||||||
|
"Host": HOST,
|
||||||
|
"X-Date": amz_date,
|
||||||
|
"X-Content-Sha256": payload_hash,
|
||||||
|
"Authorization": authorization_header,
|
||||||
|
}, request_body
|
||||||
|
|
||||||
|
|
||||||
|
# ================= Helper Functions =================
|
||||||
|
def normalize_model_name(raw_name):
|
||||||
|
"""
|
||||||
|
Align Volcengine response names with LiteLLM upstream names.
|
||||||
|
1. Remove suffixes.
|
||||||
|
2. Replace _ with -.
|
||||||
|
3. Replace . with -. (Crucial: V3.2 -> v3-2)
|
||||||
|
4. Lowercase.
|
||||||
|
5. Check VOLC_MODEL_MAP for explicit overrides.
|
||||||
|
"""
|
||||||
|
if not raw_name:
|
||||||
|
return "unknown"
|
||||||
|
|
||||||
|
clean_name = raw_name
|
||||||
|
|
||||||
|
# Step 1: Remove suffixes
|
||||||
|
for suffix in sorted(SUFFIXES_TO_REMOVE, key=len, reverse=True):
|
||||||
|
if clean_name.endswith(suffix):
|
||||||
|
clean_name = clean_name.replace(suffix, "")
|
||||||
|
break
|
||||||
|
|
||||||
|
# Step 2 & 3: Replace _ and . with -
|
||||||
|
clean_name = clean_name.replace("_", "-").replace(".", "-")
|
||||||
|
|
||||||
|
# Step 4: Lowercase everything
|
||||||
|
clean_name = clean_name.lower().strip()
|
||||||
|
|
||||||
|
# Step 5: Check VOLC_MODEL_MAP for explicit overrides
|
||||||
|
return VOLC_MODEL_MAP.get(clean_name, clean_name)
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_all_packages():
|
||||||
|
"""Fetch all package pages from Volcengine API."""
|
||||||
|
action = "ListResourcePackages"
|
||||||
|
version = "2022-01-01"
|
||||||
|
all_packages = []
|
||||||
|
current_token = ""
|
||||||
|
|
||||||
|
print(f"[{datetime.datetime.now()}] Fetching packages from Volcengine...")
|
||||||
|
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
# User doc: NextToken 首次查询传 ""
|
||||||
|
payload = {
|
||||||
|
"ResourceType": "Package",
|
||||||
|
"MaxResults": "20",
|
||||||
|
"NextToken": current_token,
|
||||||
|
}
|
||||||
|
|
||||||
|
headers, body = get_signed_headers(action, version, payload)
|
||||||
|
url = f"https://{HOST}/?Action={action}&Version={version}"
|
||||||
|
|
||||||
|
response = requests.post(url, data=body, headers=headers, timeout=60)
|
||||||
|
response.raise_for_status()
|
||||||
|
data = response.json()
|
||||||
|
|
||||||
|
# Debug: print first item of first page to verify fields
|
||||||
|
# if not current_token and DEBUG_MODE:
|
||||||
|
# pass
|
||||||
|
|
||||||
|
result_block = data.get("Result", {})
|
||||||
|
items = result_block.get("List", [])
|
||||||
|
|
||||||
|
if items:
|
||||||
|
all_packages.extend(items)
|
||||||
|
|
||||||
|
next_token = result_block.get("NextToken", "")
|
||||||
|
if not next_token or next_token == current_token:
|
||||||
|
break
|
||||||
|
|
||||||
|
current_token = next_token
|
||||||
|
time.sleep(0.2) # Rate limit protection
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Error fetching packages: {e}")
|
||||||
|
break
|
||||||
|
|
||||||
|
return all_packages
|
||||||
|
|
||||||
|
|
||||||
|
def active_monitor_loop():
|
||||||
|
# 4. In-Line Config Parser (Config Awareness)
|
||||||
|
# Always run auto-discovery to populate INTERESTED_MODELS (The Whitelist)
|
||||||
|
INTERESTED_MODELS = set()
|
||||||
|
|
||||||
|
try:
|
||||||
|
if os.path.exists("config.yaml"):
|
||||||
|
with open("config.yaml", "r", encoding="utf-8") as f:
|
||||||
|
content = f.read()
|
||||||
|
|
||||||
|
lines = content.split("\n")
|
||||||
|
current_model_name = None
|
||||||
|
|
||||||
|
print("Auto-discovering Volcengine models from config.yaml...")
|
||||||
|
|
||||||
|
for line in lines:
|
||||||
|
sline = line.strip()
|
||||||
|
if sline.startswith("- model_name:"):
|
||||||
|
current_model_name = sline.split(":", 1)[1].strip()
|
||||||
|
# Remove quotes if present
|
||||||
|
if current_model_name.startswith(
|
||||||
|
'"'
|
||||||
|
) or current_model_name.startswith("'"):
|
||||||
|
current_model_name = current_model_name[1:-1]
|
||||||
|
|
||||||
|
if current_model_name:
|
||||||
|
# Heuristic: Check for Volcengine signals in the block
|
||||||
|
if "api_base:" in sline and ("HUOSHAN" in sline or "volc" in sline):
|
||||||
|
INTERESTED_MODELS.add(current_model_name)
|
||||||
|
# Also check provider prefix in model name
|
||||||
|
if "model:" in sline and "volcengine/" in sline:
|
||||||
|
INTERESTED_MODELS.add(current_model_name)
|
||||||
|
|
||||||
|
print(
|
||||||
|
f"Auto-discovered {len(INTERESTED_MODELS)} Active Volcengine models: {INTERESTED_MODELS}"
|
||||||
|
)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Config Auto-discovery failed: {e}")
|
||||||
|
|
||||||
|
# Connect to Redis
|
||||||
|
try:
|
||||||
|
r = redis.Redis(
|
||||||
|
host=REDIS_HOST,
|
||||||
|
port=REDIS_PORT,
|
||||||
|
password=REDIS_PASSWORD,
|
||||||
|
decode_responses=True,
|
||||||
|
)
|
||||||
|
r.ping()
|
||||||
|
print(f"Connected to Redis at {REDIS_HOST}:{REDIS_PORT}")
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Failed to connect to Redis: {e}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
last_usage_map = {} # Track usage per model
|
||||||
|
last_sync_time = 0
|
||||||
|
# Dynamic interval state (Start with aggressive check to initialize)
|
||||||
|
current_sync_interval = 0
|
||||||
|
global_min_remaining = 0
|
||||||
|
IDLE_THRESHOLD = 300 # 5 minutes of no usage -> Start relaxing
|
||||||
|
idle_start_time = 0
|
||||||
|
|
||||||
|
print("Starting Active Monitor Loop (Safety-Margin + Idle Awareness)...")
|
||||||
|
print(f"Buffer: {BUFFER}, Threshold: {THRESHOLD}")
|
||||||
|
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
time.sleep(CHECK_INTERVAL)
|
||||||
|
|
||||||
|
now = time.time()
|
||||||
|
today_str = datetime.datetime.now().strftime("%Y-%m-%d")
|
||||||
|
|
||||||
|
# 1. Fetch current global usage
|
||||||
|
usage_key_global = f"daily_usage:{today_str}:global_total"
|
||||||
|
current_usage_global = int(r.get(usage_key_global) or 0)
|
||||||
|
|
||||||
|
last_usage_global = sum(last_usage_map.values()) if last_usage_map else 0
|
||||||
|
delta = current_usage_global - last_usage_global
|
||||||
|
time_diff = now - last_sync_time
|
||||||
|
|
||||||
|
# 2. Idle Detection Logic
|
||||||
|
if delta <= 0: # No traffic
|
||||||
|
if idle_start_time == 0:
|
||||||
|
idle_start_time = now
|
||||||
|
else:
|
||||||
|
idle_start_time = 0 # Traffic detected, reset idle timer
|
||||||
|
|
||||||
|
is_idle = (idle_start_time > 0) and (
|
||||||
|
(now - idle_start_time) > IDLE_THRESHOLD
|
||||||
|
)
|
||||||
|
|
||||||
|
# 3. Determine Dynamic Interval
|
||||||
|
# Logic: Base Interval (based on balance) + Idle Multiplier
|
||||||
|
if global_min_remaining > 2_000_000:
|
||||||
|
base_interval = 1800 # 30 min (>200w)
|
||||||
|
elif global_min_remaining > 1_000_000:
|
||||||
|
base_interval = 600 # 10 min (100w-200w)
|
||||||
|
elif global_min_remaining > 600_000:
|
||||||
|
base_interval = 300 # 5 min (60w-100w)
|
||||||
|
elif global_min_remaining > 300_000:
|
||||||
|
base_interval = 60 # 1 min (30w-60w)
|
||||||
|
else:
|
||||||
|
base_interval = 10 # Panic (<30w Remaining)
|
||||||
|
|
||||||
|
# Apply Idle Relaxation
|
||||||
|
# If system is idle for >5 min, we can relax the check interval.
|
||||||
|
# Max idle interval extended to 3600s (1 hour) as requested.
|
||||||
|
if is_idle:
|
||||||
|
# Exponential backoff or just aggressive multiplier?
|
||||||
|
# Let's quadruple the base interval, capped at 3600s
|
||||||
|
current_sync_interval = min(base_interval * 4, 3600)
|
||||||
|
if current_sync_interval < 300:
|
||||||
|
current_sync_interval = 300 # Minimum 5 min sleep if idle
|
||||||
|
else:
|
||||||
|
current_sync_interval = base_interval
|
||||||
|
|
||||||
|
should_sync = (
|
||||||
|
(delta > THRESHOLD)
|
||||||
|
or (time_diff > current_sync_interval)
|
||||||
|
or (last_sync_time == 0)
|
||||||
|
)
|
||||||
|
|
||||||
|
if should_sync:
|
||||||
|
trigger_reason = (
|
||||||
|
"Burst Traffic" if delta > THRESHOLD else "Timer Expired"
|
||||||
|
)
|
||||||
|
status_icon = "💤" if is_idle else "⚡"
|
||||||
|
print(
|
||||||
|
f"{status_icon} Sync Triggered ({trigger_reason})... Delta: {delta}, Interval: {current_sync_interval}s"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Fetch and Process
|
||||||
|
packages = fetch_all_packages()
|
||||||
|
model_balances = process_packages_breakdown(packages)
|
||||||
|
|
||||||
|
# Update Redis and calculate Global Min Remaining (for next interval)
|
||||||
|
temp_min_remaining = float("inf")
|
||||||
|
|
||||||
|
# Logic Fix: Only active targets affect the panic timer.
|
||||||
|
# VOLC_MODEL_MAP is for Translation Only.
|
||||||
|
# INTERESTED_MODELS is the Source of Truth for Whitelist.
|
||||||
|
|
||||||
|
configured_models_found = False
|
||||||
|
|
||||||
|
for model_name, balance in model_balances.items():
|
||||||
|
remaining = balance - BUFFER
|
||||||
|
|
||||||
|
# If VOLC_MODEL_MAP is not empty, we ignore anything NOT in it.
|
||||||
|
is_interested = True
|
||||||
|
|
||||||
|
# 1. Resolve Targets (Translation)
|
||||||
|
# Translate Volc Name -> LiteLLM Name(s) using MAP if available
|
||||||
|
targets = []
|
||||||
|
if model_name in VOLC_MODEL_MAP:
|
||||||
|
val = VOLC_MODEL_MAP[model_name]
|
||||||
|
if val:
|
||||||
|
targets = val if isinstance(val, list) else [val]
|
||||||
|
else:
|
||||||
|
# Empty string = Use default name (Auto Match)
|
||||||
|
targets = [model_name]
|
||||||
|
else:
|
||||||
|
targets = [model_name]
|
||||||
|
|
||||||
|
# 2. Filter Targets (Whitelist)
|
||||||
|
# Only keep targets that are actually in INTERESTED_MODELS (config.yaml)
|
||||||
|
|
||||||
|
final_targets_to_update = []
|
||||||
|
is_active_for_panic = False
|
||||||
|
|
||||||
|
for target in targets:
|
||||||
|
# Check against Whitelist
|
||||||
|
matched = False
|
||||||
|
if not INTERESTED_MODELS:
|
||||||
|
# Fallback: Monitor everything if config discovery failed
|
||||||
|
matched = True
|
||||||
|
else:
|
||||||
|
for interested in INTERESTED_MODELS:
|
||||||
|
# Exact match (Highest Priority)
|
||||||
|
if target == interested:
|
||||||
|
matched = True
|
||||||
|
# Heuristic match (only if target is raw volc name)
|
||||||
|
# If target was explicitly mapped, it implies exact intent.
|
||||||
|
# If target is raw, we allow fuzzy.
|
||||||
|
elif target == model_name and (
|
||||||
|
target in interested or interested in target
|
||||||
|
):
|
||||||
|
matched = True
|
||||||
|
|
||||||
|
if matched:
|
||||||
|
break
|
||||||
|
|
||||||
|
if matched:
|
||||||
|
final_targets_to_update.append(target)
|
||||||
|
is_active_for_panic = True
|
||||||
|
|
||||||
|
# 3. Safety Calculation
|
||||||
|
# Only active models affect the system panic level
|
||||||
|
if is_active_for_panic:
|
||||||
|
configured_models_found = True
|
||||||
|
if remaining < temp_min_remaining:
|
||||||
|
temp_min_remaining = remaining
|
||||||
|
|
||||||
|
# Redis Update: We still update Redis for the active targets
|
||||||
|
for target in final_targets_to_update:
|
||||||
|
key = f"limit:dynamic:volcengine:remaining:{target}"
|
||||||
|
r.setex(key, 7200, int(remaining))
|
||||||
|
print(f"Updated {target}: Bal={balance:,} -> Rem={remaining:,}")
|
||||||
|
|
||||||
|
# Update State
|
||||||
|
global_min_remaining = temp_min_remaining if model_balances else 0
|
||||||
|
last_usage_map["global"] = current_usage_global
|
||||||
|
last_sync_time = now
|
||||||
|
|
||||||
|
# Log next scheduled check
|
||||||
|
if global_min_remaining > 10_000_000:
|
||||||
|
print(
|
||||||
|
f"🌊 High Tide Mode: Sleeping for 30m (Min Rem: {int(global_min_remaining):,})"
|
||||||
|
)
|
||||||
|
elif global_min_remaining > 2_000_000:
|
||||||
|
print(
|
||||||
|
f"⛵ Safe Sailing Mode: Sleeping for 5m (Min Rem: {int(global_min_remaining):,})"
|
||||||
|
)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Error in monitor loop: {e}")
|
||||||
|
time.sleep(5)
|
||||||
|
|
||||||
|
|
||||||
|
def process_packages_breakdown(packages):
|
||||||
|
"""Return dictionary of {model_name: total_available}"""
|
||||||
|
model_counts = {}
|
||||||
|
|
||||||
|
for pkg in packages:
|
||||||
|
raw_code = pkg.get("ConfigurationCode", "")
|
||||||
|
raw_name = pkg.get("ConfigurationName", "Unknown")
|
||||||
|
target_raw = raw_code if raw_code else raw_name
|
||||||
|
|
||||||
|
model_name = normalize_model_name(target_raw)
|
||||||
|
|
||||||
|
try:
|
||||||
|
available = int(pkg.get("AvailableAmount", 0))
|
||||||
|
unit = pkg.get("Unit", "token").lower()
|
||||||
|
if unit == "token":
|
||||||
|
model_counts[model_name] = model_counts.get(model_name, 0) + available
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
|
||||||
|
return model_counts
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
active_monitor_loop()
|
||||||
Reference in New Issue
Block a user