Compare commits
3 Commits
f74f228322
...
8feb38db37
| Author | SHA1 | Date | |
|---|---|---|---|
| 8feb38db37 | |||
| e29d94f9e8 | |||
| 729e5c7bc3 |
+15
-1
@@ -116,12 +116,15 @@ def fetch_all_packages():
|
||||
version = "2022-01-01"
|
||||
all_packages = []
|
||||
current_token = ""
|
||||
seen_tokens = set()
|
||||
|
||||
while True:
|
||||
try:
|
||||
payload = {
|
||||
"ResourceType": "Package",
|
||||
"MaxResults": "20",
|
||||
# 只拉生效中的数据包,跳过 UsedUp/Expired/Refunded 等余额为 0 的无用包
|
||||
"Status": "Effective",
|
||||
"NextToken": current_token,
|
||||
}
|
||||
headers, body = get_signed_headers(action, version, payload)
|
||||
@@ -134,8 +137,19 @@ def fetch_all_packages():
|
||||
if items:
|
||||
all_packages.extend(items)
|
||||
next_token = data.get("Result", {}).get("NextToken", "")
|
||||
if not next_token or next_token == current_token:
|
||||
|
||||
# Infinite Loop Protection: VolcEngine API 翻页到头后 NextToken 会
|
||||
# 循环回起始 token(实测:数据只有 ~8 页,第 9 页绕回第一个 token),
|
||||
# 没有该保护会无限翻页。
|
||||
if not next_token:
|
||||
break
|
||||
if next_token == current_token:
|
||||
break
|
||||
if next_token in seen_tokens:
|
||||
print(f"WARNING: Detected pagination cycle with token {next_token}. Stopping.")
|
||||
break
|
||||
|
||||
seen_tokens.add(next_token)
|
||||
current_token = next_token
|
||||
time.sleep(0.2)
|
||||
except Exception as e:
|
||||
|
||||
+74
-26
@@ -41,6 +41,27 @@ except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
# ==========================================
|
||||
# 🏷️ Unified Log Tag (统一日志标识,便于过滤)
|
||||
# ==========================================
|
||||
# 本脚本所有输出统一带 [VolcSync] 前缀,方便日志筛选:
|
||||
# docker logs litellm 2>&1 | grep VolcSync
|
||||
# 实现方式:模块级重绑定 print(`print` 在模块命名空间中按调用时查找),
|
||||
# 一处覆盖 log()/dprint 与全部裸 print 调用;对模块外部无任何影响。
|
||||
LOG_TAG = "[VolcSync]"
|
||||
|
||||
_original_print = print
|
||||
|
||||
|
||||
def _tagged_print(*args, **kwargs):
|
||||
if args and isinstance(args[0], str):
|
||||
args = (args[0].lstrip("\n"), *args[1:])
|
||||
_original_print(LOG_TAG, *args, **kwargs)
|
||||
|
||||
|
||||
print = _tagged_print
|
||||
|
||||
|
||||
# ==========================================
|
||||
# 🛠️ Configuration & Constants
|
||||
# ==========================================
|
||||
@@ -157,7 +178,7 @@ def log(msg, level="INFO"):
|
||||
icon = {"DEBUG": "🔍", "INFO": "ℹ️", "WARNING": "⚠️", "ERROR": "❌"}.get(
|
||||
level, "📝"
|
||||
)
|
||||
print(f"{icon} [VolcSync] {msg}", flush=True)
|
||||
print(f"{icon} {msg}", flush=True)
|
||||
|
||||
|
||||
def dprint(msg):
|
||||
@@ -842,6 +863,10 @@ def fetch_all_packages():
|
||||
payload = {
|
||||
"ResourceType": "Package",
|
||||
"MaxResults": "20",
|
||||
# 服务端只返回生效中的数据包,跳过 UsedUp/Expired/Refunded 等
|
||||
# 余额为 0 的无用包,显著减少翻页次数与 QPS 占用。
|
||||
# 周期型包(月度重置)状态仍为 Effective,不会被误过滤。
|
||||
"Status": "Effective",
|
||||
"NextToken": current_token,
|
||||
}
|
||||
headers, body = get_signed_headers(action, version, payload)
|
||||
@@ -855,15 +880,18 @@ def fetch_all_packages():
|
||||
next_token = data.get("Result", {}).get("NextToken", "")
|
||||
|
||||
# Infinite Loop Protection
|
||||
# 火山 API 翻页到头后 NextToken 会循环回起始 token(实测数据只有
|
||||
# ~8 页时第 9 页绕回第一个 token),必须用 seen_tokens 拦下载断。
|
||||
if not next_token:
|
||||
break
|
||||
if next_token == current_token:
|
||||
break
|
||||
if next_token in seen_tokens:
|
||||
if DEBUG_MODE:
|
||||
print(
|
||||
f"DEBUG: Detected pagination cycle with token {next_token}. Stopping."
|
||||
)
|
||||
log(
|
||||
f"⚠️ VolcEngine pagination cycle detected (token {next_token} repeated). "
|
||||
f"Stopping after {len(all_packages)} packages.",
|
||||
level="WARNING",
|
||||
)
|
||||
break
|
||||
|
||||
seen_tokens.add(next_token)
|
||||
@@ -1308,21 +1336,32 @@ def active_monitor_loop():
|
||||
dynamic_key = f"limit:dynamic:volcengine:remaining:{target}"
|
||||
|
||||
# ==========================================
|
||||
# 🏔️ Smart Peak Tracking (Auto-Correction)
|
||||
# 🏔️ Daily Peak Lock (First-Sync + Balance-Gated Re-Lock)
|
||||
# ==========================================
|
||||
# Strategy:
|
||||
# 1. Capture Jumps (Recovery): If Balance jumps UP, trust it immediately. (Fixes latency)
|
||||
# 2. Capture Drops (Expiry): If Balance drops HUGE (>20%), trust it. (Fixes expiry)
|
||||
# - Why trust drops? Because if it's a false drop (glitch), Step 1 will fix it
|
||||
# automatically on the next sync when it "Jumps" back up.
|
||||
# 背景:AvailableAmount 是各资源包实例的实时剩余,但火山统计有
|
||||
# 更新滞后(高用量时段会短暂冻结,实测 20 万 token 消耗内余额不动)。
|
||||
# 旧逻辑把每次用量增长误判为"余额恢复"(Jump)锁更高峰值,
|
||||
# 导致 stable_limit 越算越多(当天 399,836 -> 599,511 一路抬升)。
|
||||
# 修正策略(日内限额单调可控):
|
||||
# 1. 首次同步锁定当日峰值(= usage + balance 的总配额估计);
|
||||
# 2. 仅当 balance 自身净上涨(新资源包发放/对账回补,≥50k)
|
||||
# 时,按新增额度抬高峰值 —— 幅度 = 历史峰值 + 发放增量,
|
||||
# 绝不把滞后虚高的 usage 一起算进去;
|
||||
# 3. balance 回落(额度扣减追上用量)时 Drop 收紧;
|
||||
# 4. usage 增长驱动的 pool 上涨一律不得抬高限额。
|
||||
# 实测:协作奖励计划包每自然日发放、偶发一天 2 个,故必须有第 2 条;
|
||||
# 次日 00:00 (CST) 自动重建当日峰值。
|
||||
|
||||
total_pool = current_usage + balance
|
||||
peak_key = f"volc:peak_pool:{today_str}:{target}"
|
||||
|
||||
# Short-term memory (Last seen value)
|
||||
# Short-term memory (Last seen value) - 仅用于日志对比/增量计算
|
||||
last_pool_key = f"volc:last_seen_pool:{target}"
|
||||
last_pool_val = redis_client.get(last_pool_key)
|
||||
last_pool = float(last_pool_val) if last_pool_val else 0.0
|
||||
last_balance_key = f"volc:last_balance:{target}"
|
||||
last_balance_val = redis_client.get(last_balance_key)
|
||||
last_balance = float(last_balance_val) if last_balance_val else 0.0
|
||||
|
||||
# Long-term memory (Day's High Water Mark)
|
||||
old_peak_val = redis_client.get(peak_key)
|
||||
@@ -1330,21 +1369,31 @@ def active_monitor_loop():
|
||||
|
||||
current_peak = historical_peak
|
||||
|
||||
# 1. JUMP DETECTION (Recharge / Latency Resolved)
|
||||
# If pool jumps significantly (>5%) from last seen, it's a fresh, true value.
|
||||
if total_pool > (last_pool * 1.05) and last_pool > 0:
|
||||
if DEBUG_MODE:
|
||||
# 1. INITIALIZATION (First sync of the day): lock once
|
||||
if historical_peak == 0:
|
||||
if DEBUG_MODE and last_pool > 0:
|
||||
print(
|
||||
f"DEBUG: 📈 Balance Jump ({int(last_pool):,} -> {int(total_pool):,}) - Locking New Peak.",
|
||||
f"DEBUG: 📌 Locking Daily Peak ({int(last_pool):,} -> {int(total_pool):,}) - First Sync Today.",
|
||||
flush=True,
|
||||
)
|
||||
current_peak = total_pool
|
||||
redis_client.setex(peak_key, 172800, int(current_peak))
|
||||
|
||||
# 2. DROP DETECTION (Resource Expiry)
|
||||
# If pool drops (even small amounts), we accept it.
|
||||
# Why? Because keeping a phantom high peak is dangerous.
|
||||
# If it was a glitch, Rule #1 will fix it next time it bounces back.
|
||||
# 2. RECHARGE (Balance-driven upward re-lock)
|
||||
# 新资源包发放 / API 对账回补 → balance 自身净上涨 ≥50k。
|
||||
# 只把"发放增量"加进已锁定的峰值,不重算整个滞后 pool。
|
||||
elif last_balance > 0 and (balance - last_balance) >= 50000:
|
||||
grant_delta = int(balance - last_balance)
|
||||
current_peak = historical_peak + grant_delta
|
||||
if DEBUG_MODE:
|
||||
print(
|
||||
f"DEBUG: 🆕 New Quota Detected (balance {int(last_balance):,} -> {int(balance):,}, +{grant_delta:,}) - Re-locking Daily Peak to {int(current_peak):,}.",
|
||||
flush=True,
|
||||
)
|
||||
redis_client.setex(peak_key, 172800, int(current_peak))
|
||||
|
||||
# 3. DROP DETECTION (Resource Expiry / latency catch-up)
|
||||
# pool 回落时收紧限额;回落之后保持不变,绝不回升。
|
||||
elif total_pool < historical_peak and historical_peak > 0:
|
||||
drop_ratio = (
|
||||
historical_peak - total_pool
|
||||
@@ -1357,13 +1406,12 @@ def active_monitor_loop():
|
||||
current_peak = total_pool
|
||||
redis_client.setex(peak_key, 172800, int(current_peak))
|
||||
|
||||
# 3. INITIALIZATION / DRIFT
|
||||
elif historical_peak == 0 or total_pool > historical_peak:
|
||||
current_peak = total_pool
|
||||
redis_client.setex(peak_key, 172800, int(current_peak))
|
||||
# 4. NO USAGE-DRIVEN RE-LOCK: 用量增长(balance 不变)不得抬高限额。
|
||||
# (旧逻辑的 Bug 就在这里:把滞后虚高当余额恢复。)
|
||||
|
||||
# Always update "Last Seen" for next comparison
|
||||
redis_client.setex(last_pool_key, 86400, int(total_pool))
|
||||
redis_client.setex(last_balance_key, 86400, int(balance))
|
||||
|
||||
# Calculate Stable Limit
|
||||
safety_margin = max(current_peak * 0.15, 250000)
|
||||
@@ -1472,7 +1520,7 @@ def active_monitor_loop():
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error in monitor loop / 监控循环出错: {e}", flush=True)
|
||||
traceback.print_exc()
|
||||
print(traceback.format_exc(), end="")
|
||||
time.sleep(5)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user