kim/litellm-config
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix: Deduplicate packages based on full JSON content hash to prevent double counting

Browse Source
This commit is contained in:
kim
2026-01-29 18:26:38 +08:00
parent 587a005665
commit da41c569c0
1 changed files with 13 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
volc_limit_sync.py
+13
View File
@@ -564,9 +564,22 @@ def fetch_all_packages():
def process_packages_breakdown(packages): def process_packages_breakdown(packages):
model_counts = {} model_counts = {}
seen_package_hashes = set()
print(f"\n--- Raw Package Processing ---", flush=True) print(f"\n--- Raw Package Processing ---", flush=True)
for pkg in packages: for pkg in packages:
try: try:
# 0. Deduplication (Exact Match)
# VolcEngine pagination bugs might return duplicate items or pages.
# We assume identical JSON objects are duplicates.
pkg_hash = hashlib.md5(json.dumps(pkg, sort_keys=True).encode()).hexdigest()
if pkg_hash in seen_package_hashes:
if DEBUG_MODE:
print(f"DEBUG: Skipping duplicate package (Hash: {pkg_hash})")
continue
seen_package_hashes.add(pkg_hash)
raw_code = pkg.get("ConfigurationCode", "") raw_code = pkg.get("ConfigurationCode", "")
raw_name = pkg.get("ConfigurationName", "Unknown") raw_name = pkg.get("ConfigurationName", "Unknown")
target_raw = raw_code if raw_code else raw_name target_raw = raw_code if raw_code else raw_name