Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-18 00:35:04 +02:00
Code Issues Packages Projects Releases Wiki Activity

added xet resolver and downloader

Browse Source
This commit is contained in:
Concedo
2026-06-26 17:58:43 +08:00
parent e35e415668
commit 73607e9e01
1 changed files with 25 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
koboldcpp.py
+25 -3
View File
@@ -10501,6 +10501,27 @@ def sanitize_string(input_string):
sanitized_string = re.sub( r'[^\w\d\.\-_]', '', input_string)
return sanitized_string
def resolve_huggingface_xet_url(input_url):
if "https://huggingface.co/" not in input_url or "/resolve/" not in input_url:
return input_url
try:
req = urllib.request.Request(input_url, headers={'User-Agent': 'Mozilla/5.0'}, method="HEAD")
with urllib.request.urlopen(req, timeout=10) as response:
resolved_url = response.geturl()
except Exception:
try:
req = urllib.request.Request(input_url, headers={'User-Agent': 'Mozilla/5.0'})
with urllib.request.urlopen(req, timeout=10) as response:
resolved_url = response.geturl()
except Exception as e:
print(f"Could not pre-resolve Hugging Face URL, using original URL: {e}")
return input_url
# resolved_host = urllib.parse.urlparse(resolved_url).netloc.lower()
if ("xet-bridge" in resolved_url) and resolved_url != input_url:
print(f"Resolved Hugging Face xet URL to {resolved_url}", flush=True)
return resolved_url
return input_url
def downloader_internal(input_url, output_filename, capture_output, min_file_size=64): # 64 bytes required by default
download_dir_path = args.downloaddir
if "https://huggingface.co/" in input_url and "/blob/main/" in input_url:
@@ -10535,6 +10556,7 @@ def downloader_internal(input_url, output_filename, capture_output, min_file_siz
dl_success = False
out_dir = os.path.dirname(os.path.abspath(output_filename)) or os.getcwd()
out_name = os.path.basename(output_filename)
download_url = resolve_huggingface_xet_url(input_url)
aria2_candidates = []
if os.name == 'nt':
basepath = os.path.abspath(os.path.dirname(__file__))
@@ -10548,7 +10570,7 @@ def downloader_internal(input_url, output_filename, capture_output, min_file_siz
"--summary-interval=10", "--console-log-level=error", "--log-level=error",
"--download-result=default", "--continue=true", "--allow-overwrite=true",
"--file-allocation=none", "--max-tries=3", "--retry-wait=5",
"-d", out_dir, "-o", out_name, input_url
"-d", out_dir, "-o", out_name, download_url
]
for aria2_exe, aria2_name in aria2_candidates:
if dl_success:
@@ -10562,7 +10584,7 @@ def downloader_internal(input_url, output_filename, capture_output, min_file_siz
try:
if not dl_success and shutil.which("curl") is not None:
rc = subprocess.run(["curl", "-fLo", output_filename, input_url],
rc = subprocess.run(["curl", "-fLo", output_filename, download_url],
capture_output=capture_output, text=True, check=True, encoding="utf-8")
dl_success = (rc.returncode == 0 and os.path.exists(output_filename) and os.path.getsize(output_filename) > min_file_size)
except subprocess.CalledProcessError as e:
@@ -10570,7 +10592,7 @@ def downloader_internal(input_url, output_filename, capture_output, min_file_siz
try:
if not dl_success and shutil.which("wget") is not None:
rc = subprocess.run(["wget", "-O", output_filename, input_url],
rc = subprocess.run(["wget", "-O", output_filename, download_url],
capture_output=capture_output, text=True, check=True, encoding="utf-8")
dl_success = (rc.returncode == 0 and os.path.exists(output_filename) and os.path.getsize(output_filename) > min_file_size)
except subprocess.CalledProcessError as e: