在这个教程中，我们与"下犬"(Underdog Saluki 27B)合作，这是Qwen3.8-27B的7.9GB构件，它的制造者们说，工具调用完好无损，每重量约2位，甚至从自己的120任务基准测试上取出完整的54GB模型。 Saluki基于ISTA-DASLab的GSQ-RCO对同型号的量化，所以我们把两个文件并存于股票llama.cpp. 我们从文件本身开始：两个GGUF头的逐个分数，然后用字节比较显示是Saluki是ISTA位数的remix还是被改变的模型。 然后我们用一个GPU提供每个文件，读取其内存和速度，并在伯克利函数调用“领导板”的120个原始任务上得分，使用BFCL自己的检查器进行严格和宽松的解析。
导入 Os
导入再
导入点
导入 sys
导入 json
导入数学
导入副本
导入语句
导入 glub
导入时间
导入类型
导入随机
导入快取
导入结构
导入散列
导入 tar 文件
导入平台
导入回溯
导入子进程
导入收藏
导入urlib。请求
导入urlib.error
成果 = # # 步 - > 单行结果，用于总表
声称 = # (步骤、 主张、 持有)行；持有的是 True、 False 或 None (本行没有决定)
现值= ["1"]
def 横幅( 标题):
打印 ("\n" + "=" * 78)
打印(标题)
打印 ("=" * 78)
def 区域( 名称):
包装( fn):
df 运行( *a, ** kw):
现值 [0] = name.split (") [0]
横幅( 名称)
t0 = 时间。 time ()
尝试：
输出 = fn(*a,**kw)
ResULTS[名] = 如果出现(出，字符串) 其他"ok"
返回出来
例外作为e:
ResULTS[名]=f"SKIPPED/FAILED -" {type (e). name:{e}"
打印 (f)\n[!] {name} 未完成：{type (e). name:{e}}
回溯.print exc(限制=3)
无
终于：
打印(f)\n [{time.time () - t0: 1f} s]")
json.dump ({"结果"):RESULTS,"声称"：声称}，开通(os.path.join (WORK,"结果.json"),"w"，缩出=1).
返回运行
返回包装
df 索赔(案文，持有):
声称。 附录((CURRENT[0]，正文，持有))
持有的返回
横幅 ("1. 平台、 指针和帮助")
System, MACHINE = 平台。 system ()，平台。 机器 ()
ON CUDA = TEM ="Linux"和 slutil.which ("nvidia-smi")不是无。
profile ON CUDA或 (SYSTEM ="Darwin"和MACHINE ="arm64"),"需要NVIDIA GPU在Linux(Colab T4)或Apple-硅Mac上".
Work = os.environ.get ("SALUKI WORK") 或 ("/content/saluki" 如果 os.path.isdir ("/content") 或 os.path.abspath ("saluki work"));
d in(“模型”、“llama”、“bfcl”、“跑道”):
os.makedirs (os. path.join (WORK, d)，存在 ok=True) (英语).
{\fn方正粗倩简体\fs12\an8\1cHFFFF00\b0}笔记本下载的每件东西都被尺寸和SHA -256绑住
LLAMA TAG = "b11565"#llama.cpp 2026-10-12发布，承诺f9a95aeb2
LLAMA ASSET = {"Linux":("llama-b11565-bin-ubuntu-cuda-12.8-x64.tar.gz",171559844), (中文(简体) ).
"61b8cf4e49739afe4008b06a92390dff999e706e9e7e9749c015544a9f241627" (中文(简体) ).
"达尔文": ("llama-b11565-bin-macos-arm64.tar.gz",12105509).
"19289548468babcd5131cecccf3a9bc7c67030cc7f0a978dafc0db1ba7891068")}[SYSTEM]
国防部={
"萨卢克":dict(repo="康威研究/下道克-萨卢克-27B-1.0",rev="4f60eba09f4b6db63c72915d328d97e1d93edb8c"
file="Underdog-Saluki-27B-1.0-IQ2-mix.gguf"，体积为7898369152".
sha256="4a673518b11b1c4445f9b9a9d3c40356f5ba39f6a6260dcb25f3fbd475d9efb")，原名"sha".
"ISTA": dict(repo="ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF", rev="d562806dbafae37109975e970aae91b43e73b440",
s.gguf)，大小为8422841472) 互联网档案馆的存檔，存档日期2013-12-02.
sha256="f0ae5006da0ce6225935339e4e989369f94de95d2263cf969519f8420c9ae02c")，是中国国家重点文物保护单位。
}
# BFCL v4 数据和 BFCL 自己的检查器代码，在一个大猩猩承诺。
GORILLA = ("https://raw.githubusercontent.com/ShishirPatil/gorilla/6ea57973c7a6097fd7c5915698c54c17c5b1b6c8/"
(bfcl eval/") (中文(简体) ).
BFCL FILES = { (英语).
" Data/BFCL v4 simple python.json":"82d63ba502eb2520c6b5d1d9a5c4b590e03ff261565175561f6228a367d1991" (中文(简体) ).
"data/BFCL_v4_multiple.json": "aef168155ebd74b7ac2401198b201343bc7d16d7a3d7e0d4e6d8ee82c6969b2a",
"data/BFCL_v4_irrelevance.json": "2b6ed4c2e992cdcf5f1678a701851f944bef7550ee026ed1ddb89efed5be01a6",
"data/BFCL_v4_parallel.json": "19f51a82eff42e5d62541aa500115a056eb78f437c2ba1f10415fd7c8e5dda84",
"数据/BFCL v4 并行-多相.json":"8863ea84333239f55c5f016154cf0830853c89f693c6ea270396a2f121960579".
"数据/可能回答/BFCL v4 simple python.json":"90克d5bc653690ee8e459b5b3f3fc9458606f7f3fcbb5b51b7dc581f8c86dc".
"数据/可能回答/BFCL v4 multiple.json":"244e00ce9395df948bcafc7bee64e8f9c87ef70887587d83cae45b13699f3047".
"数据/可能回答/BFCL v4 并行.json":"8a6aa19c1addc6a5a2f7e40f9db30cc7e95815e7b830c90589ab318229e0f0".
"数据/可能回答/BFCL v4 并行-多相.json":"5ebf24f458c1f16300c05505d83d6f0a1b68b79be273a033febd0d4f840507e3".
" eval checker/ast eval/ast checker.py":"2aae7a68461a8f76c0 Be3894c8901b66b56967a1989d3ab066051ef3b97f1538" (中文(简体) ).
"constants/enums.py": "2182becfa2a1d071ee1db30db593b4758c6bf866aa12d2d4b8daf09175ea518a",
"constants/type_mappings.py": "1702fb67afbe2c492608e58e2b7d02e46381f50166b47f3c952f76e34c7cd3bd",
"utils.py":"3703b9bb63f83581c60b8e0b82aac74c360c9ca781e99e7d4eb28b0cce2285dd" (中文(简体) ).
"model_handler/utils.py": "f78fd3edce603b333dc9a88ee2c041dc547d51f71aa449ffebc044c4b1e353f3",
}
# 下犬座的五种任务，每个任务24个；我们自己绘制出种子样本，因为任务清单没有公布。
KINDS = [("simple python","调用一个正确的函数"), ("多功能","从几个中选择正确的函数")
("无关","没有工具适合时知道"), ("并行","相同的功能，几个呼叫").
("并行多功能","几种不同的功能").
N_PER_KIND, SEED, CTX, PORT = 24, 2026, 16384, 8080
#llama-server自身默认温度为0.8和上 k 40，因此每个请求都说明其采样器。
GREEDY = dict(温度=0，上 k=0，上 p=1.0，下-p=0)
df sH(cmd, **kw): (中文(简体) ).
返回子进程。 run( cmd，抓取输出= True，文本= True, ** kw)
daf sha256 file(路径，块=1 + 22):
h = 大麻脂.sha256()
以 open( path, “rb ”) 为 f:
b in iter( lambda: f.read( chunk), b") 中：
h. 更新(b)
返回 h. hexdigest () ()
取出(url, dest，大小=无；sha=无；静=法：
""可续下载有卷曲，再用尺寸和SHA-256检查。 返回所花费的下载秒数。”
t0 = 时间。 time ()
如果不是(os.path. sexts(dest)和(大小为None或os.path.getsize(dest) ===)):
试验范围(5):
r = sh(["curl", "-fsSL", "-C", "-", "--retry", "5", "-o", dest, url])
如果 r. returncode = 0和 (大小为 None 或 os.path.getsize(dest) = 大小):
换行
时间。 sleep( 5 * (尝试 + 1))
得到=os.path.getsize(最大)
profile 大小为 None 或有 == 大小，f"{os.path.basename(dest)}: {got} 字节，预期为 {大小}"
若为sha:
profile sha256 file (dest) == sha, f"{os.path.basename (dest)}: SHA-256不匹配" (中文(简体) ).
返回时间。 time () - t0
print(f)"平面：{SYSTEM}{MACHINE}, Python {平面：python version ()}, glibc {平面：libc ver () [1] 或 “n/a'}"
如果系统"Linux":
print("os :", open("/etc/os-release").read().split('PRETTY_NAME="')[1].split('"')[0])
gpu = sh(["nvidia-smi", "--query-gpu=name,memory.total,driver_version", "--format=csv,noheader"]).stdout.strip()
打印 ("gpu:", gpu)
# CUDA 12.8建造了没有CUDA运行时间的llama.cpp飞船；Colab的形象有"活页轮".
导入导入lib.util
如果导入lib.util.find spec ("nvidia.cuda runtime") 则无为或导入lib.util.find spec ("nvidia.cublas") 为无为：
sub process.run ([sys. executive,"-m","pip","install","-q","nvidia-cuda-runtime-cu12 == 12.9.79") 互联网档案馆的存檔，存档日期2011-12-09.
"nvidia-cublas-cu12 ==: 12.9.2.10"]，检查=true (中文(简体) ).
导入 nvidia.cuda 运行时间，nvidia.cublas
CUDA LIBS = [os.path.join (list(m. path ) [0],"lib"为m in (nvidia.cuda runtime,nvidia.cublas)] (中文(简体) ).
其他情况：
芯片 = s (["sysctl","-n","machdep.cpu.brand string"]). stdout.strip ().
mem = int(sh(["sysctl", "-n", "hw.memsize"]).stdout) / 2**30
打印( f) gpu: {chip}, {me: 0f} GB 统一内存(Metal)".
CUDA_LIBS = []
HW = gpu.split(",")[0] if SYSTEM == "Linux" else chip
print(f"work dir : {WORK} ({shutil.disk_usage(WORK).free / 1e9:.0f} GB free)")
笔记本触摸的每样东西都会被固定在任何东西运行之前所设置的单元格记录了llama.cpp的发布，既按大小记录了模型文件，又按14个BFCL文件，SHA-256，它为每个请求修正了取样器，因为llama-服务器否则会回到我们没有选择的默认状态。 它在Linux或Apple-硅Mac上检测到平台，NVIDIA GPU，在Colab上它确保了CUDA12运行时库的预建llama. cpp需求为可导入。 cpp需求为可导入。
@section ("2. 下载 llama. cpp b11565 和两个2位文件，并检查每个字节与它的取号").
def 下载 step ():
环球 LLAMA DIR, SERVER BIN, SERVER ENV
名称、 大小、 sha = LLAMA ASSET
tar = os.path.join(WORK,"llama"，名称)
抓取(f)"https://github.com/ggml-org/llama.cpp/发布/下載/{LLAMA TAG}/{name}"，焦油，大小，sha).
以 tarfile.open(tar) 为t:
t.extractall(os.path.join(WORK,"llama"),"过滤="数据")
SERVER BIN = glob.glob(os.path.join (WORK,"llama","**","llama-服务器")，递归=True [0]
LLAMA DIR = os.path.dirname(SERVER BIN) 互联网档案馆的存檔，存档日期2013-12-21.
b 在 glob.glob(os.path.join (LLAMA DIR, “llama-*”)中):
os.chmod(b, 0o755) (中文(简体) ).
SERVER ENV = dict(os.environ) 互联网档案馆的存檔，存档日期2014-12-21.
如果 CUDA LIBS: (笑声)
活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活活
缺少 = [l for l in s (['ldd, sERVER BIN], senv=SERVER ENV].stdout.splitlines () 如果在l中"找不到"的话]
声明不缺失，f"未解决的库：{ missing}"
r = s ([SERVER BIN,"-版"),env=SERVER ENV).
ver = re.search(r"version: .*", r.stdout + r.stderr).group(0)
打印(f"llama.cpp:{name} (SHA-256已选中)- > {ver}")
在BFCL FILES. items():
dest = os.path.join(WORK, "bfcl", rel.replace("/", "__"))
获取( GORILLA + rel, dest, sha= digest)
print(f)"BFCL v4: {len(BFCL FILES))}大猩猩@6ea5797的文件 (5个任务文件，4个答题文件，5个检查器来源)，所有 SHA-256 已选中)
用于标记，m in MODELS. items ():
path = os.path.join(WORK,"模型", m ["文件"])
有=os.path.path (path) and os.path.getsize (path) = ["大小"]
url = f"https://huggingface.co/{m['repo']}/resolve/{m['rev']}/{m['file']}"
dl = 获取(url，路径，m ["大小"]) # 大小检查，下方为散列
t0 = 时间。 time ()
OK=sha256 file (path) == ["sha256"] (中文(简体) ).
如果在 {dl:.0f} s ({ m ['size'] / dl / 1e6:.0f} MB/s 中有其他 f 下载，则如何 = “已经在磁盘上”
打印( f) { tag: 7s}: { m [' file']} { m [' six']:,} 字节，{how}; SHA-256 { “ matches” if ok else “DOES Not MATCH” ”
f"the Hub's LFS pin ({time.time() - t0:.0f} s to hash)")
claim(f) “{m['file']}与被钉在 {m['repo'}} ['rev'][:7] 上的文件字节相同，ok)
坚持好
返回 f"llama.cpp {LLAMA TAG}; Saluki 7.90 GB和 ISTA IQ2 XS 8.42 GB 由 SHA-256 验证"
下载 step ()
我们下载了llama.cpp b11565的 CUDA 架构，既2位文件和BFCL 数据，又检查了每个字节与它的指针。 Hugging Face的下载是可重复的，散列是Hub自己的LFS文摘，在固定的修改中，所以一个默默地被替换的文件会停止笔记本而不是改变其结果。 BFCL 的任务文件，答覆文件，和检查器代码都来自一个大猩猩的承诺。 手机打印下载时间有多长，这是Colab上运行最慢的部分。
GGML_TYPES = {0: ("F32", 1, 4), 1: ("F16", 1, 2), 8: ("Q8_0", 32, 34), 10: ("Q2_K", 256, 84), 12: ("Q4_K", 256, 144),
16: ("IQ2_XXS", 256, 66), 17: ("IQ2_XS", 256, 74), 18: ("IQ3_XXS", 256, 98), 19: ("IQ1_S", 256, 50),
21: ("IQ3_S", 256, 110), 22: ("IQ2_S", 256, 82), 23: ("IQ4_XS", 256, 136), 29: ("IQ1_M", 256, 56),
30: ("BF16", 1, 2)")#名称，每块值，每块字节(ggml-common.h)
def 读取为 gguf(路径):
"""GGUF v3 标题：元数据，收分表(名称，暗号，类型，偏移)和收分数数据起始地".
f = open(路径，“rb ”)
u = lambda fmt: struct.unpack (" <" + fmt, f.read (struct.calcsize (fmt)) [0] (中文(简体) ).
s = lambda: f.read (u ("Q"))) 代码 ("utf-8","替换")
scalar = {0:"B", 1:"b", 2:"H", 3:"h", 4:"我", 5:"i", 6:"f", 7:""?", 10:"Q", 11:"q", 12:"d"}
def val(t): (中文(简体) ).
如果第8节：
返回 s( )
如果第9节：
et, n = u ("I"), u ("Q") 维基语录链接：名人名言 - 文学作品 - 谚语 - 谚语 - 谚语
返回 [val(et) for in(n)] 范围
返回 u( scalar[t] )
(原始内容存档于2018-10-21) (英语). leady f.read (4) == b "GGUF"
版本，n tensors,n kv=u("I"),u("Q"),u("Q")
kv = {}
为
本文由 AI宝库自动抓取并翻译，版权归原作者所有；如需查看原文请点下方链接。
In this tutorial, we work with Underdog Saluki 27B, a 7.9 GB build of Qwen3.8-27B that its makers say keeps tool calling intact at about two bits per weight and even edges out the full 54 GB model on their own 120-task benchmark. Saluki is built on ISTA-DASLab’s GSQ-RCO quantization of the same model, so we put both files side by side in stock llama.cpp. We start with the files themselves: a tensor-by-tensor diff of the two GGUF headers, then a byte-level comparison that shows whether Saluki is a remix of ISTA’s bits or a changed model. We then serve each file on one GPU, read its memory and speed, and score both on the same seeded 120 tasks from the Berkeley Function Calling Leaderboard, using BFCL’s own checker with strict and lenient parsing.
import os
import re
import ast
import sys
import json
import math
import copy
import enum
import glob
import time
import types
import random
import shutil
import struct
import hashlib
import tarfile
import platform
import traceback
import subprocess
import collections
import urllib.request
import urllib.error
RESULTS = {} # step -> one-line result, for the summary table
CLAIMS = [] # (step, claim, held) rows; held is True, False or None (not decided in this run)
CURRENT = ["1"]
def banner(title):
print("\n" + "=" * 78)
print(title)
print("=" * 78)
def section(name):
def wrap(fn):
def run(*a, **kw):
CURRENT[0] = name.split(".")[0]
banner(name)
t0 = time.time()
try:
out = fn(*a, **kw)
RESULTS[name] = out if isinstance(out, str) else "ok"
return out
except Exception as e:
RESULTS[name] = f"SKIPPED / FAILED -> {type(e).__name__}: {e}"
print(f"\n[!] {name} did not complete: {type(e).__name__}: {e}")
traceback.print_exc(limit=3)
return None
finally:
print(f"\n[{time.time() - t0:.1f} s]")
json.dump({"results": RESULTS, "claims": CLAIMS}, open(os.path.join(WORK, "results.json"), "w"), indent=1)
return run
return wrap
def claim(text, held):
CLAIMS.append((CURRENT[0], text, held))
return held
banner("1. Platform, pins and helpers")
SYSTEM, MACHINE = platform.system(), platform.machine()
ON_CUDA = SYSTEM == "Linux" and shutil.which("nvidia-smi") is not None
assert ON_CUDA or (SYSTEM == "Darwin" and MACHINE == "arm64"), "needs an NVIDIA GPU on Linux (Colab T4) or an Apple-silicon Mac"
WORK = os.environ.get("SALUKI_WORK") or ("/content/saluki" if os.path.isdir("/content") else os.path.abspath("saluki_work"))
for d in ("models", "llama", "bfcl", "runs"):
os.makedirs(os.path.join(WORK, d), exist_ok=True)
# Everything the notebook downloads is pinned by size and SHA-256.
LLAMA_TAG = "b11565" # llama.cpp release of 2026-10-12, commit f9a95aeb2
LLAMA_ASSET = {"Linux": ("llama-b11565-bin-ubuntu-cuda-12.8-x64.tar.gz", 171559844,
"61b8cf4e49739afe4008b06a92390dff999e706e9e7e9749c015544a9f241627"),
"Darwin": ("llama-b11565-bin-macos-arm64.tar.gz", 12105509,
"19289548468babcd5131cecccf3a9bc7c67030cc7f0a978dafc0db1ba7891068")}[SYSTEM]
MODELS = {
"Saluki": dict(repo="ConwayResearch/Underdog-Saluki-27B-1.0", rev="4f60eba09f4b6db63c72915d328d97e1d93edb8c",
file="Underdog-Saluki-27B-1.0-IQ2-mix.gguf", size=7898369152,
sha256="4a673518b11b1c4445f9b9a9d3c40356f5ba39f6a6260dcb25f3fbfd475d9efb"),
"ISTA": dict(repo="ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF", rev="d562806dbafae37109975e970aae91b43e73b440",
file="Qwen3.8-27B-GSQ-RCO-IQ2_XS.gguf", size=8422841472,
sha256="f0ae5006da0ce6225935339e4e989369f94de95d2263cf969519f8420c9ae02c"),
}
# BFCL v4 data and BFCL's own checker code, at one gorilla commit.
GORILLA = ("https://raw.githubusercontent.com/ShishirPatil/gorilla/6ea57973c7a6097fd7c5915698c54c17c5b1b6c8/"
"berkeley-function-call-leaderboard/bfcl_eval/")
BFCL_FILES = {
"data/BFCL_v4_simple_python.json": "82dd63ba502eb2520c6b5d1d9a5c4b590e03ff261565175561f6228a367d1991",
"data/BFCL_v4_multiple.json": "aef168155ebd74b7ac2401198b201343bc7d16d7a3d7e0d4e6d8ee82c6969b2a",
"data/BFCL_v4_irrelevance.json": "2b6ed4c2e992cdcf5f1678a701851f944bef7550ee026ed1ddb89efed5be01a6",
"data/BFCL_v4_parallel.json": "19f51a82eff42e5d62541aa500115a056eb78f437c2ba1f10415fd7c8e5dda84",
"data/BFCL_v4_parallel_multiple.json": "8863ea8433239f55c5f016154cf0830853c89f693c6ea270396a2fa121960579",
"data/possible_answer/BFCL_v4_simple_python.json": "90cd5bc653690ee8e459b5b3f3fc9458606f7f3fcbf795bb51b7dc581f8c86dc",
"data/possible_answer/BFCL_v4_multiple.json": "244e00ce9395df948bcafc7bee64e8f9c87ef70887587d83cae45b13699f3047",
"data/possible_answer/BFCL_v4_parallel.json": "8a6aa19c1adddc6a5a2f7e40f9dbf30cc7e95815e7b830c90589ab318229e0f0",
"data/possible_answer/BFCL_v4_parallel_multiple.json": "5ebf24f458c1f16300c05505d83d6f0a1b68b79be273a033febd0d4f840507e3",
"eval_checker/ast_eval/ast_checker.py": "2aae7a68461a8f76c0be3894c8901b66b56967a1989d3ab066051e3fb97f1538",
"constants/enums.py": "2182becfa2a1d071ee1db30db593b4758c6bf866aa12d2d4b8daf09175ea518a",
"constants/type_mappings.py": "1702fb67afbe2c492608e58e2b7d02e46381f50166b47f3c952f76e34c7cd3bd",
"utils.py": "3703b9bb63f83581c60b8e0b82aac74c360c9ca781e99e7d4eb28b0cce2285dd",
"model_handler/utils.py": "f78fd3edce603b333dc9a88ee2c041dc547d51f71aa449ffebc044c4b1e353f3",
}
# Underdog Bench's five kinds of tasks, 24 each; we draw our own seeded sample because its task list is not published.
KINDS = [("simple_python", "Call the one right function"), ("multiple", "Pick the right function from several"),
("irrelevance", "Know when no tool fits"), ("parallel", "Same function, several calls"),
("parallel_multiple", "Several different functions")]
N_PER_KIND, SEED, CTX, PORT = 24, 2026, 16384, 8080
# llama-server's own defaults are temperature 0.8 and top_k 40, so every request states its sampler.
GREEDY = dict(temperature=0, top_k=0, top_p=1.0, min_p=0)
def sh(cmd, **kw):
return subprocess.run(cmd, capture_output=True, text=True, **kw)
def sha256_file(path, chunk=1 << 22):
h = hashlib.sha256()
with open(path, "rb") as f:
for b in iter(lambda: f.read(chunk), b""):
h.update(b)
return h.hexdigest()
def fetch(url, dest, size=None, sha=None, quiet=False):
"""Resumable download with curl, then a size and SHA-256 check. Returns seconds spent downloading."""
t0 = time.time()
if not (os.path.exists(dest) and (size is None or os.path.getsize(dest) == size)):
for attempt in range(5):
r = sh(["curl", "-fsSL", "-C", "-", "--retry", "5", "-o", dest, url])
if r.returncode == 0 and (size is None or os.path.getsize(dest) == size):
break
time.sleep(5 * (attempt + 1))
got = os.path.getsize(dest)
assert size is None or got == size, f"{os.path.basename(dest)}: {got} bytes, expected {size}"
if sha:
assert sha256_file(dest) == sha, f"{os.path.basename(dest)}: SHA-256 mismatch"
return time.time() - t0
print(f"platform : {SYSTEM} {MACHINE}, Python {platform.python_version()}, glibc {platform.libc_ver()[1] or 'n/a'}")
if SYSTEM == "Linux":
print("os :", open("/etc/os-release").read().split('PRETTY_NAME="')[1].split('"')[0])
gpu = sh(["nvidia-smi", "--query-gpu=name,memory.total,driver_version", "--format=csv,noheader"]).stdout.strip()
print("gpu :", gpu)
# The CUDA 12.8 build of llama.cpp ships without the CUDA runtime; Colab's image has it as pip wheels.
import importlib.util
if importlib.util.find_spec("nvidia.cuda_runtime") is None or importlib.util.find_spec("nvidia.cublas") is None:
subprocess.run([sys.executable, "-m", "pip", "install", "-q", "nvidia-cuda-runtime-cu12==12.9.79",
"nvidia-cublas-cu12==12.9.2.10"], check=True)
import nvidia.cuda_runtime, nvidia.cublas
CUDA_LIBS = [os.path.join(list(m.__path__)[0], "lib") for m in (nvidia.cuda_runtime, nvidia.cublas)]
else:
chip = sh(["sysctl", "-n", "machdep.cpu.brand_string"]).stdout.strip()
mem = int(sh(["sysctl", "-n", "hw.memsize"]).stdout) / 2**30
print(f"gpu : {chip}, {mem:.0f} GB unified memory (Metal)")
CUDA_LIBS = []
HW = gpu.split(",")[0] if SYSTEM == "Linux" else chip
print(f"work dir : {WORK} ({shutil.disk_usage(WORK).free / 1e9:.0f} GB free)")
Everything the notebook touches is pinned before anything runs. The setup cell records the llama.cpp release, both model files and fourteen BFCL files by size and SHA-256, and it fixes the sampler for every request, because llama-server otherwise falls back to defaults we did not choose. It detects the platform, an NVIDIA GPU on Linux or an Apple-silicon Mac, and on Colab it ensures the CUDA 12 runtime libraries the prebuilt llama. cpp needs are importable.cpp needs are importable.
@section("2. Download llama.cpp b11565 and both 2-bit files, and check every byte against its pin")
def download_step():
global LLAMA_DIR, SERVER_BIN, SERVER_ENV
name, size, sha = LLAMA_ASSET
tar = os.path.join(WORK, "llama", name)
fetch(f"https://github.com/ggml-org/llama.cpp/releases/download/{LLAMA_TAG}/{name}", tar, size, sha)
with tarfile.open(tar) as t:
t.extractall(os.path.join(WORK, "llama"), filter="data")
SERVER_BIN = glob.glob(os.path.join(WORK, "llama", "**", "llama-server"), recursive=True)[0]
LLAMA_DIR = os.path.dirname(SERVER_BIN)
for b in glob.glob(os.path.join(LLAMA_DIR, "llama-*")):
os.chmod(b, 0o755)
SERVER_ENV = dict(os.environ)
if CUDA_LIBS:
SERVER_ENV["LD_LIBRARY_PATH"] = ":".join(CUDA_LIBS + [LLAMA_DIR, SERVER_ENV.get("LD_LIBRARY_PATH", "")])
missing = [l for l in sh(["ldd", SERVER_BIN], env=SERVER_ENV).stdout.splitlines() if "not found" in l]
assert not missing, f"unresolved libraries: {missing}"
r = sh([SERVER_BIN, "--version"], env=SERVER_ENV)
ver = re.search(r"version: .*", r.stdout + r.stderr).group(0)
print(f"llama.cpp: {name} (SHA-256 checked) -> {ver}")
for rel, digest in BFCL_FILES.items():
dest = os.path.join(WORK, "bfcl", rel.replace("/", "__"))
fetch(GORILLA + rel, dest, sha=digest)
print(f"BFCL v4: {len(BFCL_FILES)} files from gorilla@6ea5797 (5 task files, 4 answer files, 5 checker sources), all SHA-256 checked")
for tag, m in MODELS.items():
path = os.path.join(WORK, "models", m["file"])
had = os.path.exists(path) and os.path.getsize(path) == m["size"]
url = f"https://huggingface.co/{m['repo']}/resolve/{m['rev']}/{m['file']}"
dl = fetch(url, path, m["size"]) # size check now, hash below
t0 = time.time()
ok = sha256_file(path) == m["sha256"]
how = "already on disk" if had else f"downloaded in {dl:.0f} s ({m['size'] / dl / 1e6:.0f} MB/s)"
print(f"{tag:7s}: {m['file']} {m['size']:,} bytes, {how}; SHA-256 {'matches' if ok else 'DOES NOT MATCH'} "
f"the Hub's LFS pin ({time.time() - t0:.0f} s to hash)")
claim(f"{m['file']} is byte-identical to the file pinned at {m['repo']}@{m['rev'][:7]}", ok)
assert ok
return f"llama.cpp {LLAMA_TAG}; Saluki 7.90 GB and ISTA IQ2_XS 8.42 GB verified by SHA-256"
download_step()
We download the CUDA build of llama.cpp b11565, both 2-bit files and the BFCL data, and check every byte against its pin. The Hugging Face downloads are resumable, and the hashes are the Hub’s own LFS digests at a fixed revision, so a silently replaced file would stop the notebook rather than change its results. BFCL’s task files, answer files, and checker code all come from one gorilla commit. The cell prints how long the downloads took, which is the slowest part of the run on Colab.
GGML_TYPES = {0: ("F32", 1, 4), 1: ("F16", 1, 2), 8: ("Q8_0", 32, 34), 10: ("Q2_K", 256, 84), 12: ("Q4_K", 256, 144),
16: ("IQ2_XXS", 256, 66), 17: ("IQ2_XS", 256, 74), 18: ("IQ3_XXS", 256, 98), 19: ("IQ1_S", 256, 50),
21: ("IQ3_S", 256, 110), 22: ("IQ2_S", 256, 82), 23: ("IQ4_XS", 256, 136), 29: ("IQ1_M", 256, 56),
30: ("BF16", 1, 2)} # name, values per block, bytes per block (ggml-common.h)
def read_gguf(path):
"""GGUF v3 header: metadata, tensor table (name, dims, type, offset) and where the tensor data starts."""
f = open(path, "rb")
u = lambda fmt: struct.unpack("<" + fmt, f.read(struct.calcsize(fmt)))[0]
s = lambda: f.read(u("Q")).decode("utf-8", "replace")
scalar = {0: "B", 1: "b", 2: "H", 3: "h", 4: "I", 5: "i", 6: "f", 7: "?", 10: "Q", 11: "q", 12: "d"}
def val(t):
if t == 8:
return s()
if t == 9:
et, n = u("I"), u("Q")
return [val(et) for _ in range(n)]
return u(scalar[t])
assert f.read(4) == b"GGUF"
version, n_tensors, n_kv = u("I"), u("Q"), u("Q")
kv = {}
for _
Auto-fetched and translated by AI Gem Box. All rights belong to the original author.